I just did a similar thing and this is what I have to see if the checkbox in the first column is checked. This assumes you are initializing using DataTable and not dataTable as the former returns the API and the latter returns the jQuery type object

Initialze the table

var dtAPI = $('[id$=products-table]').DataTable();

Function that logs the various stuff for debugging and adds the data-id attribute to an array

function printSelected(){
            var selectedPBEIds = [];
            dtAPI.rows().every( function ( rowIdx, tableLoop, rowLoop ) {
                var cellZero = dtAPI.cell(rowIdx,0).node();
                if($("input", cellZero).prop('checked')){
                    console.log($("input",cellZero).attr('data-id'));
                    console.log($(dtAPI.cell(rowIdx,1).node()).attr('data-label'));
                    selectedPBEIds.push($("input",cellZero).attr('data-id'));
                }
            });


        }

First column of table with checkbox is as follows

                <th scope="row">
                    <div>
                        <apex:inputCheckbox html-data-id="{!p.pbe.Id}" value="{!p.isSelected}"/>
                    </div>
                </th>

Datatables is great and has great documentation although it is tough to find what you need if you do not know what you need as it is written like you already understand the API.

I just did a similar thing and this is what I have to see if the checkbox in the first column is checked. This assumes you are initializing using DataTable and not dataTable as the former returns the API and the latter returns the jQuery type object

It is important to note that it is tough to give you an exact answer as a lot of it depends on the structure of your table and td contents. Use this as a starting point

Initialze the table

var dtAPI = $('[id$=products-table]').DataTable();

Function that logs the various stuff for debugging and adds the data-id attribute to an array

function printSelected(){
            var selectedPBEIds = [];
            dtAPI.rows().every( function ( rowIdx, tableLoop, rowLoop ) {
                var cellZero = dtAPI.cell(rowIdx,0).node();
                if($("input", cellZero).prop('checked')){
                    console.log($("input",cellZero).attr('data-id'));
                    console.log($(dtAPI.cell(rowIdx,1).node()).attr('data-label'));
                    selectedPBEIds.push($("input",cellZero).attr('data-id'));
                }
            });


        }

First column of table with checkbox is as follows

                <th scope="row">
                    <div>
                        <apex:inputCheckbox html-data-id="{!p.pbe.Id}" value="{!p.isSelected}"/>
                    </div>
                </th>

Datatables is great and has great documentation although it is tough to find what you need if you do not know what you need as it is written like you already understand the API.