Splitting up 100,000,000 records to be processed 100,000 at a time
myDataTable <- <100,000,000 records>
chunk <- 100000
iterator <- idiv(nrow(myDataTable), chunkSize=chunk)
startRow <- 1
endRow <- chunk
try(
for (i in 1:10000) {
myChunkOfData <- myDataTable[startRow:endRow]
startRow <- endRow + 1
endRow <- endRow + nextElem(iterator)
}
, silent = TRUE)