I'm trying to figure out the fastest way without affecting performance much while moving either synchronously or asynchronously 100.000+ files using Node.js
I made sync and async tests with 3 different looping forEach
async.each
and for
. The loop iterations didn't impact timing results much. And the heaviest logic affecting performance and speed is moving files either synchronously or asynchronously.
The test cases showed, that synchronous renameSync()
is slower at 20% then asynchronous rename()
. In the same time asynchronous rename()
using 3 times more CPU power, then synchronous renameSync()
.
Am I right to assume that it would be wiser to use synchronous renameSync()
because speed boost in asynchronous rename()
is not so significant and equal to 20%. While CPU utilization overhead in async version seems to be huge - 300%?
Or did I miss anything? Maybe there are better workarounds in terms of speed/performance.
The timings:
eachAsyncRenameAsync()
node rename.js 1.60s user 11.20s system 280% cpu 4.559 total
node rename.js 1.65s user 11.82s system 284% cpu 4.732 total
node rename.js 1.64s user 11.84s system 292% cpu 4.606 total
eachSyncRenameSync()
node rename.js 0.69s user 5.01s system 97% cpu 5.851 total
node rename.js 0.67s user 4.88s system 97% cpu 5.687 total
node rename.js 0.68s user 5.01s system 99% cpu 5.734 total
eachAsyncRenameSync()
node rename.js 0.67s user 4.99s system 97% cpu 5.797 total
node rename.js 0.68s user 4.95s system 97% cpu 5.754 total
node rename.js 0.66s user 4.89s system 97% cpu 5.690 total
eachSyncRenameAsync()
node rename.js 1.63s user 11.12s system 274% cpu 4.638 total
node rename.js 1.66s user 12.29s system 286% cpu 4.874 total
node rename.js 1.66s user 12.23s system 289% cpu 4.795 total
forSyncRenameAsync()
node rename.js 1.72s user 12.04s system 283% cpu 4.862 total
node rename.js 1.69s user 11.88s system 276% cpu 4.904 total
node rename.js 1.64s user 11.89s system 287% cpu 4.712 total
forSyncRenameSync()
node rename.js 0.64s user 4.94s system 97% cpu 5.715 total
node rename.js 0.66s user 5.01s system 97% cpu 5.807 total
node rename.js 0.65s user 4.93s system 99% cpu 5.616 total
The code:
const fs = require('fs')
const path = require('path')
const { each } = require('async')
let dir = '/opt/bin/test/Random 100000'
let dir2 = '/opt/bin/test/Random 100000 2'
function eachAsyncRenameAsync(files) {
each(files, file => {
fs.rename(path.join(dir,file), path.join(dir2,file), err => {
if(err) { console.log(err) }
})
})
}
function eachSyncRenameSync(files) {
files.forEach(file => {
fs.renameSync(path.join(dir,file), path.join(dir2,file))
})
}
function eachAsyncRenameSync(files) {
each(files, file => {
fs.renameSync(path.join(dir,file), path.join(dir2,file))
})
}
function eachSyncRenameAsync(files) {
files.forEach(file => {
fs.rename(path.join(dir,file), path.join(dir2,file), err => {
if(err) { console.log(err) }
})
})
}
function forSyncRenameAsync(files) {
for (i=0; i<files.length; i++) {
fs.rename(path.join(dir, files[i]), path.join(dir2, files[i]), err => {
if(err) { console.log(err) }
})
}
}
function forSyncRenameSync(files) {
for (i=0; i<files.length; i++) {
fs.renameSync(path.join(dir, files[i]), path.join(dir2, files[i]))
}
}
// Reading dir asynchronously and moving files
fs.readdir(dir, (err, files) => {
if (err) { console.log(err) }
else {
console.log('eachAsyncRenameAsync()')
eachAsyncRenameAsync(files)
}
})