Make forEach asynchronous in JavaScript

Question

I'm trying to understand the asynchronous programming Node.js but stalled on this code.

This function in their callback returns an array of files in a directory:

function openDir(path, callback) {
    path = __dirname + path;
    fs.exists(path, function (exists) {
        if (exists) {
            fs.readdir(path, function (err, files) {
                if (err) {
                    throw err;
                }
                var result = [];
                files.forEach(function (filename, index) {
                    result[index] = filename;
                });
                return callback(result);
            });
        }
    });
}

But when I use asynchronous code inside.forEach, it returns nothing:

function openDir(path, callback) {
    path = __dirname + path;
    fs.exists(path, function (exists) {
        if (exists) {
            fs.readdir(path, function (err, files) {
                if (err) {
                    throw err;
                }
                var result = [];
                files.forEach(function (filename, index) {
                    fs.stat(path + filename, function (err, stats) {
                        if (err) {
                            throw err;
                        }
                        result[index] = filename;
                    });
                });
                return callback(result);
            });
        }
    });
}

I understand why it happens, but don't understand how to write correct code.

Answer 1

try this:

function openDir(path, callback) {
    path = __dirname + path;
    fs.exists(path, function (exists) {
        var totalFiles = 0;;
        if (exists) {
            fs.readdir(path, function (err, files) {
                if (err) {
                    throw err;
                }
                var result = [];
                files.forEach(function (filename, index) {
                    fs.stat(path + filename, function (err, stats) {
                        if (err) {
                            throw err;
                        }
                        result[index] = filename;
                        totalFiles++;
                        if(totalFiles === files.length){
                            callback(result);
                        }
                    });
                });
            });
        }
    });
}

you can also use the Async module, to help on these kinds of situations

Answer 2

The other answers may work well, but they are currently quite different semantically from the original code: they both execute stats in parallel, rather than sequentially. The forEach will initiate as many asynchronous stats operation as there are files in the list of files. The completion order of those operations may quite well be different from the original order of the list. This may substantially affect the error handling logic.

The following approach implements a state machine, which is aimed to executes stats asynchronously, yet sequentially (untested):

function openDir(path, callback) {
    path = __dirname + path;
    fs.exists(path, function (exists) {
        if (!exists)
            callback(null, null); // node (err, result) convention
        else {
            fs.readdir(path, function (err, files) {
                if (err)
                    callback(err, null); // node (err, result) convention
                else {
                    var results = [];
                    var i = 0;
                    nextStep(); // process the first file (the first step)

                    function nextStep() {
                        if (i >= files.length) // no more files?
                            callback(null, result); // node (err, result) convention
                        else {
                            fs.stat(path + files[i], function (err, stats) {
                                if (err)
                                    callback(err, null); // node (err, result) convention
                                else {
                                    results[i++] = stats;
                                    // proceed to the next file
                                    nextStep();
                                }
                            });
                        }
                    }
                }
            }
        }
    });                   
});

Promises may help to reduce the nesting level of the famous "Pyramid of Doom" like above.

Answer 3

The issue is that fs.stat is also async, but you could probably do something like:

var result = [],
    expectedLoadCount = files.length,
    loadCount = 0;

files.forEach(function (filename, index) {
    fs.stat(path + filename, function (err, stats) {
        if (err) {
            throw err;
        }
        result[index] = filename;
        if (++loadCount === expectedLoadCount) callback(result);
    });
});