NodeJS: Confusion about async “readdir” and “stat”

Question

In the docs it shows two versions of readdir and stat. Both of which have an async and sync version readir/readdirSync and stat/statSync.

Because readidir and stat are async I would expect them to return a Promise but when trying to use async/await the script doesnt wait for readdir to resolve and if I use .then/.catch I get an error cannot read .then of undefined.

All I'm trying to do here is map the directories that exist inside of the directory the script is being ran inside of to the dirsOfCurrentDir map.

Returns error cannot read .then of undefined

const fs = require('fs');

const directory = `${ __dirname }/${ process.argv[2] }`;
const dirsOfCurrentDir = new Map();

fs.readdir(directory, (err, files) => {
  let path;

  if (err)
    return console.log(err);

  files.forEach(file => {
    path = directory + file;

    fs.stat(path, (err, stats) => {
      if (err)
        return console.log(err);

      dirsOfCurrentDir.set(file, directory);
    });
  });
}).then(() => console.log('adasdasd'))

console.log(dirsOfCurrentDir)

Returns Map {}

const foo = async () => {
  await fs.readdir(directory, (err, files) => {
    let path;

    if (err)
      return console.log(err);

    files.forEach(file => {
      path = directory + file;

      fs.stat(path, (err, stats) => {
        if (err)
          return console.log(err);

        dirsOfCurrentDir.set(file, directory);
      });
    });
  });
};

foo()
console.log(dirsOfCurrentDir)

Edit

I ended up going with the synchronous versions of both of these functions readdirSync and statSync. While I would feel better using the async methods or promisify I still have not figured out how to get my code working correctly using either.

const fs = require('fs');

const directory = `${ __dirname }/${ process.argv[2] }`;
const dirsOfCurrentDir = new Map();

const dirContents = fs.readdirSync(directory);

dirContents.forEach(file => {
  const path = directory + file;
  const stats = fs.statSync(path);

  if (stats.isDirectory())
    dirsOfCurrentDir.set(file, path);
});

console.log(dirsOfCurrentDir); // logs out the map with all properties set

Answer 1

Because readidir and stat are async I would expect them to return a Promise

First off, make sure you know the difference between an asynchronous function and an async function. A function declared as async using that specific keyword in Javascript such as:

async function foo() {
    ...
}

does always return a promise (per the definition of a function declared with the async keyword).

But an asynchronous function such as fs.readdir() may or may not return a promise, depending upon its internal design. In this particular case, the original implementation of the fs module in node.js only uses callbacks, not promises (its design predates the existence of promises in node.js). Its functions are asynchronous, but not declared as async and thus is uses regular callbacks, not promises.

So, you have to either use the callbacks or "promisify" the interface to convert it into something that returns a promise so you can use await with it.

There is an experimental interface in node.js v10 that offers built-in promises for the fs module.

const fsp = require('fs').promises;

fsp.readdir(...).then(...)

---

There are lots of options for promisifying functions in an earlier version of node.js. You can do it function by function using util.promisify():

const promisify = require('util').promisify;
const readdirP = promisify(fs.readdir);
const statP = promisify(fs.stat);

---

Since I'm not yet developing on node v10, I often use the Bluebird promise library and promisify the whole fs library at once:

const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs'));

fs.readdirAsync(...).then(...)

---

To just list the sub-directories in a given directory, you could do this:

const fs = require('fs');
const path = require('path');
const promisify = require('util').promisify;
const readdirP = promisify(fs.readdir);
const statP = promisify(fs.stat);

const root = path.join(__dirname, process.argv[2]);

// utility function for sequencing through an array asynchronously
function sequence(arr, fn) {
    return arr.reduce((p, item) => {
        return p.then(() => {
            return fn(item);
        });
    }, Promise.resolve());
}

function listDirs(rootDir) {
    const dirsOfCurrentDir = new Map();
    return readdirP(rootDir).then(files => {
        return sequence(files, f => {
            let fullPath = path.join(rootDir, f);
            return statP(fullPath).then(stats => {
                if (stats.isDirectory()) {
                    dirsOfCurrentDir.set(f, rootDir)
                }
            });
        });
    }).then(() => {
        return dirsOfCurrentDir;
    });  
}

listDirs(root).then(m => {
    for (let [f, dir] of m) {
        console.log(f);
    }
});

---

Here's a more general implementation that lists files and offers several options for both what to list and how to present the results:

const fs = require('fs');
const path = require('path');
const promisify = require('util').promisify;
const readdirP = promisify(fs.readdir);
const statP = promisify(fs.stat);

const root = path.join(__dirname, process.argv[2]);

// options takes the following:
//     recurse: true | false - set to true if you want to recurse into directories (default false)
//     includeDirs: true | false - set to true if you want directory names in the array of results
//     sort: true | false - set to true if you want filenames sorted in alpha order
//     results: can have any one of the following values
//              "arrayOfFilePaths" - return an array of full file path strings for files only (no directories included in results)
//              "arrayOfObjects" - return an array of objects {filename: "foo.html", rootdir: "//root/whatever", full: "//root/whatever/foo.html"}

// results are breadth first

// utility function for sequencing through an array asynchronously
function sequence(arr, fn) {
    return arr.reduce((p, item) => {
        return p.then(() => {
            return fn(item);
        });
    }, Promise.resolve());
}

function listFiles(rootDir, opts = {}, results = []) {
    let options = Object.assign({recurse: false, results: "arrayOfFilePaths", includeDirs: false, sort: false}, opts);

    function runFiles(rootDir, options, results) {
        return readdirP(rootDir).then(files => {
            let localDirs = [];
            if (options.sort) {
                files.sort();
            }
            return sequence(files, fname => {
                let fullPath = path.join(rootDir, fname);
                return statP(fullPath).then(stats => {
                    // if directory, save it until after the files so the resulting array is breadth first
                    if (stats.isDirectory()) {
                        localDirs.push({name: fname, root: rootDir, full: fullPath, isDir: true});
                    } else {
                        results.push({name: fname, root: rootDir, full: fullPath, isDir: false});
                    }
                });
            }).then(() => {
                // now process directories
                if (options.recurse) {
                    return sequence(localDirs, obj => {
                        // add directory to results in place right before its files
                        if (options.includeDirs) {
                            results.push(obj);
                        }
                        return runFiles(obj.full, options, results);
                    });
                } else {
                    // add directories to the results (after all files)
                    if (options.includeDirs) {
                        results.push(...localDirs);
                    }
                }
            });
        });
    }

    return runFiles(rootDir, options, results).then(() => {
        // post process results based on options
        if (options.results === "arrayOfFilePaths") {
            return results.map(item => item.full);
        } else {
            return results;
        }
    });
}

// get flat array of file paths, 
//     recursing into directories, 
//     each directory sorted separately
listFiles(root, {recurse: true, results: "arrayOfFilePaths", sort: true, includeDirs: false}).then(list => {
    for (const f of list) {
        console.log(f);
    }
}).catch(err => {
    console.log(err);
});

You can copy this code into a file and run it, passing . as an argument to list the directory of the script or any subdirectory name you want to list.

If you wanted fewer options (such as no recursion or directory order not preserved), this code could be reduced significantly and perhaps made a little faster (run some async operations in parallel).