best way to get folder and file list in Javascript

Question

I'm using node-webkit, and am trying to have a user select a folder, and I'll return the directory structure of that folder and recursively get its children.

I've got this working fairly simply with this code (in an Angular Controller).

var fs = require('fs');

    $scope.explorer=[];
    $scope.openFile = function(){
            $scope.explorer = [tree_entry($scope.path)];    
            get_folder($scope.path, $scope.explorer[0].children);
    };

    function get_folder(path, tree){
        fs.readdir(path, function(err,files){
            if (err) return console.log(err);

            files.forEach( function (file,idx){
                tree.push(tree_entry(file));
                fs.lstat(path+'/'+file,function(err,stats){
                    if(err) return console.log(err);
                    if(stats.isDirectory()){
                        get_folder(path+'/'+file,tree[idx].children);
                    }
                });
            });
        });
        console.log($scope.explorer);

        return;
    }

    function tree_entry(entry){
        return { label : entry, children: []}
    }

Taking a moderate sized folder with 22 child folders and about 4 levels deep, it is taking a few minutes to get the entire directory structure.

Is there something that I'm obviously doing wrong here? I can't believe it takes that long, seeing as I'm using the built in Node fs methods. Or is there a way to get the entire contents of a directory without touching each and every file?

I'm going to want to be able to use an Angular filter on the file names all the way down the tree, and possibly on the contents too, so delaying processing the entire tree isn't likely a solution that would work.

Answer 1

I don't like adding new package into my project just to handle this simple task.

And also, I try my best to avoid RECURSIVE algorithm.... since, for most cases it is slower compared to non Recursive one.

So I made a function to get all the folder content (and its sub folder).... NON-Recursively

var getDirectoryContent = function(dirPath) {
    /* 
        get list of files and directories from given dirPath and all it's sub directories
        NON RECURSIVE ALGORITHM
        By. Dreamsavior
    */
    var RESULT = {'files':[], 'dirs':[]};

    var fs = fs||require('fs');
    if (Boolean(dirPath) == false) {
        return RESULT;
    }
    if (fs.existsSync(dirPath) == false) {
        console.warn("Path does not exist : ", dirPath);
        return RESULT;
    }

    var directoryList = []
    var DIRECTORY_SEPARATOR = "\\";
    if (dirPath[dirPath.length -1] !== DIRECTORY_SEPARATOR) dirPath = dirPath+DIRECTORY_SEPARATOR;

    directoryList.push(dirPath); // initial

    while (directoryList.length > 0) {
        var thisDir  = directoryList.shift(); 
        if (Boolean(fs.existsSync(thisDir) && fs.lstatSync(thisDir).isDirectory()) == false) continue;

        var thisDirContent = fs.readdirSync(thisDir);
        while (thisDirContent.length > 0) { 
            var thisFile  = thisDirContent.shift(); 
            var objPath = thisDir+thisFile

            if (fs.existsSync(objPath) == false) continue;
            if (fs.lstatSync(objPath).isDirectory()) { // is a directory
                let thisDirPath = objPath+DIRECTORY_SEPARATOR; 
                directoryList.push(thisDirPath);
                RESULT['dirs'].push(thisDirPath);

            } else  { // is a file
                RESULT['files'].push(objPath); 

            } 
        } 

    }
    return RESULT;
}

the only drawback of this function is that this is Synchronous function... You have been warned ;)

Answer 2

Why to invent the wheel?

There is a very popular NPM package, that let you do things like that easy.

var recursive = require("recursive-readdir");

recursive("some/path", function (err, files) {
  // `files` is an array of file paths
  console.log(files);
});

Lear more:

• https://www.npmjs.com/package/recursive-readdir

Answer 3

In my project I use this function for getting huge amount of files. It's pretty fast (put require("FS") out to make it even faster):

var _getAllFilesFromFolder = function(dir) {

    var filesystem = require("fs");
    var results = [];

    filesystem.readdirSync(dir).forEach(function(file) {

        file = dir+'/'+file;
        var stat = filesystem.statSync(file);

        if (stat && stat.isDirectory()) {
            results = results.concat(_getAllFilesFromFolder(file))
        } else results.push(file);

    });

    return results;

};

usage is clear:

_getAllFilesFromFolder(__dirname + "folder");