Recursively create directories from a multi-level JSON (with async.js)

Question

DEFINITION

File#createDirectoriesFromJSON (json, cb);

json: JSON object.

cb: Function. Parameters: error (Error), created (Boolean, true if at least one directory has been created).

Supose that the File class contains a property named _path that contains the path of a directory.

USAGE

var json = {
    b: {
        c: {
            d: {},
            e: {}
        },
        f: {}
    },
    g: {
        h: {}
    }
};

//this._path = "."
new File (".").createDirectoriesFromJSON (json, function (error, created){
    console.log (created); //Prints: true

    //callback binded to the File instance (to "this"). Hint: cb = cb.bind (this)
    this.createDirectoriesFromJSON (json, function (error, created){
        console.log (created); //Prints: false (no directory has been created)
    });
});

RESULT

Under "." the directory tree showed in the json object has been created.

./b/c/d
./b/c/e
./b/f
./b/g/h

IMPLEMENTATION

This is what I have without async.js:

File.prototype.createDirectoriesFromJSON = function (json, cb){
    cb = cb.bind (this);

    var created = false;
    var exit = false;

    var mkdir = function (path, currentJson, callback){
        var keys = Object.keys (currentJson);
        var len = keys.length;
        var done = 0;

        if (len === 0) return callback (null);

        for (var i=0; i<len; i++){
            (function (key, i){
                var dir = PATH.join (path, key);
                FS.mkdir (dir, function (mkdirError){
                    exit = len - 1 === i;

                    if (mkdirError && mkdirError.code !== "EEXIST"){
                        callback (mkdirError);
                        return;
                    }else if (!mkdirError){
                        created = true;
                    }

                    mkdir (dir, currentJson[key], function (error){
                        if (error) return callback (error);
                        done++;
                        if (done === len){
                            callback (null);
                        }
                    });
                });
            })(keys[i], i);
        }
    };

    var errors = [];

    mkdir (this._path, json, function (error){
        if (error) errors.push (error);
        if (exit){
            errors = errors.length === 0 ? null : errors;
            cb (errors, errors ? false : created);
        }
    });
};

Just for curiosity I want to rewrite the function using async.js. The problem here is that the function is recursive and parallel. For example, the "b" folder is created in parallel with "g". The same for "b/c" and "b/f", and "b/c/d" and "b/c/e".

FGRibreau · Accepted Answer · 2012-10-29 08:58:42Z

My attempt:

var _path  = require('path');
var _fs    = require('fs');
var _async = require('async');

function File() {
    this._path = __dirname + '/test';
}

File.prototype.createDirectoriesFromJSON = function(json, cb) {
    var created = [], errors  = [];

    function iterator(path, currentJson, key, fn){
        var dir = _path.join(path, key);

        _fs.mkdir(dir, function(mkdirError) {

            if(mkdirError && mkdirError.code !== "EEXIST") {
                errors.push(mkdirError);
            } else if(!mkdirError) {
                created.push(dir);
            }

            mkdir(dir, currentJson[key], fn);
        });
    }

    function mkdir(path, currentJson, callback) {
        var keys = Object.keys(currentJson);

        if(keys.length === 0) return callback(null);

        _async.forEach(keys, iterator.bind(this, path, currentJson), callback);
    }

    mkdir(this._path, json, cb.bind(this, errors, created));
};


new File().createDirectoriesFromJSON({
    b: {
        c: {
            d: {},
            e: {}
        },
        f: {}
    },
    g: {
        h: {}
    }
}, function(errors, successes) {
    // errors is an array of errors
    // successes is an array of successful directory creation
    console.log.apply(console, arguments);
});

Tested with:

$ rm -rf test/* && node test.js && tree test

[] [ '/Users/fg/Desktop/test/b',
  '/Users/fg/Desktop/test/g',
  '/Users/fg/Desktop/test/b/c',
  '/Users/fg/Desktop/test/b/f',
  '/Users/fg/Desktop/test/g/h',
  '/Users/fg/Desktop/test/b/c/d',
  '/Users/fg/Desktop/test/b/c/e' ] null
test
|-- b
|   |-- c
|   |   |-- d
|   |   `-- e
|   `-- f
`-- g
    `-- h

7 directories, 0 files

Notes:

Since errors.push(mkdirError); means that the directory couldn't be created, return fn(null); could be appended to it to stop the directory creation from this branch.
cb will receive a third argument that will always be null.
I would preferably use wrench .mkdirSyncRecursive() for this kind of task, or substack async mkdirp.

[Update] Using mkdirp and lodash (or underscore) the code can be even clearer:

var _path   = require('path');
var _fs     = require('fs');

var _async  = require('async');
var _mkdirp = require('mkdirp');
var _       = require('lodash'); // or underscore

function File() {
    this._path = __dirname + '/test';
}

File.prototype.flattenJSON = function(json){
    function walk(path, o, dir){
        var subDirs = Object.keys(o[dir]);
        path +=  '/' + dir;

        if(subDirs.length === 0){
            return path;
        }

        return subDirs.map(walk.bind(null, path, o[dir]));
    }

    return _.flatten(Object.keys(json).map(walk.bind(null, this._path, json)));
};

File.prototype.createDirectoriesFromJSON = function(json, cb) {
    var paths   = this.flattenJSON(json)
    ,   created = []
    ,   errors  = [];

    function iterator(path, fn){
        _mkdirp(path, function(mkdirError) {

            if(mkdirError && mkdirError.code !== "EEXIST") {
                errors.push(mkdirError);
            } else if(!mkdirError) {
                created.push(path);
            }

            return fn(null);
        });
    }

    _async.forEach(paths, iterator, cb.bind(this, errors, created));
};


new File().createDirectoriesFromJSON({
    b: {
        c: {
            d: {},
            e: {}
        },
        f: {}
    },
    g: {
        h: {}
    }
}, function(errors, successes) {
    // errors is an array of error
    // successes is an array of successful directory creation
    console.log.apply(console, arguments);
});

Tested with:

$ rm -rf test/* && node test2.js && tree test

[] [ '/Users/fg/Desktop/test/b/f',
  '/Users/fg/Desktop/test/g/h',
  '/Users/fg/Desktop/test/b/c/d',
  '/Users/fg/Desktop/test/b/c/e' ] null
test
|-- b
|   |-- c
|   |   |-- d
|   |   `-- e
|   `-- f
`-- g
    `-- h

7 directories, 0 files

Note:

iterator could be removed by using partial function application, however underscore/lodash only support partial from left to right thus I did not wanted to require another library to do so.

Wow! very nice examples. I also thought the second approach but it does more loops than the first one because it has to calculate the full paths. After errors.push(mkdirError); shouldn't be a return?
If you talk about the second example then no because it would block the async.forEach (the first iteration that call "fn" with an error will immediately stop the forEach and call the final callback)

logical Chimp · Accepted Answer · 2012-10-27 11:25:02Z

Don't know if it is the best solution, but i've solved this in the past by creating an additional 'monitor' type object. In short, change your initialisation to something like:

var monitor = {
    var init = function(json, cb) {
        this.outerDirLength = Object.keys (currentJson);
        this.processedOuterDirs = 0; //track 'report progress'  calls
        this.completedOuterDirs = 0; //'report progress' calls with no errors = success
        this.errors = [];
        this.finishedCallback = cb;
    }
    var reportProgress = function(error) {
        this.processedOuterDirs++;
        if (error) this.errors.push(error);
        else this.completedOuterDirs++;
        if (this.isComplete()) this.finish();
    }
    var finish = function () {
        var errors = this.errors.length === 0 ? null : this.errors;
        this.finishedCallback(errors, errors ? false : this.completedOuterDirs);
    }
    var isComplete = function() {
        return this.processedOuterDirs == this.outerDirLength;
    }
};
monitor.init(json, cb);
if (monitor.isComplete()) {
    //handle case of JSON with zero definitions
    monitor.finish();
    return;
}
mkdir (this._path, json, function (error){
    monitor.reportProgress(error);
});

Note that the above has not been tested (or even test-compiled) but should give you the idea... If you were to make mkdir truely async, you'd probably change the above so that at the start of each call to mkdir it calculated how many dirs to create, and incremented the expected target on the monitor, and then updated the monitor as each one is created.

Collectives™ on Stack Overflow

Recursively create directories from a multi-level JSON (with async.js)

2 Answers 2

2 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

2 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related