Hot Code Push NodeJS

Question

I've been trying to figure out this "Hot Code Push" on Node.js. Basically, my main file (that is run when you type node app.js) consists of some settings, configurations, and initializations. In that file I have a file watcher, using chokidar. When I file has been added, I simply require the file. If a file has been changed or updated I would delete the cache delete require.cache[path] and then re-require it. All these modules don't export anything, it just works with the single global Storm object.

Storm.watch = function() {
    var chokidar, directories, self = this;
    chokidar = require('chokidar');
    directories = ['server/', 'app/server', 'app/server/config', 'public'];
    clientPath = new RegExp(_.regexpEscape(path.join('app', 'client')));
    watcher = chokidar.watch(directories, {
    ignored: function(_path) {
        if (_path.match(/\./)) {
            !_path.match(/\.(js|coffee|iced|styl)$/);
        } else {
            !_path.match(/(app|config|public)/);
        }
    },
    persistent: true
    });


    watcher.on('add', function(_path){
    self.fileCreated(path.resolve(Storm.root, _path));
    //Storm.logger.log(Storm.cliColor.green("File Added: ", _path));
    //_console.info("File Updated");
    console.log(Storm.css.compile('     {name}: {file}', "" +
        "name" +
        "{" +
        "color: white;" +
        "font-weight:bold;" +
        "}" +
        "hr {"  +
        "background: grey" +
        "}")({name: "File Added", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
    });

    watcher.on('change', function(_path){
    _path = path.resolve(Storm.root, _path);
    if (fs.existsSync(_path)) {
        if (_path.match(/\.styl$/)) {
            self.clientFileUpdated(_path);
        } else {
            self.fileUpdated(_path);
        }
    } else {
        self.fileDeleted(_path);
    }
    //Storm.logger.log(Storm.cliColor.green("File Changed: ", _path));
    console.log(Storm.css.compile('     {name}: {file}', "" +
        "name" +
        "{" +
        "color: yellow;" +
        "font-weight:bold;" +
        "}" +
        "hr {"  +
        "background: grey" +
        "}")({name: "File Changed", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
    });

    watcher.on('unlink', function(_path){
    self.fileDeleted(path.resolve(Storm.root, _path));
    //Storm.logger.log(Storm.cliColor.green("File Deleted: ", _path));
    console.log(Storm.css.compile('     {name}: {file}', "" +
        "name" +
        "{" +
        "color: red;" +
        "font-weight:bold;" +
        "}" +
        "hr {"  +
        "background: grey" +
        "}")({name: "File Deleted", file: _path.replace(Storm.root, ""), hr: "=================================================="}));
    });

    watcher.on('error', function(error){
    console.log(error);
    });


};


Storm.watch.prototype.fileCreated = function(_path) {

    if (_path.match('views')) {
    return;
    }

    try {
    require.resolve(_path);
    } catch (error) {
    require(_path);
    }

};


Storm.watch.prototype.fileDeleted = function(_path) {
    delete require.cache[require.resolve(_path)];
};

Storm.watch.prototype.fileUpdated = function(_path) {
    var self = this;
    pattern = function(string) {
    return new RegExp(_.regexpEscape(string));
    };

    if (_path.match(pattern(path.join('app', 'templates')))) {
    Storm.View.cache = {};
    } else if (_path.match(pattern(path.join('app', 'helpers')))) {
    self.reloadPath(path, function(){
        self.reloadPaths(path.join(Storm.root, 'app', 'controllers'));
    });
    } else if (_path.match(pattern(path.join('config', 'assets.coffee')))) {
    self.reloadPath(_path, function(error, config) {
        //Storm.config.assets = config || {};
    });
    } else if (_path.match(/app\/server\/(models|controllers)\/.+\.(?:coffee|js|iced)/)) {
    var isController, directory, klassName, klass;

    self.reloadPath(_path, function(error, config) {
        if (error) {
            throw new Error(error);
        }
    });

    Storm.serverRefresh();

    isController = RegExp.$1 == 'controllers';
    directory    = 'app/' + RegExp.$1;
    klassName = _path.split('/');
    klassName = klassName[klassName.length - 1];
    klassName = klassName.split('.');
    klassName.pop();
    klassName = klassName.join('.');
    klassName = _.camelize(klassName);

    if (!klass) {
        require(_path);
    } else {
        console.log(_path);
        self.reloadPath(_path)
    }

    } else if (_path.match(/config\/routes\.(?:coffee|js|iced)/)) {
    self.reloadPath(_path);
    } else {
    this.reloadPath(_path);
    }

};

Storm.watch.prototype.reloadPath = function(_path, cb) {

    _path = require.resolve(path.resolve(Storm.root, path.relative(Storm.root, _path)));
    delete require.cache[_path];
    delete require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))];
    //console.log(require.cache[path.resolve(path.join(Storm.root, "server", "application", "server.js"))]);
    require("./server.js");

    Storm.App.use(Storm.router);

    process.nextTick(function(){
    Storm.serverRefresh();
    var result = require(_path);
    if (cb) {
        cb(null, result);
    }
    });
};


Storm.watch.prototype.reloadPaths = function(directory, cb) {



};

Some of the code is incomplete / not used as I'm trying a lot of different methods.

What's Working:

For code like the following:

function run() {
   console.log(123);
}

Works perfectly. But any asynchronous code fails to update.

Problem = Asynchronous Code

app.get('/', function(req, res){
   // code here..
});

If I then update the file when the nodejs process is running, nothing happens, though it goes through the file watcher and the cache is deleted, then re-established. Another instance where it doesn't work is:

// middleware.js
function hello(req, res, next) {
  // code here...
}

// another file:
app.use(hello);

As app.use would still be using the old version of that method.

Question:

How could I fix the problem? Is there something I'm missing?

Please don't throw suggestions to use 3rd party modules like forever. I'm trying to incorporate the functionality within the single instance.

EDIT:

After studying meteors codebase (there's surprisingly little resources on "Hot Code Push" in node.js or browser.) and tinkering around with my own implementation I've successfully made a working solution. https://github.com/TheHydroImpulse/Refresh.js . This is still at an early stage of development, but it seems solid right now. I'll be implementing a browser solution too, just for sake of completion.

Answer 1

Deleting require's cache doesn't actually "unload" your old code, nor does it undo what that code did.

Take for example the following function:

var callbacks=[];
registerCallback = function(cb) {
    callbacks.push(cb);
};

Now let's say you have a module that calls that global function.

registerCallback(function() { console.log('foo'); });

After your app starts up, callbacks will have one item. Now we'll modify the module.

registerCallback(function() { console.log('bar'); });

Your 'hot patching' code runs, deletes the require.cached version and re-loads the module.

What you must realize is that now callbacks has two items. First, it has a reference to the function that logs foo (which was added on app startup) and a reference to the function that logs bar (which was just added).

Even though you deleted the cached reference to the module's exports, you can't actually delete the module. As far as the JavaScript runtime is concerned, you simply removed one reference out of many. Any other part of your application can still be hanging on to a reference to something in the old module.

This is exactly what is happening with your HTTP app. When the app first starts up, your modules attach anonymous callbacks to routes. When you modify those modules, they attach a new callback to the same routes; the old callbacks are not deleted. I'm guessing that you're using Express, and it calls route handlers in the order they were added. Thus, the new callback never gets a chance to run.

---

To be honest, I wouldn't use this approach to reloading you app on modification. Most people write app initialization code under the assumption of a clean environment; you're violating that assumption by running initialization code in a dirty environment – that is, one which is already up and running.

Trying to clean up the environment to allow your initialization code to run is almost certainly more trouble than it's worth. I'd simply restart the entire app when your underlying files have changed.

Answer 2

Meteor solves this problem by allowing modules to "register" themselves as part of the hot code push process.

They implement this in their reload package:

https://github.com/meteor/meteor/blob/master/packages/reload/reload.js#L105-L109

I've seen that Meteor.reload API used in some plugins on GitHub, but they also use it in the session package:

https://github.com/meteor/meteor/blob/master/packages/session/session.js#L103-L115

if (Meteor._reload) {
  Meteor._reload.onMigrate('session', function () {
    return [true, {keys: Session.keys}];
  });

  (function () {
    var migrationData = Meteor._reload.migrationData('session');
    if (migrationData && migrationData.keys) {
      Session.keys = migrationData.keys;
    }
  })();
}

So basically, when the page/window loads, meteor runs a "migration", and it's up to the package to define the data/methods/etc. that get recomputed when a hot code push is made.

It's also being used by their livedata package (search reload).

Between refreshes they're saving the "state" using window.sessionStorage.