Callback hell in nodejs?

Question

In below code am I in callbackhell? How to overcome such scenario without using any async modules in pure javascript?

emailCallBack(e_data, email);
if (email_list.length) {
  checkEmail(email_list.pop());
} else {
  completionCallback();
}

The above code is copied in multiple location to make code work as expected.

function processInviteEmails(email_list, user_id, emailCallBack, completionCallback){
      function checkEmail(email){
        try {
          check(email).isEmail();
          //is valid email
          checkConnected(email, user_id, function(connect_status, user_row, user_meta_row, connect_row){
            var e_data;
            //insert to connect and send msg to queue
            if(connect_status === 'not connected'){
              var cur_date = moment().format('YYYY-MM-DD');
              var dbData = {
                "first_name": '',
                "last_name": '',
                "email": email,
                "user_id": user_id,
                "status": "invited",
                "unsubscribe_token": crypto.randomBytes(6).toString('base64'),
                "created": cur_date,
                "modified": cur_date
              };
              ConnectModel.insert(dbData, function(result){
                if (result.insertId > 0) {
                  //send to email queue
                  //Queue Email
                  MailTemplateModel.getTemplateData('invitation', function(res_data){
                    if(res_data.status === 'success'){
                      var unsubscribe_hash = crypto.createHash("md5")
                        .update(dbData.unsubscribe_token + email)
                        .digest('hex');
                      var unsubscribe_link = app.locals.SITE_URL+'/unsubscribe/' + result.insertId + '/' + unsubscribe_hash;
                      var template_row = res_data.template_row;
                      var user_full_name = user_row.user_firstname+' '+ user_row.user_lastname;
                      var invitation_link = 'http://'+user_row.url_alias+'.'+ app.locals.SITE_DOMAIN;
                      var mailOptions = {
                        "type": 'invitation',
                        "to": dbData.email,
                        "from_name" : user_full_name,
                        "subject": template_row.message_subject
                          .replace('[[USER]]',  user_full_name),
                        "text": template_row.message_text_body
                          .replace('[[USER]]', user_full_name)
                          .replace('[[INVITATION_LINK]]', invitation_link)
                          .replace('[[UNSUBSCRIBE_LINK]]', unsubscribe_link),
                        "html": template_row.message_body
                          .replace('[[USER]]', user_full_name)
                          .replace('[[INVITATION_LINK]]', invitation_link)
                          .replace('[[UNSUBSCRIBE_LINK]]', unsubscribe_link)
                      };
                      mailOptions = JSON.stringify(mailOptions);
                      //send email to queue
                      sqsHelper.addToQueue(cfg.sqs_invitation_url, mailOptions, function(data){
                        if(data){
                          e_data = null;
                        }
                        else{
                          e_data = new Error('Unable to Queue ');
                        }
                        emailCallBack(e_data, email);
                        if (email_list.length) {
                          checkEmail(email_list.pop());
                        } else {
                          completionCallback();
                        }
                      });
                    }
                    else{
                      e_data = new Error('Unable to get email template');
                      emailCallBack(e_data, email);
                      if (email_list.length) {
                        checkEmail(email_list.pop());
                      } else {
                        completionCallback();
                      }
                    }
                  });
                }
                else{
                  e_data = new Error('Unable to Insert connect');
                  emailCallBack(e_data, email);
                  if (email_list.length) {
                    checkEmail(email_list.pop());
                  } else {
                    completionCallback();
                  }
                }
              });
            }
            else{
              e_data = new Error('Already connected');
              emailCallBack(e_data, email);
              if (email_list.length) {
                checkEmail(email_list.pop());
              } else {
                completionCallback();
              }
            }
          });
        } catch (e) {
          //invalid email
          emailCallBack(e, email);
          if (email_list.length) {
            checkEmail(email_list.pop());
          } else {
            completionCallback();
          }
        }
      }
      checkEmail(email_list.pop());
    }

Answer 1

"If you try to code bussiness db login using pure node.js, you go straight to callback hell"

I've recently created a simple abstraction named WaitFor to call async functions in sync mode (based on Fibers): https://github.com/luciotato/waitfor

check the database example:

Database example (pseudocode)

pure node.js (mild callback hell):

var db = require("some-db-abstraction");

function handleWithdrawal(req,res){  
    try {
        var amount=req.param("amount");
        db.select("* from sessions where session_id=?",req.param("session_id"),function(err,sessiondata) {
            if (err) throw err;
            db.select("* from accounts where user_id=?",sessiondata.user_ID),function(err,accountdata) {
                if (err) throw err;
                    if (accountdata.balance < amount) throw new Error('insufficient funds');
                    db.execute("withdrawal(?,?),accountdata.ID,req.param("amount"), function(err,data) {
                        if (err) throw err;
                        res.write("withdrawal OK, amount: "+ req.param("amount"));
                        db.select("balance from accounts where account_id=?", accountdata.ID,function(err,balance) {
                            if (err) throw err;
                            res.end("your current balance is "  + balance.amount);
                        });
                    });
                });
            });
        }
        catch(err) {
            res.end("Withdrawal error: "  + err.message);
    }  

Note: The above code, although it looks like it will catch the exceptions, it will not. Catching exceptions with callback hell adds a lot of pain, and i'm not sure if you will have the 'res' parameter to respond to the user. If somebody like to fix this example... be my guest.

using wait.for:

var db = require("some-db-abstraction"), wait=require('wait.for');

function handleWithdrawal(req,res){  
    try {
        var amount=req.param("amount");
        sessiondata = wait.forMethod(db,"select","* from session where session_id=?",req.param("session_id"));
        accountdata= wait.forMethod(db,"select","* from accounts where user_id=?",sessiondata.user_ID);
        if (accountdata.balance < amount) throw new Error('insufficient funds');
        wait.forMethod(db,"execute","withdrawal(?,?)",accountdata.ID,req.param("amount"));
        res.write("withdrawal OK, amount: "+ req.param("amount"));
        balance=wait.forMethod(db,"select","balance from accounts where account_id=?", accountdata.ID);
        res.end("your current balance is "  + balance.amount);
        }
    catch(err) {
        res.end("Withdrawal error: "  + err.message);
}  

Note: Exceptions will be catched as expected. db methods (db.select, db.execute) will be called with this=db

Your Code

In order to use wait.for, you'll have to STANDARDIZE YOUR CALLBACKS to function(err,data)

If you STANDARDIZE YOUR CALLBACKS, your code might look like:

//run in a Fiber
function processInviteEmails(email_list, user_id, emailCallBack, completionCallback){

    while (email_list.length) {

      var email = email_list.pop();

      try {

          check(email).isEmail(); //is valid email or throw

          var connected_data = wait.for(checkConnected,email,user_id);
          if(connected_data.connect_status !== 'not connected') throw new Error('Already connected');

          //insert to connect and send msg to queue
          var cur_date = moment().format('YYYY-MM-DD');
          var dbData = {
            "first_name": '',
            "last_name": '',
            "email": email,
            "user_id": user_id,
            "status": "invited",
            "unsubscribe_token": crypto.randomBytes(6).toString('base64'),
            "created": cur_date,
            "modified": cur_date
          };

          result = wait.forMethod(ConnectModel,'insert',dbData);
          // ConnectModel.insert shuold have a fn(err,data) as callback, and return something in err if (data.insertId <= 0) 

          //send to email queue
          //Queue Email
          res_data = wait.forMethod(MailTemplateModel,'getTemplateData','invitation');
          // MailTemplateModel.getTemplateData shuold have a fn(err,data) as callback
          // inside getTemplateData, callback with err=new Error('Unable to get email template') if (data.status !== 'success') 

          var unsubscribe_hash = crypto.createHash("md5")
            .update(dbData.unsubscribe_token + email)
            .digest('hex');
          var unsubscribe_link = app.locals.SITE_URL+'/unsubscribe/' + result.insertId + '/' + unsubscribe_hash;
          var template_row = res_data.template_row;
          var user_full_name = user_row.user_firstname+' '+ user_row.user_lastname;
          var invitation_link = 'http://'+user_row.url_alias+'.'+ app.locals.SITE_DOMAIN;
          var mailOptions = {
            "type": 'invitation',
            "to": dbData.email,
            "from_name" : user_full_name,
            "subject": template_row.message_subject
              .replace('[[USER]]',  user_full_name),
            "text": template_row.message_text_body
              .replace('[[USER]]', user_full_name)
              .replace('[[INVITATION_LINK]]', invitation_link)
              .replace('[[UNSUBSCRIBE_LINK]]', unsubscribe_link),
            "html": template_row.message_body
              .replace('[[USER]]', user_full_name)
              .replace('[[INVITATION_LINK]]', invitation_link)
              .replace('[[UNSUBSCRIBE_LINK]]', unsubscribe_link)
          };
          mailOptions = JSON.stringify(mailOptions);
          //send email to queue ... callback(err,data)
          wait.forMethod(sqsHelper,'addToQueue',cfg.sqs_invitation_url, mailOptions); 

      } catch (e) {
          // one of the callback returned err!==null 
          emailCallBack(e, email);
      }

    } // loop while length>0

    completionCallback();

  }

  // run the loop in a Fiber (keep node spinning)
  wait.launchFiber(processInviteEmails,email_list, user_id, emailCallBack, completionCallback);

see? no callback hell

Answer 2

I've put another solution in my blog. It is ugly but it is the most readable thing I could do with pure javascript.

var flow1 = new Flow1(
    {
        execute_next_step: function(err) {
            if (err) {
                console.log(err);
            };
        }
    }
);

flow1.execute_next_step();

function Flow1(parent_flow) {
    this.execute_next_step = function(err) {
        if (err) return parent_flow.execute_next_step(err);
        if (!this.next_step) this.next_step = 'START';
        console.log('Flow1:', this.next_step);
        switch (this.next_step) {
            case 'START':
                this.next_step = 'FIRST_ASYNC_TASK_FINISHED';
                firstAsyncTask(this.execute_next_step.bind(this));
                break;
            case 'FIRST_ASYNC_TASK_FINISHED':
                this.firstAsyncTaskReturn = arguments[1];
                this.next_step = 'ANOTHER_FLOW_FINISHED';
                this.another_flow = new AnotherFlow(this);
                this.another_flow.execute_next_step();
                break;
            case 'ANOTHER_FLOW_FINISHED':
                this.another_flow_return = arguments[1];
                this.next_step = 'FINISH';
                this.execute_next_step();
                break;
            case 'FINISH':
                parent_flow.execute_next_step();
                break;
        }
    }
}

function AnotherFlow(parent_flow) {
    this.execute_next_step = function(err) {
        if (err) return parent_flow.execute_next_step(err);
        if (!this.next_step) this.next_step = 'START';
        console.log('AnotherFlow:', this.next_step);
        switch (this.next_step) {
            case 'START':
                console.log('I dont want to do anything!. Calling parent');
                parent_flow.execute_next_step();
                break;
        }
    }
}

Answer 3

Yes you are in callback hell. The solution assuming you don't want to use async (which I doubt you can justify other than prejudice) consists of:

1) Make more top-level functions. Each function should perform either 1 or 2 IO operations as a rule of thumb.

2) Call those functions, making your code follow a pattern of a long list of short core functions organized into business logic by a small list of control flow "glue" functions.

Instead of:

saveDb1 //lots of code
  saveDb2 //lots of code
    sendEmail //lots of code

Aim for:

function saveDb1(arg1, arg2, callback) {//top-level code}
function saveDb2(arg1, arg2, callback) {//top-level code}
function sendEmail(arg1, arg2, callback) {//top-level code}
function businessLogic(){//uses the above to get the work done}

3) Use more function arguments instead of relying so much on closures

4) Emit events and DECOUPLE YOUR CODE! See how you have nested code writing stuff to the database and then building an email and adding it to a queue? Don't you see how those two do not need to exist one on top of the other? Emails lend themselves very well to a core business logic emitting events and an email module listening to those events and queueing the mail.

5) Decouple application-level service connection code from specific transaction business logic. Dealing with connections to network services should be handled more broadly and not embedded with a specific set of business logic.

6) Read other modules for examples

As to should you use an async library, you can and should make up your own mind about that but AFTER you know, and know pretty well, each and every one of these approaches:

1. callbacks and basic functional javascript techniques
2. events
3. promises
4. Helper libraries (async, step, nimble, etc)

Any serious node.js developer knows how to use and work within ALL of those paradigms. Yes, everyone has their favored approach and maybe some nerd rage about the non-favored approaches, but none of these are difficult and it's bad to get set in your decision without being able to point to some non-trivial code you wrote from scratch in each paradigm. Also, you should try several helper libraries and understand how they work and why they are going to save you boilerplate. Studying the work of Tim Caswell's Step or Caolan McMahon's async is going to be very enlightening. Have you seen the everyauth source code's use of promises? I don't like it personally but I surely have to admit that the author has squeezed damn near every last bit of repetition out of that library, and the way he uses promises will turn your brain into a pretzel. These people are wizards with much to teach. Don't scoff at those libraries just for hipster points or whatever.

Also a good external resource is callbackhell.com.