Handling asynchronous database queries in node.js

Question

I have this issue with querying mongodb asynchronously from node.js. Here is my code

var values = [];
var positives = new Array();
var negatives = new Array();

var server1 = new mongodb.Server('localhost',27017, {auto_reconnect: true});
var db1 = new mongodb.Db('clicker', server1);   

db1.open(function(err, db) {
    if(!err) {

        db1.collection('feedback', function(err, collection) {
            for (var i=0;i <5; i++) {
                collection.find(
                    {value:1},
                    {created_on: 
                        {       
                            $gte:startTime + (i*60*1000 - 30*1000),
                            $lt: startTime + (i*60*1000 + 30*1000)
                        }
                    },
                    function(err_positive, result_positive) {
                        result_positive.count(function(err, count){
                                console.log("Total matches: " + count);
                                positives[i] = count;
                        });
                    }

                );              

                collection.find(
                    {value:0},
                    {created_on: 
                        {
                            $gte:startTime + (i*60*1000 - 30*1000),
                            $lt: startTime + (i*60*1000 + 30*1000)
                        }
                    },
                    function(err_negative, result_negative) {
                        result_negative.count(function(err, count){
                                console.log("Total matches: " + count);
                                negatives[i] = count;
                        });
                    }   
                );                                  
            }

        });

    } else {
        console.log('Error connecting to the database');
    }      

});

Actually, I am trying to get some values from the database. And then I need to manipulate these values. It's just that I need to subtract negative count from positive count and then initialize the value array with positivecount-negative count. Now since the results are obtained asynchronously. How am I supposed to manipulate those values and put them in the values array.

Answer 1

Before I explain further, I'd like to note that there is a bug in your code:

function(err_positive, result_positive) {
    result_positive.count(function(err, count){
        console.log("Total matches: " + count);
        positives[i] = count;  // <--- BUG: i id always 5 because it
    });                        //           is captured in a closure
}

Classic closures & loops problem. See: Please explain the use of JavaScript closures in loops

Now, how to handle asynchronous functions in loops. The basic idea is that you need to keep track of how many asynchronous calls have completed and run your code once the final call returns. For example:

var END=5;
var counter=end;
for (var i=0;i<END; i++) {
  collection.find(
    {value:1},
    {created_on: 
      {       
        $gte:startTime + (i*60*1000 - 30*1000),
        $lt: startTime + (i*60*1000 + 30*1000)
      }
    },
    (function(j){
      return function(err_positive, result_positive) {
        result_positive.count(function(err, count){
            console.log("Total matches: " + count);
            positives[j] = count;
        });

        counter--;
        if (!counter) {
          /*
           * Last result, now we have all positives.
           *
           * Add code that need to process the result here.
           *
           */
        }
      }
    })(i)
  ); 
}

However, if we keep doing this it's obvious that we'll end up creating a bunch of temporary variables and end up with horribly nested code. But this being javascript, we can encapsulate the logic for this pattern in a function. Here's my implementation of this "wait-for-all-to-complete" logic in javascript: Coordinating parallel execution in node.js

But since we're using node.js, we can use the convenient async module form npm: https://npmjs.org/package/async

With async, you can write your code like this:

var queries = [];

// Build up queries:
for (var i=0;i <5; i++) {
  queries.push((function(j){
    return function(callback) {
      collection.find(
        {value:1},
        {created_on: 
          {       
            $gte:startTime + (j*60*1000 - 30*1000),
            $lt: startTime + (j*60*1000 + 30*1000)
          }
        },
        function(err_positive, result_positive) {
          result_positive.count(function(err, count){
            console.log("Total matches: " + count);
            positives[j] = count;          
            callback();
          });
        }

      );
    }
  })(i));
  queries.push((function(j){
    return function(callback) {
      collection.find(
        {value:0},
        {created_on: 
          {
            $gte:startTime + (j*60*1000 - 30*1000),
            $lt: startTime + (j*60*1000 + 30*1000)
          }
        },
        function(err_negative, result_negative) {
          result_negative.count(function(err, count){
            console.log("Total matches: " + count);
            negatives[j] = count;
            callback();
          });
        }   
      );
    }
  })(i));  
}

// Now execute the queries:
async.parallel(queries, function(){
  // This function executes after all the queries have returned
  // So we have access to the completed positives and negatives:

  // For example, we can dump the arrays in Firebug:
  console.log(positives,negatives);
});