Node.js Api calls in an Async loop

Question

I'm having difficulty making multiple api calls and getting the returned results in the right place. This what I'm trying to achieve: Two loops, nested. The outer loop loops over a complex json object and places some object's values in an array call box. The inner loop calls the api and the places the returned results in an array named bag. So I have box and bag full of data. When both loops and api calls end, I would like to have access to the data in both box and bag and do stuff with it. I'm just not sure of the best way to go have access to both arrays at the same point in execution. This is what I have so far, but of course bag and box are empty because they get called before all the loops and api calls end.

var data = {}//a json object with data;
var bag = [];
var box = [];

async.forEach(data.item, function(item, callback){

    async.forEach(item.name, function(name, callback){

        callApi(item.name, function(obj){

           bag.push(obj);
        });

        callback();

    }

   box.push(item);
   callback();

});


function callApi(name, callback){

    asyncApiCall(url+name, function(err, result){

        callback(result);       
    });

}

//when all loops and api requests are done
//do stuff with contents of box and bag here;
console.log(bag, box);

Answer 1

You can use a counter to ensure all async calls have returned and then call a callback. An example:

function performAsyncOperation(data, onCompletion) {
    var bag = [];
    var box = [];
    var count  = 0;

    async.forEach(data.item, function(item, callback){
        async.forEach(item.name, function(name, callback){
            count += 1;
            callApi(item.name, function(obj){
               bag.push(obj);
               count -= 1;
               if (count == 0) {
                   onCompletion(null, { bag:bag, box:box });
               }
            });
            callback();
        }
       box.push(item);
       callback();
    });
}

count and onCompletion are the important parts above.

Answer 2

Since you have the full dataset, you should first (synchronously) gather the list of names into one single array.

Then, only do one level of async.forEach. Trying to do this with two level is unnecessary and results in more complex code.

If what you refer to as async is what you get using npm install async, here's a piece of code that illustrates what you could use (uses lodash to build the list of names, as this tends to be more consise.

var _ = require("lodash");
var async = require("async");

var data = {
  item: [
    {name: "a"},
    {name: "b"}
  ]
};

var bag = [];

var items = _(_(data.item).collect(function(item) {
  return item.name;
})).value();

async.eachSeries(items, function(name, cb) {
  callApi(name, function(obj) {
    bag.push(obj);
    cb();
  });
}, function done() {
  console.log("bag", bag);
});

function callApi(name, cb) {
  cb(name);
}

And the output is:

bag [ 'a', 'b' ]