Using node.js async and require to fetch multiple urls

Question

Good day -- I've conducted many queries on this subject but can't find a good example that fits my situation.

Goal: To call multiple urls using request and async (these urls are different, some are xml and some are json), with the ability to scale the amount of links, with the end result of passing the data to the view for parsing.

Code:

var express = require('express');
var router = express.Router();
var request = require('request');
var parseString = require('xml2js').parseString;
var async = require('async');

var bis;

var url_bis = {
  url: 'https://www.bis.org/list/cbspeeches/index.rss',
  headers: {
    'User-Agent': 'request'
  }
};

function callback_bis(error, response, body) {
  if (!error && response.statusCode == 200) {
    bis = "";
    parseString(body, function (err, result) {
        bis = result;
        bis = bis["rdf:RDF"]["item"];
    });
  }
}

var doj;

var url_doj = {
  url: 'http://www.justice.gov/feeds/opa/justice-news.xml',
  headers: {
    'User-Agent': 'request'
  }
};

function callback_doj (error, response, body) {
  if (!error && response.statusCode == 200) {
    doj = "";
    parseString(body, function (err, result) {
        _doj = result;
        doj = _doj.rss.channel[0].item;

    });
  }
}   

function feed_doj() {
    request(url_doj, callback_doj);
}
function feed_bis() {
    request(url_bis, callback_bis);
}

router.get('/', function(req, res, next) {

async.parallel({
    bis: feed_bis,
    doj: feed_doj

}, function(err, results) {
    console.log(results)
    // how to get the bis/doj response here in results?
});

});

module.exports = router;

I dont know the the module async, but could you not simple store the response of each request in a global variable? — gapvision
– gapvision, Commented Aug 22, 2015 at 16:50
I did attempt that on the first variation without async, but it seemed that when I would refresh the page the data would not be passed to the view unless I refreshed twice. I'm not sure why it was behaving that way. — sn4ke
– sn4ke, Commented Aug 22, 2015 at 18:00

greim · Accepted Answer · 2015-08-22 17:33:58Z

2

This is super-easy with promises and the Promise.all() function:

var request = require('request-promise');

var calls = [
  request({ url: 'http://...', headers: { ... } }),
  request({ url: 'http://...', headers: { ... } }),
  // ...as many more as you want...
];

Promise.all(calls).then(function(results) {
  // do something with results[0]
  // do something with results[1]
});

The Promise API is native in newer versions of JS, and can be easily polyfilled in older JS engines, so it reduces the need for helper libs like async.

This example runs all the requests in parallel. Promise.all() waits for them all to finish, then gives you an array of results matching the original requests. Note also use of the request-promise module, which just piggybacks on top of the request module, but returns a promise instead of taking a callback.

answered Aug 22, 2015 at 17:33

greim

9,4676 gold badges36 silver badges35 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

David Knipe · Accepted Answer · 2015-08-22 21:12:02Z

async needs to know when you've finished each task. To achieve this, the tasks are called with arguments which are callback functions. Your task functions, feed_doj and feed_bis, should call the callback functions. This is also the standard way of getting the output from the tasks - don't use global functions, it's widely considered bad form. Do this instead:

function feed_bis(callback) {
  function callback_bis(error, response, body) {
    if (!error && response.statusCode == 200) {
        parseString(body, function (err, result) {
        var bis = result["rdf:RDF"]["item"];
        callback(null, bis);
      });
    }
  }
  request(url_bis, callback_bis);
}

// And same again for doj

Once the callback functions have been called for both bis and doj, the main callback will be called with arguments (err, result); err will be null because you haven't done anything to handle error cases, and result will be an object with properties named bis and doj.

edsadr · Accepted Answer · 2015-08-22 20:15:27Z

0

You can accomplish this using the parallel method included in the async module,you just push all request in an array and then implement this: https://github.com/caolan/async/blob/master/README.md#parallel it is designed to execute a final callback when all processes end.

answered Aug 22, 2015 at 20:15

edsadr

1,0856 silver badges17 bronze badges

Collectives™ on Stack Overflow

Using node.js async and require to fetch multiple urls

3 Answers 3

Comments

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related