How set a timeout every 50 iterations in an array.map() loop?

Question

Question: How can I make my files.map(...) pause every 50 iterations?

Problem: the gm().size() is a very expensive function. which completely shits the bed after about 300 iterations. I have a theory that this will be remedied if I let the function keep up.

     //interaction happens that will traverse a bunch of folder and create an array of files paths
     glob(filePath + '/**/*{.png,.jpg,.gif}', function (er, files) {
        var chunksize = 50; // sets the iteration size
        if (er) return er;
          service.stuff[name] = files.map(function (entry, i) {

            return {
              identity: getIdentity()  //returns the identity(size) of images
            };

            function getIdentity() {
              if(i % chunksize == 0) { // if the 50th iteration
                (function(chunksize, i){

                  setTimeout(function () {
                  var entrySize = gm(entry)  //graphics magic will return size of images based on path.
                    .size(function (err, size) {
                      return size;
                    });
                  }, 2000); //pause for 2 seconds.

                }());

              } else {  
                var entrySize = gm(entry)
                    .size(function (err, size) {
                      return size;
                    });
              }


              return entrySize.data; //returns identity data.
            }

          });
      });

I'd strongly recommend the bluebird promise library with promisifyAll and then the form Promise.reduce(files, (function (total, file) { yourFn(file) }), 0). You can step through sequentially, image by image, instead of overloading your system. More here bluebirdjs.com/docs/features.html#promisification-on-steroids — Tyler
– Tyler, Commented Nov 5, 2015 at 0:13

Dan Prince · Accepted Answer · 2015-11-05 00:00:24Z

1

Alternatively, implement your own batch processor. This alternative to map will only process options.batchSize items at a time, then takes an options.timeoutMs break to give the application time to do other things.

function batchMap(array, fn, options, callback) {
  var batchSize = options.batchSize || 100,
      timeoutMs = options.timeoutMs || 0;

  function map(done, todo) {
    if(todo.length > 0) {
      setTimeout(function() {
        var mapped = todo.slice(0, batchSize).map(fn);
        map(done.concat(mapped), todo.slice(batchSize));
      }, timeoutMs);
    } else {
      callback(null, done);
    }
  }

  map([], array);
}

answered Nov 5, 2015 at 0:00

Dan Prince

30.1k15 gold badges93 silver badges124 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Linus Oleander · Accepted Answer · 2015-11-04 23:51:03Z

1

You can use async.mapSeries for this. It waits for each iteration to finish before continuing to the next.

npm install async

var async = require("async");
var noFile = 0;
var done = function (err, result) {
  console.log(result);
}

var yourLogic = function(file){

}
var processFile = function(file, callback) {
  if(noFile > 50) {
    setTimeout(function() {
      noFile++;
      callback(null, yourLogic(file));
    }, 1000);
  } else {
    noFile++;
    callback(null, yourLogic(file));
  }
}

async.mapSeries(files, processFile, done);

answered Nov 4, 2015 at 23:51

Linus Oleander

18.2k15 gold badges71 silver badges104 bronze badges

Comments

a sad dude · Accepted Answer · 2015-11-05 00:18:21Z

1

Sorry, I just feel like shamelessly plugging my new library so I will. It goes something like this:

var CL = require('coastline');

CL.run(function* () {
  
  var files = yield glob(filePath + '/**/*{.png,.jpg,.gif}', CL.cb());
  var chunksize = 50;
  service.stuff[name] = yield CL.map(files, function* (entry, i) {
    
    if (i && i % chunksize == 0) yield CL.sleep(2000);
    
    var entrySize = yield gm(entry).size(CL.cb());
    return {
      identity: entrySize.data
    };
    
  });
                                     
});

Edit: checked and it works, except there is no .data in entrySize, just .width and .height?

Edit: removed var i since we can use array index.

edited Nov 5, 2015 at 0:18

answered Nov 5, 2015 at 0:05

a sad dude

2,79020 silver badges20 bronze badges

Collectives™ on Stack Overflow

How set a timeout every 50 iterations in an array.map() loop?

3 Answers 3

Comments

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related