long loop in node.js leak memory

Question

please help me, I want to make my script to access to multiple URL in the same time to get some resource. This script is running normally,but the problem is that it leak memory and server will down. Pleas anyone help to give me better solution. Thank you! Please help to check source code below:

var fs = require('fs');
var request = require('request');
var dir = './';
var dateTime = require('node-datetime');

const express = require('express');
const app = express();

var urls = [
  'http://www.google.com',
  'https://www.bing.com',
  'http://www.yahoo.com',
  "http://stackoverflow.com",
  "http://github.com",
  "http://www.yahoo.co.jp"
];

var urlLength=urls.length;
function remoteControllerPress(){
    var dt = dateTime.create();
    var newCurrentTimeDir = dt.format('Y-m-d-H-M-S').toString();
    var dirName = dir+newCurrentTimeDir;
    if (!fs.existsSync(dirName)){
        fs.mkdirSync(dirName);
    }
    console.log("==============calling at "+newCurrentTimeDir + '\n');

    for(var i = 0; i<urlLength; i++) {
     (function() {
      var j = i;
      var url = urls[j];
      process.nextTick(function() {


        request(url, function(error, response, body) {
          console.log("---------URL: "+url);
          if (!error && response.statusCode == 200) {
            console.log("URL: " + url + " statusCode: " + response.statusCode);

            fs.writeFile(dirName+"/file_"+j, body.toString(), function(err) {
                if(err) {
                    return console.log(err);
                }
                console.log(dirName+"/file_"+ j +" was saved!\n\n");
            }); 


          } else if (error) {
            console.log("error.code:", error.code);
          }
        });

      });
     })();
    }

    setTimeout(function(){
        remoteControllerPress();
    },6000);
}
console.log("==============running...\n ");
remoteControllerPress();

app.post('/', function (req, res) {
  res.send('hello world');
});

app.listen(
    3000, 
    () => console.log('Server start listening on port 3000...')
);

Pierre Mallet · Accepted Answer · 2018-01-02 16:09:23Z

1

Your remoteControllerPress function seems to have a reference to itself since you call it recursively with the timeout. Thus the garbage collector have trouble to clean memory.

A simple (ugly ?) workaround should be to use setInterval

     }

    /*setTimeout(function(){
        remoteControllerPress();
    },6000); <-- remove that */ 
}
console.log("==============running...\n ");
setInterval(remoteControllerPress, 6000); // <- change this

answered Jan 2, 2018 at 16:09

Pierre Mallet

7,2412 gold badges22 silver badges31 bronze badges

Sign up to request clarification or add additional context in comments.

3 Comments

munyso Over a year ago

Thank Pierre Mallet for replying me, But the memory still increase with this solution. First startup memory is 45Mb then it increase in each loop to 105Mb

Pierre Mallet Over a year ago

It increase by 60Mb every 6 secondes or it go to 105 after first loop then stay stable ?

munyso Over a year ago

It 's always increase until crash

Daphoque · Accepted Answer · 2018-01-02 16:16:19Z

Use async operator to do async loop https://caolan.github.io/async/docs.html

Use setInterval with setImmediate instead of using setTimeout in order to clear context at each call

var async = require("async");

function remoteControllerPress(){

    var dt = dateTime.create();
    var newCurrentTimeDir = dt.format('Y-m-d-H-M-S').toString();
    var dirName = dir+newCurrentTimeDir;
    if (!fs.existsSync(dirName)){
        fs.mkdirSync(dirName);
    }
    console.log("==============calling at "+newCurrentTimeDir + '\n');

    // TO do it 5 by 5
    async.eachOfLimit(urls, 5, function(url, i, cb){

       request(url, function(error, response, body) {

          console.log("---------URL: "+url);


          if (!error && response.statusCode == 200) {

            console.log("URL: " + url + " statusCode: " + response.statusCode);

            fs.writeFile(dirName+"/file_"+j, body.toString(), function(err) {
                if(err)
                {
                   console.log(err);
                }
                console.log(dirName+"/file_"+ j +" was saved!\n\n");

                return cb();
            }); 


          }
          else if (error)
          {
            console.log("error.code:", error.code);
            return cb();
          }
          else
          {
            return cb();
          }
        });
    }, function(err){

        return;

    });


}
console.log("==============running...\n ");

setInterval(function(){

  setImmediate(remoteControllerPress);

},6000);

Collectives™ on Stack Overflow

long loop in node.js leak memory

2 Answers 2

3 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

3 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related