Node js CSV to JSON (Large file)

Question

I need transform rows in csv file to json. I use csvtojson package and works like a charm, but my csv file is very huge, about 21.000.000 lines. :(

My node app does can process all lines.

See my snippet bellow.

csv({noheader:true,
      delimiter:";",
    workerNum: 4})
  .fromFile(csvFilePath)
  .on('json',(jsonObj, rowIndex)=>{
      var TN = jsonObj.field1;
      var cod_op = jsonObj.field2;
      var msisdn = TN.toString();

     //UP TO FIREBASE HERE!!!

        noticia.child(msisdn).set(cod_op);
        console.log(TN + ' ' + rowIndex);



  })
  .on('done',(error)=>{
      console.log()
  })

lesimoes · Accepted Answer · 2017-02-18 11:12:12Z

I solved by my self with csv-stream module

Here is a snippset

  var options = {
    delimiter : ';', // default is ,
    endLine : '\n', // default is \n,
    columns : ['VERSION_ID', 'TN','RN1','DATA','OPERACAO','CNL','EOT'], // by default read the first line and use values found as columns
    columnOffset : 7, // default is 0
    escapeChar : '"', // default is an empty string
    enclosedChar : '"' // default is an empty string
}

var csvStream = csv.createStream(options);
i = 0
request('http://yourhugefile.csv').pipe(csvStream)
    .on('error',function(err){
        console.error(err);
    })
    .on('data',function(data){
        // outputs an object containing a set of key/value pair representing a line found in the csv file.
        //console.log(data);
    })
    .on('column',function(key,value){
      if(key == 'TN' || key == 'RN1')
          firebaseUpload(value);
        // outputs the column name associated with the value found
        //console.log('#' + key + ' = ' + value);
    })

Collectives™ on Stack Overflow

Node js CSV to JSON (Large file)

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related