JSON.parse() throws error when JSON gets too long

Question

I have spawned a python child_process in node.js that returns JSON data. In my python file I used json.dumps() before passing it back to node. Whenever the list "results" in the JSON contains more than 62 entries, my node.js server dies when trying to parse it. Otherwise it works all fine?

Even when the node app dies, the data gets still sent back correctly to the client.

The error message is only thrown when calling JSON.parse(), but I cannot leave it out here, because the received data from python is in a buffered format:

<Buffer 7b 0d 0a 20 20 22 70 61 72 61 6d 65 74 65 72 73 22 3a 20 5b 0d 0a 20 20 20 20 7b 0d 0a 20 20 20 20 20 20 22 70 72 6f 70 65 72 74 79 22 3a 20 22 42 65 ... 9136 more bytes>
<Buffer 0d 0a>

Has anyone an idea what the problem could be?

The thrown error message is:

undefined:2

SyntaxError: Unexpected end of JSON input
    at JSON.parse (<anonymous>)
    at Socket.<anonymous> (D:\Projects\data-farming-framework-api\app.js:52:23)

    at Socket.emit (events.js:310:20)
    at addChunk (_stream_readable.js:286:12)
    at readableAddChunk (_stream_readable.js:268:9)
    at Socket.Readable.push (_stream_readable.js:209:10)
    at Pipe.onStreamRead (internal/stream_base_commons.js:186:23)

My node.js code:

app.post('/api/ps/executeExperiments', async (req, res, next) => {
  let path = req.body.path;
  let experiments = JSON.stringify(req.body.data);
  let python = await spawn('python', ['./plantsim.py', 'execute_experiments()', path, experiments]);
  python.stdout.on('data', data => {
    if (data.toString() == 'ERROR') {
      callback(new Error('ERROR'))
    }
    let result = JSON.parse(data);
    res.send(result);
  });
});

The returned JSON has following format:

{
  parameters: [
    { property: 'Bearbeitungszeit', object: 'Einzelstation' },
    { property: 'Bearbeitungszeit', object: 'Einzelstation2' }
  ],
  results: [
    { id: 1, input: 252, output: 249, values: [Array] },
    { id: 2, input: 198, output: 195, values: [Array] },
    { id: 3, input: 165, output: 162, values: [Array] },
    { id: 4, input: 204, output: 201, values: [Array] },
    { id: 5, input: 169, output: 166, values: [Array] },
    { id: 6, input: 239, output: 236, values: [Array] },
    { id: 7, input: 225, output: 222, values: [Array] },
    { id: 8, input: 312, output: 309, values: [Array] },
    { id: 9, input: 324, output: 321, values: [Array] },
    { id: 10, input: 248, output: 245, values: [Array] },
    { id: 11, input: 199, output: 196, values: [Array] },
    { id: 12, input: 186, output: 183, values: [Array] },
    { id: 13, input: 161, output: 158, values: [Array] },
    { id: 14, input: 149, output: 146, values: [Array] },
    { id: 15, input: 175, output: 172, values: [Array] },
    { id: 16, input: 201, output: 198, values: [Array] },
    { id: 17, input: 242, output: 239, values: [Array] },
    { id: 18, input: 162, output: 159, values: [Array] },
    { id: 19, input: 184, output: 181, values: [Array] },
    { id: 20, input: 157, output: 154, values: [Array] },
    { id: 21, input: 160, output: 157, values: [Array] },
    { id: 22, input: 290, output: 287, values: [Array] },
    { id: 23, input: 358, output: 355, values: [Array] },
    { id: 24, input: 153, output: 150, values: [Array] },
    { id: 25, input: 155, output: 152, values: [Array] },
    { id: 26, input: 195, output: 192, values: [Array] },
    { id: 27, input: 174, output: 171, values: [Array] },
    { id: 28, input: 269, output: 266, values: [Array] },
    { id: 29, input: 341, output: 338, values: [Array] },
    { id: 30, input: 263, output: 260, values: [Array] },
    { id: 31, input: 156, output: 153, values: [Array] },
    { id: 32, input: 167, output: 164, values: [Array] },
    { id: 33, input: 295, output: 292, values: [Array] },
    { id: 34, input: 230, output: 227, values: [Array] },
    { id: 35, input: 189, output: 186, values: [Array] },
    { id: 36, input: 172, output: 169, values: [Array] },
    { id: 37, input: 254, output: 251, values: [Array] },
    { id: 38, input: 182, output: 179, values: [Array] },
    { id: 39, input: 206, output: 203, values: [Array] },
    { id: 40, input: 232, output: 229, values: [Array] },
    { id: 41, input: 259, output: 256, values: [Array] },
    { id: 42, input: 159, output: 156, values: [Array] },
    { id: 43, input: 305, output: 302, values: [Array] },
    { id: 44, input: 151, output: 148, values: [Array] },
    { id: 45, input: 181, output: 178, values: [Array] },
    { id: 46, input: 302, output: 299, values: [Array] },
    { id: 47, input: 187, output: 184, values: [Array] },
    { id: 48, input: 245, output: 242, values: [Array] },
    { id: 49, input: 149, output: 146, values: [Array] },
    { id: 50, input: 279, output: 276, values: [Array] },
    { id: 51, input: 215, output: 212, values: [Array] },
    { id: 52, input: 170, output: 167, values: [Array] },
    { id: 53, input: 288, output: 285, values: [Array] },
    { id: 54, input: 329, output: 326, values: [Array] },
    { id: 55, input: 283, output: 280, values: [Array] },
    { id: 56, input: 193, output: 190, values: [Array] },
    { id: 57, input: 177, output: 174, values: [Array] },
    { id: 58, input: 348, output: 345, values: [Array] },
    { id: 59, input: 265, output: 262, values: [Array] },
    { id: 60, input: 219, output: 216, values: [Array] },
    { id: 61, input: 352, output: 349, values: [Array] },
    { id: 62, input: 151, output: 148, values: [Array] }
  ]
}

Values is also a list, however i don't know why it is not displayed in the terminal correctly. The data is still there as it gets passed to the client correctly.

Would data not be coming from the child_process? I. E not strictly coming back as Json, is there a end event? Or finish? I'd then collect the data, and keep Json parsing till the final event is triggered — Isolated
– Isolated, Commented May 10, 2020 at 16:33

jfriend00 · Accepted Answer · 2020-05-10 18:51:09Z

data events deliver chunks of data on arbitrary boundaries. As long as you are getting the entire JSON in one chunk (something you do not control), your code probably just happens to work. But, as soon as the data gets large enough of takes long enough to generate that it gets sent in more than one chunk, then your code dies trying to parse an individual chunk of data. This is why it is size dependent. At some size of data, it will start arriving in more than one chunk. And, you can't parse each chunk as JSON separately (thus your JSON.parse() error.

Instead, if you're going to parse it into JSON, you need to accumulate all the chunks and then when it's done, parse the entire thing. JSON is not a format that can easily be incrementally parsed (there is JSON stream code that can do it, but it's a lot of work).

app.post('/api/ps/executeExperiments', async (req, res, next) => {
  let path = req.body.path;
  let experiments = JSON.stringify(req.body.data);
  let python = spawn('python', ['./plantsim.py', 'execute_experiments()', path, experiments]);

  let allData = [];
  // collect from all the data events here
  python.stdout.on('data', data => {
    allData.push(data.toString());
  }).on('error', e => {
     console.log(e);
     res.sendStatus(500);
  }).on('close', () => {
     // we have all data now
     let result = allData.join(allData, "");
     if (result.startsWith('ERROR')) {
        console.log(result);
        res.sendStatus(500);
     } else {
        res.send(allData);
     }
  });
});

In your existing code what is callback()? You don't show that and it has me confused so I removed it.

You also need to send an error response to the incoming POST request if you get an error.

And, there's no point to using await on the return value from spawn(). It doesn't return a promise so await doesn't do anything useful.

Also, it appears there is no need to even parse the JSON. If you're going to just parse it and then send it right off with res.send(), all res.send() is going to do is turn it back into JSON again. So, perhaps all you really need to do it to just stream python.stdout as your response?

xdeepakv · Accepted Answer · 2020-05-10 22:08:51Z

The issue is not with the long file, the issue is with the way you tried to collect data. python.stdout.on is the stream of data. Data comes in the chunk. So you need to collect all data then return to response. You can simplify, using basic util method as given below.

const spawn = require("child_process").spawn;

function run(path, experiments) {
  let command = spawn("python", [
    "./plantsim.py",
    "execute_experiments()",
    path,
    experiments,
  ]);
  return new Promise((resolve) => {
    var result = "";
    command.stdout.on("data", function (data) {
      result += data.toString();
    });
    command.on("close", function (code) {
      resolve(result);
    });
  });
}
app.post("/api/ps/executeExperiments", async (req, res, next) => {
  let path = req.body.path;
  let experiments = JSON.stringify(req.body.data);
  const data = await run(path, experiments);
  res.send(result);
});

If you are just returning file[no parse or transform] you can just pipe the data to the response.

Sample:

const { spawn } = require("child_process");
const express = require("express");
const app = express();

app.get("/json", (req, res) => {
  let command = spawn("cat", [
    __dirname + "/test.json",
  ]);
  command.stdout.pipe(res)
  command.on("close", function (data) {
    console.log("done writing");
  });
});
app.listen(3000)

Collectives™ on Stack Overflow

JSON.parse() throws error when JSON gets too long

2 Answers 2

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related