Node.js: Reading long JSON file always cuts last closing bracket character

Question

I have a the following:

let base64strings = ["U2FsdGVkX1+lWCC8X04S/duu480r0BkPvea...", ....]

let writableStream = fs.createWriteStream('test.json')
let index = 0

for (const base64string of base64strings) {
  writableStream.cork()

  let textForStream = ''

  if (index === 0) {
    textForStream += '['
  } else {
    textForStream += ','
  }

  textForStream += JSON.stringify({ index, data: base64string })

  if (index === (base64strings.length - 1)) {
    textForStream += ']'
  }

  writableStream.write(textForStream)

  writableStream.uncork()

  index++
}

let check = fs.readFileSync('test.json', { encoding: 'utf-8' })

let arr = JSON.parse(check) // <--- this breaks with a *SyntaxError: Unexpected end of JSON input*

That is because the last character of check which is supposed to be a ] is now missing. It gets cut off every time when I read the file, even though it's there in the test.json when I open it up.

However, if I then open up test.json in my editor everything looks fine:

[{"index":0,"data":"U2FsdGVkX1+lWCC8X04S/duu480r0BkPvea..."},{"index":1,"data":"U2FsdGVkX1+lWCC8X04S/duu480r0BkPvea..."}, ...]

What am I doing wrong?

I guess it's the { encoding: 'utf-8' } that messes things up but I don't how to fix this.

Answer 1

I removed the cork() and uncork() calls and I'm now waiting for drain after each loop and it works:

function forDrain(stream) {
  return new Promise((resolve) => {
    stream.once('drain', resolve)
  })
}

let base64strings = ["U2FsdGVkX1+lWCC8X04S/duu480r0BkPvea...", ....]

let writableStream = fs.createWriteStream('test.json')
let index = 0

for (const base64string of base64strings) {   
  let textForStream = ''

  if (index === 0) {
    textForStream += '['
  } else {
    textForStream += ','
  }

  textForStream += JSON.stringify({ index, data: base64string })

  if (index === (base64strings.length - 1)) {
    textForStream += ']'
  }

  writableStream.write(textForStream)

  await forDrain(writableStream)
    
  index++
}

writableStream.end()

let check = fs.readFileSync('test.json', { encoding: 'utf-8' })

console.log(JSON.parse(check).join('') === base64strings.join('')) // true