上传前文件未在node.js中关闭

Question

I'm trying to upload a file to S3 after creation however the file doesn't seem to be closing with the fs.createWriteStream. Hence I keep getting a 0 byte file being uploaded.

function createManifest(manfile) {
console.log(['a'].toString(), ['a', 'b'].toString());

    var arrayLength = files.length;
    var lastItem = arrayLength - 1;
    console.log( chalk.blue("The last item value is:",lastItem))
    console.log( chalk.yellow("The arrayLength value is:",arrayLength))
    var logStream = fs.createWriteStream("manny_temp.json", {'flags': 'a'});
// use {'flags': 'a'} to append and {'flags': 'w'} to erase and write a new file
//    logStream.on('open', function(fd) {
logStream.write('{'+ "\r\n");
logStream.write("\"entries\": [" + "\r\n");

    for (var i = 0; i < arrayLength; i++) {

            console.log( chalk.inverse(files[i]))
            console.log( chalk.blue("The i value is:",i))
            if ( i == lastItem) {
                logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}" + "\r\n");
            } else {
                    logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}," + "\r\n");
             }
    }
    logStream.write('    ]' + "\r\n");
    logStream.write('}');
//      }).on('end', function() {
            logStream.end();
            //fs.renameSync(logStream.path, manfile.toString());
            return callback(filepath);
//      logStream.close();
    //fs.renameSync(logStream.path, "manny.json");

  //   });
}

I've tried a multitude of ways to get the file to close so that the next function can upload the file upon creation, and even added a sleep, but it always seems to leave a hanging inode.

Using the fs.write seems to write only one line vs writing all the lines in the array / streaming data.

Does anyone have a suggestion?

Answer 1

You need to listen for the close event on the logStream before calling the callback. That is what signals that the backing file descriptor has been closed (no more writing will take place). I should note, this is different than listening for the finish event since finish merely indicates the stream is closed, but not necessarily that the file descriptor is closed, so it is technically less safe to rely on if you're going to do something with the file soon after).

Replace this:

return callback(filepath);

with this:

logStream.on('close', function() {
  callback(filepath);
});

It is possible to improve on this further, by instead using the (err, result) -style callbacks common in the node ecosystem:

logStream.on('close', function() {
  callback(null, filepath);
});
logStream.on('error', function(err) {
  callback(err);
});

That way you can capture any errors that may result from opening or writing to the file.

I'm also assuming you do have callback defined in a parent scope somewhere.

Answer 2

Your question is asked without the basic research though, Here is an attempt to make it more readable code and executable at glance. Hope you will find an answer with below code.

If your question is still the same, re-write your question in a better way.

var chalk = require('chalk');
var fs = require('fs');

function createManifest(manfile) {
var files = ['/test.json'];
console.log(['a'].toString(), ['a', 'b'].toString());
    var arrayLength = files.length;
    var lastItem = arrayLength - 1;
    console.log( chalk.blue("The last item value is:",lastItem))
    console.log( chalk.yellow("The arrayLength value is:",arrayLength))
    var logStream = fs.createWriteStream("manny_temp.json", {'flags': 'a'});
    logStream.write('{'+ "\r\n");
    logStream.write("\"entries\": [" + "\r\n");

    for (var i = 0; i < arrayLength; i++) {

            console.log( chalk.inverse(files[i]))
            console.log( chalk.blue("The i value is:",i))
            if ( i == lastItem) {
                logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}" + "\r\n");
            } else {
                    logStream.write("{\"url\":\"s3://mybucket/" +files[i] + "\",\"key\":true}," + "\r\n");
             }
    }
    logStream.write('    ]' + "\r\n");
    logStream.write('}'); 
    logStream.end();
    logStream.on('finish', function(){
        fs.renameSync("manny_temp.json", "manny.json"); 
    }); 
};

createManifest();

Answer 3

mscdex is nearly right. You need to listen for the 'finish' event, not the 'close' event because that is emitted by readable streams but this is a writable stream.

And you can pass your handler to 'end' instead. See https://nodejs.org/dist/latest-v4.x/docs/api/stream.html#stream_event_finish

logStream.end(function() {
    fs.renameSync("manny_temp.json", "manny.json");
    callback(filepath);
});

I tried to edit mscdex's answer to this effect but this was rejected for some reason.

Edit: You'd need to define or pass in 'callback', as well as 'filepath'.