FileStream from busboy to MongoDB

Question

So I have an incoming FileStream from busboy that I want to save to MongoDB. I think I need to have it as a File or some sort of buffer to be able to save it. I'm sure I could do it by first saving it to disk using fs and then reading it, but that seems cumbersome. This is my full route code so far:

// Upload a new study plan
router.route("/add").post((req, res, next) => {
    let busboy = new Busboy({headers: req.headers});

    // A field was recieved
    busboy.on('field', function (fieldname, val, valTruncated, keyTruncated) {

        if (req.body.hasOwnProperty(fieldname)) { // Handle arrays
            if (Array.isArray(req.body[fieldname])) {
                req.body[fieldname].push(val);
            } else {
                req.body[fieldname] = [req.body[fieldname], val];
            }
        } else { // Else, add field and value to body
            req.body[fieldname] = val;
        }
    });

    // A file was recieved
    busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
        const saveTo = path.join('.', filename);
        let readFile = null;

        file.on("data", () => {
            console.log("Got file data!");
        })

        file.on("end", () => {
            //How do I save the file to MongoDB?
        })
    });

    // We're done here boys!
    busboy.on('finish', function () {
        //console.log(req.body);
        console.log('Upload complete');
        res.end("That's all folks!");
    });
    return req.pipe(busboy);
});

I want to append {"pdf": file} to my req.body which has the rest of the data...

Answer 1

There's no need for saving your file to disk, you can stream it directly from busboy to mongo with some kind of a streaming interface - I'm not sure how you wish to save the file, but if that's just a simple file structure I guess you should use Mongo's GridFS .

I assume you got your connection and client from somewhere, so we'll just use that. We need a GridFS bucket from the cliet:

const db = client.db(dbName);
const bucket = new mongodb.GridFSBucket(db);

We'll use it when we want to save the file:

    // A file was recieved
    busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
        const saveTo = path.join('.', filename);
        // here we PIPE the file to DB.
        file.pipe(bucket.openUploadStream(saveTo));
    });

Now there's also a matter of responding when the files are actually saved - as this is done asynchronously. So we need to keep count of running operations like this:

    // place this in the request callback.
    // here's our counter - we'll increment it for each operation started.
    let ops = 0;
    const dec = () => --ops || res.end("That's all folks!");

Now we change the code above a little so we won't respond until the files are saved in Mongo:

    // A file was recieved
    busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
        ops++; // we increment ops for each file saved
        const saveTo = path.join('.', filename);
        // here we PIPE the file to DB (pass the db and collection here).
        file.pipe(bucket.openUploadStream(saveTo))
            .on('finish', dec);
    });

    ops++; // we're waiting for finish of the request also so another increment
    busboy.on('finish', dec);

As you see every time a file upload is started we increment the ops and when it's finished, we decrement it. The || operator will execute the res.end method when ops reach 0 .

Answer 2

So while Michal's answer is probably not wrong, it was not what I was after. I finally found a solution by using the Buffer object. Here is my code:

router.route("/add").post((req, res, next) => {
    let busboy = new Busboy({headers: req.headers});
    let buffers = [];

    // A field was recieved
    busboy.on('field', function (fieldname, val, valTruncated, keyTruncated) {

        if (req.body.hasOwnProperty(fieldname)) { // Handle arrays
            if (Array.isArray(req.body[fieldname])) {
                req.body[fieldname].push(val);
            } else {
                req.body[fieldname] = [req.body[fieldname], val];
            }
        } else { // Else, add field and value to body
            req.body[fieldname] = val;
        }
    });

    // A file was recieved
    busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {

        file.on("data", (data) => {
            buffers.push(data);
        });

        file.on("end", () => {
            req.body[fieldname] = Buffer.concat(buffers);
        });
    });

    // We're done here boys!
    busboy.on('finish', function () {
        console.log(req.body);

        const plan = new StudyPlan(req.body);
        plan.save()
            .then(_ => console.log("YEEAEH!"))
            .catch(err => {console.log(err);});

        res.end();
    });
    return req.pipe(busboy);
});