Piping a multipart/form-data request to another request while changing some fields names in Node.js

Question

I want to stream a file upload request in multipart/form-data to another server and change some fields name at the same time.

I don't want to store temporarily a file on disk and don't want to store the file completely in memory either.

I tried to use multer, busboy and multiparty. I think I got closer by using custom Transform streams but it is not working yet.

const express = require('express');
const request = require('request');
const { Transform } = require('stream');

const router = express.Router();

class TransformStream extends Transform {
  _transform(chunk, encoding, callback) {
    // here I tried to manipulate the chunk
    this.push(chunk);
    callback();
  }

  _flush(callback) {
    callback();
  }
}

router.post('/', function pipeFile(req, res) {
  const transformStream = new TransformStream();
  req.pipe(transformStream).pipe(request.post('http://somewhere.com'));
  res.sendStatus(204);
});

I tried to manipulate chunks in _transform without success (EPIPE). It sounds quit hacky, are they any better solutions?

Answer 1

Here is a solution using replacestream along with content-disposition .

const replaceStream = require('replacestream');
const contentDisposition = require('content-disposition');

router.post('/', function pipeFile(req, res) {

  let changeFields = replaceStream(/Content-Disposition:\s+(.+)/g, (match, p1) => {
    // Parse header
    let {type, parameters} = contentDisposition.parse(p1);

    // Change the desired field
    parameters.name = "foo";

    // Prepare replacement
    let ret = `Content-Disposition: ${type}`;
    for(let key in parameters) {
      ret += `; ${key}="${parameters[key]}"`;
    }

    return ret;
  })

  req.pipe(changeFields)
  .pipe(request.post('http://somewhere.com'))
  .on('end', () => {
    res.sendStatus(204);
  });
});

Answer 2

This worked for a single file multipart upload using express, multiparty, form-data, pump and got.

const stream = require('stream');
const express = require('express');
const multiparty = require("multiparty");
const got = require("got");
const FormData = require('form-data');
const pump = require('pump');
const app = express();

app.post('/upload', (req, res) => {
  const url = "<<multipart image upload endpoint>>";   
  var form = new multiparty.Form();
  form.on("part", function(formPart) {
    var contentType = formPart.headers['content-type'];
    var formData = new FormData();
    formData.append("file", formPart, {
      filename: formPart.filename,
      contentType: contentType,
      knownLength: formPart.byteCount
    });
    const resultStream = new stream.PassThrough();
    try {
      // Pipe the formdata to the image upload endpoint stream and the result to the result stream
      pump(formData, got.stream.post(url, {headers: formData.getHeaders(), https:{rejectUnauthorized: false}}), resultStream, (err) =>{
        if(err) {
          res.send(error);
        }
        else {
          // Pipe the result of the image upload endpoint to the response when there are no errors.
          resultStream.pipe(res);
        }
        resultStream.destroy();
      });
    }
    catch(err) {
      resultStream.destroy();
      console.log(err);
    }
  });
  form.on("error", function(error){
      console.log(error);
  })
  form.parse(req);   
});