我想使用nodejs和angularjs上传一个csv文件，然后将其保存到cassandra

Question

This is my node js code i want to upload a csv file using nodejs and angularjs and then save it to cassandra.

    app.use(function(req, res, next) { //allow cross origin requests
        res.setHeader("Access-Control-Allow-Methods", "POST, PUT, OPTIONS, DELETE, GET");
        res.header("Access-Control-Allow-Origin", "http://localhost");
        res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
        next();
    });

    app.use(express.static('../client'));
    app.use(bodyParser.json());  

    var storage = multer.diskStorage({ //multers disk storage settings
        destination: function (req, file, cb) {
            cb(null, './uploads/');
        },
        filename: function (req, file, cb) {
            var datetimestamp = Date.now();
            cb(null, file.fieldname + '-' + datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length -1]);
        }
    });

    var upload = multer({ //multer settings
                    storage: storage
                }).single('file');

    /** API path that will upload the files */
    app.post('/upload', function(req, res) {
        upload(req,res,function(err){
            if(err){
                 res.json({error_code:1,err_desc:err});
                 return;
            }
             res.json({error_code:0,err_desc:null});
        });
    });

I want to upload my file directly to cassandra through the gui. any lead or help is thankful.

Answer 1

You should find a way to read from the multipart form data without storing to the webserver hard drive.

You can use busboy (middleware connect-busboy ) to read the file data from the response as a stream. Once you can read from the upcoming data, you can insert it to Cassandra.

You have to define:

• Max level of concurrency: how many csv lines you want to insert to C* in parallel.
• How to handle failure: what to do when a single insert fails.

After that, you can read lines and insert those in parallel to Cassandra like this:

// Use some control flow methods (dependency)
const async = require('async');
const csv = stream; //using busboy to obtain the file stream
// Maximum amount of rows to insert in parallel
const limit = 256;
csv.on('readable', function () {
  //its emitted once there is a chunk of data
  var r;
  var rows = [];
  async.whilst(function condition() {
    while ((r = csv.read()) != null && rows.length < limit) {
      rows.push(r);
    }
    return rows.length > 0;
  }, function eachGroup(next) {
    // We have a group of 256 rows or less to save
    // we can do it in a batch
    // or we can do it in parallel with async.each()
    async.each(rows, function (r, eachCallback) {
      //adapt the csv row to parameters
      //sample
      var params = r.split(',);
      client.execute(query, params, { prepare: true}, eachCallback);
    }, next);
  }, function groupFinished(err) {
    if (err) {
      //something happened when saving
      //do something with err
      return;
    }
    // The chunk of csv rows emitted by csv stream where saved
  });
}).on('end', function () {
  // no more data
});