mongo db寫入后如何退出nodejs腳本

Question

我有40萬行要輸入，我需要將其分解。 不幸的是，在完成所有操作之前，我無法退出該腳本。 當然，它總是會耗盡內存。 我認為在.on（'end'，function（）處設置一個值會很有用，但是一旦.on數據完成，我將看不到該值。

'use strict';
var mongoose = require('mongoose');
var fs = require('fs');
var parse = require('csv-parse');
var Schema = mongoose.Schema;
var done;

mongoose.connect('mongodb://127.0.0.1:27017/auth');

var userSchema = new mongoose.Schema({
  username: {
    type: String,
    unique: true
  },
  password: String,
  email: {
    type: String,
    unique: true
  },
  isActive: String,
  roles: {
    account: {
      type: mongoose.Schema.Types.ObjectId,
      ref: 'Account'
    }
  },
  timeCreated: {
    type: Date,
    default: Date.now
  },
  search: [String]
});

var accountSchema = new mongoose.Schema({
  user: {
    id: {
      type: mongoose.Schema.Types.ObjectId,
      ref: 'User'
    },
    name: {
      type: String,
      default: ''
    }
  },
  isVerified: {
    type: String,
    default: ''
  },
  verificationToken: {
    type: String,
    default: ''
  },
  name: {
    first: {
      type: String,
      default: ''
    },
    middle: {
      type: String,
      default: ''
    },
    last: {
      type: String,
      default: ''
    },
    full: {
      type: String,
      default: ''
    }
  },
  company: {
    type: String,
    default: ''
  },
  phone: {
    type: String,
    default: ''
  },
  zip: {
    type: String,
    default: ''
  },
  memberid: {
    type: String,
    default: ''
  },
  status: {
    id: {
      type: String,
      ref: 'Status'
    },
    name: {
      type: String,
      default: ''
    },
    userCreated: {
      id: {
        type: mongoose.Schema.Types.ObjectId,
        ref: 'User'
      },
      name: {
        type: String,
        default: ''
      },
      time: {
        type: Date,
        default: Date.now
      }
    }
  },
  userCreated: {
    id: {
      type: mongoose.Schema.Types.ObjectId,
      ref: 'User'
    },
    name: {
      type: String,
      default: ''
    },
    time: {
      type: Date,
      default: Date.now
    }
  },
  search: [String]
});

var User = mongoose.model('User', userSchema);
var Account = mongoose.model('Account', accountSchema);

fs.createReadStream('./ipart')
  .pipe(parse({
    delimiter: ','
  }))
  .on("data-invalid", function(data) {})
  .on('data', function(csvrow) {
    var u = {
      isActive: 'yes',
      username: csvrow[0],
      email: csvrow[0],
      search: [
        csvrow[1] + ' ' + csvrow[2],
        csvrow[0],
      ]
    };

    User.create(u, function(err, createdUser) {
      if (err) {
        console.log(err);
        return;
      }
      var user = createdUser;
      var displayName = csvrow[1] + ' ' + csvrow[2] || '';
      var nameParts = displayName.split(' ');
      var acct = {
        isVerified: 'no',
        'name.first': nameParts[0],
        'name.last': nameParts[1] || '',
        'name.full': displayName,
        user: {
          id: user._id,
          name: user.username
        },
        search: [
          nameParts[0],
          nameParts[1] || ''
        ]
      };

      Account.create(acct, function(err, account) {
        if (err) {
          return workflow.emit('exception', err);
        }
        var fieldstoset = {
          roles: {
            account: account._id
          }
        };
        User.findByIdAndUpdate(account.user.id, fieldstoset, function(err, user) {
          if (err) throw err;
        });
      });
    });
  })
  .on('end', function() {
    console.log('complete');
  });

Answer 1

您確實需要使用批量插入，我在某些地方找到了此代碼並將其粘貼給您

         var Potato = mongoose.model('Potato', PotatoSchema);
         var potatoBag = [/* a humongous amount of potato objects */];
         Potato.collection.insert(potatoBag, onInsert);
         function onInsert(err, docs) {
           if (err) {
             // TODO: handle error
           } else {
             console.info('%d potatoes were successfully stored.', docs.length);
           }
         }

Answer 2

我建議您將導入CSV數據的整個邏輯分解為以下幾個步驟：1.編寫一個簡單的腳本文件，將CSV導入到這樣的臨時集合中：

YourImportScript

#!/bin/bash 
mongoimport -d YourDBName -c YourTempCollectionName --drop --type csv --file pathToYourCSVFile.csv --headerline

2.在創建用戶之前運行腳本：

var exec = require('child_process').exec;
function importCSV(callback) {
    exec("./pathToYourImportScript/YourImportScript", function (error, stdout, stderr) {
        console.log(stdout);
        if (error !== null)
            console.log('exec error: ' + error);
        });

        callback()
    }

MongoImport將很快導入CSV。

3. 從臨時集合中獲取文檔，然后將其插入到用戶集合中。 您還可以使用異步模塊來巧妙地控制代碼模式的流程：

 async.series([ function (callback) { //CSV Import function }, function (callback) { //User Manupulation function }]); 

最好將標題放入CSV列中，因為從temp集合導入文檔時可以創建模型，並且通過諸如username:myCSVModel.username而不是username: csvrow[0]列標題獲取用戶的屬性會更容易username: csvrow[0] 。