Mongo陷阱上的异步操作：如何执行同步，我有正确的逻辑

Question

What I want to achieve : I want to create a car which has to have a correct mapping with its dealer and then store this car in the list/array of my company document present in my companies collection in myKaarma db.

The problem that I am facing: all the logic is correct but the problem is arising due to the asynchronous nature, even though I am using the callbacks. I know the issue but don't know how to solve it.

Let me explain what the problem is :

My companies model using Mongoose:

// jshint  node :true
"use strict";

const MONGOOSE = require("mongoose"),
      DB = MONGOOSE.connection;

let companySchema = MONGOOSE.Schema({

      company_name: {
            type: String, required: true
      },
      company_location: {
            type: String, require: true
      },
      cars: [{
            model: {
                  type: String,required: true
            },
            year: {
                  type: Number, required: true
            },
            PriceInINR: {
                  type: Number, required: true
            },
            trim: {
                  type: String, required: true

            },
            engine: {
                  type: String, required: true
            },
            body: {
                  type: String, required: true
            },
            color: {
                  type: String, required: true
            },
            transmission_type: {
                  type: String, required: true
            },
            dealer_id: {
                  type: String, required: true
            }
      }]


});


let collection_name = "companies";
let CompanyModel = MONGOOSE.model(collection_name, companySchema);
createAscendingIndex_on_company_name(DB);

module.exports = CompanyModel;





//   indexing  at schema level -->  using node js
function createAscendingIndex_on_company_name(DB, callback) {
      let collection = DB.collection(collection_name);

      // ? Create the index
      collection.createIndex({
            company_name: 1, // specifies : indexing type is ascending indexing
      }, {
            unique: true
      }, function (err, result) {
            if (err) {
                  console.log("error while setting up indexing on companies collection");
            }
            console.log("index created  ", result, "<<<<<<<<", collection_name, " collection");
            // callback("result");
      });
}
//? NOTE : Creating indexes in MongoDB is an idempotent operation. So running db.names.createIndex({name:1}) would create the index only if it didn't already exist.

As you will notice that I have indexed and also made company_name as unique, so no duplicate entry can be there this is the problem

In my code when I do this: // ? check if the company exists : and if not then create one // ? check if the company exists : and if not then create one , the issue I have is, as the nodejs being asynchronous & very fast : [so suppose I have 5 car records] so all the five cars actually go into the code where I check:

 CompanyModel.find({
                       company_name: company_name
                    }, (err, companies) => {.....}

and it is so fast that it like all go at the same time and as of course there does not exist any such company right now in the company document so all of them passes the if condition

if (companies.length === 0) {...});

so now when in my records there are 3 cars with same company all of them enter almost simultaneously and all passes these condition again simultaneously, but as soon as they pass this above condition I ask Mongo to create the company document

    let company = new CompanyModel({ 
           company_name: company_name,                                                   
          company_location: company_location,
          cars: [car]
     });
   company.save((err) => {...}

but now, all the 3 records are here to create a new company object and add to the collection. But here as soon as one of them create the document and added to the collection, at the same time now the other two also created their objects but now as there is already an object created and added so Mongo throws the unique exception here.

What I wanted to happen is when we found an duplicate object then then new car with the same company should be just pushed into the array that the company document has its with the field cars

Note: this scenario is only in the case of when the company does not present in the collection, but if it is already present then my code works fine, it successfully pushes all the cars into the cars field of the respective company.

This is the function that is doing what I want to :

  function map_and_save_cars_in_garage() {

        let carList = require("./test.json");
        let totalCar = carList.length;

        console.log(carList);

        carList.forEach((carRecord, index) => {

              let company_name = carRecord.make.toLowerCase();
              let company_location = "USA";

              // build a car
              let car = {
                    model: carRecord.model,
                    year: carRecord.year,
                    PriceInINR: carRecord.priceInr,
                    trim: carRecord.trim,
                    engine: carRecord.engine,
                    body: carRecord.body,
                    color: carRecord.color,
                    transmission_type: carRecord.transmission,
                    dealer_id: undefined, // --> just for now
              };


              // ? search for the correct dealer --> for mapping
              let dealer_email = "bitBattle_2018_" + carRecord.DealerID + "_@myKarmaa.com";

              DealerModel.find({
                    email: dealer_email
              }, (err, dealer) => {
                    if (err) {
                          console.log("Error : dealer not found for this car");
                          throw new Error(err);
                    }
                    car.dealer_id = dealer[0]._id; // ? update the dealer_id

                    // ? check if the company exists : and if not then create one
                    CompanyModel.find({
                          company_name: company_name
                    }, (err, companies) => {
                          if (err) {
                                console.log("Error : while finding the compay");
                                throw new Error(err);
                          }
                          console.log(company_name, companies);
                          if (companies.length === 0) {
                                console.log("No such Company car exists in the garrage --> creating one");

                                let company = new CompanyModel({
                                      company_name: company_name,
                                      company_location: company_location,
                                      cars: [car]
                                });

                                company.save((err) => {
                                      if (err) {
                                            console.log("Error : while adding company ");
                                            throw new Error(err);
                                      }
                                      console.log(index, "<<<<<<<<      INDEX  ", totalCar);
                                      if (index === totalCar - 1) {
                                            console.log("done");
                                            res.send("build complete");
                                            // build_complete();
                                      }
                                });

                          } else {
                                console.log("Company already exists in garage : add this car with all other cars of this company");
                                let company = companies[0]; // ? as its sure that they are unique

                                let query = {
                                      _id: company._id
                                };
                                let updat_command = {
                                      $push: {
                                            cars: car
                                      }
                                };

                                CompanyModel.updateOne(query, updat_command, (err) => {
                                      if (err) {
                                            console.log("Error : while pushing car to the compay's cars");
                                            throw new Error(err);
                                      }
                                      console.log(index, "<<<<<<<<      INDEX  ", totalCar);
                                      if (index === totalCar - 1) {
                                            console.log("done");
                                            res.send("build complete");
                                            // build_complete();
                                      }
                                });

                          }

                    });

              });
              console.log(index, "<<<<<<<<      INDEX--OUTER   ", totalCar);
        });

  }

Output:

[nodemon] restarting due to changes...
[nodemon] starting `node app.js`
[[20:39:12.519]] [LOG]   server live
[[20:39:12.626]] [LOG]   Connected to DB :  SUCCESS
[[20:39:12.642]] [LOG]   index created   email_1 <<<<<<<< buyers  collection
[[20:39:12.647]] [LOG]   index created   email_1 <<<<<<<< dealers  collection
[[20:39:12.795]] [LOG]   index created   company_name_1 <<<<<<<< companies  collection
[[20:39:42.081]] [LOG]   start saving cars
[[20:39:42.084]] [LOG]   [ { id: '2',
    vin: '5GAKRBKD9EJ323900',
    make: 'Buick',
    model: 'ENCLAVE',
    year: '2014',
    priceInr: '2537993',
    trim: 'Leather FWD',
    engine: 'SPORT UTILITY 4-DR',
    body: '3.6L V6 DOHC 24V',
    color: 'Silver',
    transmission: 'Manual',
    DealerID: '103' },
  { id: '4',
    vin: '2GKALSEKXD6184074',
    make: 'GMC',
    model: 'TERRAIN',
    year: '2013',
    priceInr: '3851710',
    trim: 'SLE2 FWD',
    engine: 'SPORT UTILITY 4-DR',
    body: '2.4L L4 DOHC 16V FFV',
    color: 'Yellow',
    transmission: 'Manual',
    DealerID: '103' },
  { id: '6',
    vin: '1GC1KXE86EF127166',
    make: 'Chevrolet',
    model: 'SILVERADO 2500HD',
    year: '2014',
    priceInr: '840547',
    trim: 'LT Crew Cab 4WD',
    engine: 'CREW CAB PICKUP 4-DR',
    body: '6.6L V8 OHV 32V TURBO DIESEL',
    color: 'Grey',
    transmission: 'Automatic',
    DealerID: '103' },
  { id: '8',
    vin: '1GKKRTED1CJ211299',
    make: 'GMC',
    model: 'Acadia',
    year: '2012',
    priceInr: '3805008',
    trim: 'Denali FWD',
    engine: 'SPORT UTILITY 4-DR',
    body: '3.6L V6 DOHC 24V',
    color: 'Metallic White',
    transmission: 'Automatic',
    DealerID: '103' },
  { id: '10',
    vin: '1GKKVTKD9EJ282303',
    make: 'GMC',
    model: 'ACADIA',
    year: '2014',
    priceInr: '1730235',
    trim: 'Denali AWD',
    engine: 'SPORT UTILITY 4-DR',
    body: '3.6L V6 DOHC 24V',
    color: 'Black',
    transmission: 'Manual',
    DealerID: '103' },
  { id: '12',
    vin: '1GKS1AKC0FR200193',
    make: 'GMC',
    model: 'YUKON',
    year: '2015',
    priceInr: '3129397',
    trim: 'SLE 2WD',
    engine: 'SPORT UTILITY 4-DR',
    body: '5.3L V8 OHV 16V',
    color: 'Silver',
    transmission: 'Manual',
    DealerID: '103' } ]
[[20:39:42.089]] [LOG]   0 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.089]] [LOG]   1 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.090]] [LOG]   2 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.090]] [LOG]   3 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.090]] [LOG]   4 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.090]] [LOG]   5 '<<<<<<<<      INDEX--OUTER   ' 6
[[20:39:42.120]] [LOG]   gmc []
[[20:39:42.120]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.134]] [LOG]   buick []
[[20:39:42.134]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.138]] [LOG]   gmc []
[[20:39:42.138]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.143]] [LOG]   chevrolet []
[[20:39:42.143]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.146]] [LOG]   gmc []
[[20:39:42.146]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.150]] [LOG]   1 '<<<<<<<<      INDEX  ' 6
[[20:39:42.150]] [LOG]   gmc []
[[20:39:42.151]] [LOG]   No such Company car exists in the garrage --> creating one
[[20:39:42.153]] [LOG]   0 '<<<<<<<<      INDEX  ' 6
[[20:39:42.154]] [LOG]   Error : while adding company
events.js:183
      throw er; // Unhandled 'error' event
      ^

Error: MongoError: E11000 duplicate key error collection: myKaarma.companies index: company_name_1 dup key: { : "gmc" }
    at company.save (/Users/prashant/Desktop/appathon/route/api.js:179:55)
    at /Users/prashant/Desktop/appathon/node_modules/mongoose/lib/model.js:4437:16
    at $__save.error (/Users/prashant/Desktop/appathon/node_modules/mongoose/lib/model.js:397:16)
    at /Users/prashant/Desktop/appathon/node_modules/kareem/index.js:246:48
    at next (/Users/prashant/Desktop/appathon/node_modules/kareem/index.js:167:27)
    at next (/Users/prashant/Desktop/appathon/node_modules/kareem/index.js:169:9)
    at Kareem.execPost (/Users/prashant/Desktop/appathon/node_modules/kareem/index.js:217:3)
    at _handleWrapError (/Users/prashant/Desktop/appathon/node_modules/kareem/index.js:245:21)
    at _cb (/Users/prashant/Desktop/appathon/node_modules/kareem/index.js:304:16)
    at /Users/prashant/Desktop/appathon/node_modules/mongoose/lib/model.js:258:9
    at /Users/prashant/Desktop/appathon/node_modules/kareem/index.js:135:16
    at _combinedTickCallback (internal/process/next_tick.js:131:7)
    at process._tickCallback (internal/process/next_tick.js:180:9)

How can I get out of this?

Answer 1

If you're using higher than node 7 (I hope so..) you can use async/await to make this code much simpler to deal with. You can also use findOne from mongoose so that you don't have to deal with arrays, since you know there is only one of each result.

The trick to this code working is that it waits until the previous car has been inserted into the database before inserting another one.

async function map_and_save_cars_in_garage() {

    let carList = require("./test.json");
    let totalCar = carList.length;

    for (let carRecord of carList) {

        let company_name = carRecord.make.toLowerCase();
        let company_location = "USA";

        // build a car
        let car = {
            model: carRecord.model,
            year: carRecord.year,
            PriceInINR: carRecord.priceInr,
            trim: carRecord.trim,
            engine: carRecord.engine,
            body: carRecord.body,
            color: carRecord.color,
            transmission_type: carRecord.transmission,
            dealer_id: undefined, // --> just for now
        };

        let dealer_email = "bitBattle_2018_" + carRecord.DealerID + "_@myKarmaa.com";

        try {
            let dealer = await DealerModel.findOne({
                    email: dealer_email
                }).exec();

            car.dealer_id = dealer._id;

            let company = await CompanyModel.findOne({
                    company_name: company_name
                }).exec();

            if (!company) {
                console.log("No such Company car exists in the garrage --> creating one");

                let company = new CompanyModel({
                        company_name: company_name,
                        company_location: company_location,
                        cars: [car]
                    });

                await company.save();
            } else {
                console.log("Company already exists in garage : add this car with all other cars of this company");

                await CompanyModel.updateOne({
                    _id: company._id
                }, {
                    $push: {
                        cars: car
                    }
                }).exec();
            }
        } catch (err) {
            throw new Error(err);
        }
    }

    console.log("done");
    res.send("build complete");
}

Another thing that I might try is not waiting for each car to be created but creating an array (which will be accessed instantly, compared to a database), containing the newly inserted companies, like so:

async function map_and_save_cars_in_garage() {

    let carList = require("./test.json");
    let totalCar = carList.length;

    let newCompanies = {};

    for (let carRecord of carList) {
        (async function () {
            let company_name = carRecord.make.toLowerCase();
            let company_location = "USA";

            // build a car
            let car = {
                model: carRecord.model,
                year: carRecord.year,
                PriceInINR: carRecord.priceInr,
                trim: carRecord.trim,
                engine: carRecord.engine,
                body: carRecord.body,
                color: carRecord.color,
                transmission_type: carRecord.transmission,
                dealer_id: undefined, // --> just for now
            };

            let dealer_email = "bitBattle_2018_" + carRecord.DealerID + "_@myKarmaa.com";

            try {
                let dealer = await DealerModel.findOne({
                        email: dealer_email
                    }).exec();

                car.dealer_id = dealer._id;

                // Check for company in newCompanies
                let company = newCompanies[company_name];

                // If company is not in newcompanies it will be undefined so this if statement will be executed
                if (!company) {
                    // If company is not found in database this will be null
                    await CompanyModel.findOne({
                        company_name: company_name
                    }).exec();
                }

                // If company is null then create a new one
                if (!company) {
                    console.log("No such Company car exists in the garrage --> creating one");

                    let company = new CompanyModel({
                            company_name: company_name,
                            company_location: company_location,
                            cars: [car]
                        });

                    // Add company to newCompanies
                    newCompanies[company_name] = company;
                    await company.save();
                } else {
                    console.log("Company already exists in garage : add this car with all other cars of this company");

                    await CompanyModel.updateOne({
                        _id: company._id
                    }, {
                        $push: {
                            cars: car
                        }
                    }).exec();
                }
            } catch (err) {
                throw new Error(err);
            }
        })();
    }

    console.log("done");
    res.send("build complete");
}

This will not have to wait for previous cars to be added to the database.