Wait till all items from mongoDB query are processed before continuing

Question

I'm working on a pretty complex project where a lot of mongoDB queries have to be processed in order to show data. Right now, when performing a query over a big set of data, not all data shows as soon as the page loads. For now I fixed that problem by setting a timeout function, before using the next() function.

So this is where one of the functions is called:

router.get('/playwall', account.checkSession, playwalls.get, function(req, res, next) {
  res.render('dashboard/playwalls/index');
});

After checking if the user is logged in, the playwalls.get function fires. This is how that function works right now:

const playwalls = {
  /* GET ALL PLAYWALLS FOR THAT USER, FOR ADMIN GET THEM ALL
  -------------------------------------------------------------- */
  get(req, res, next) {
    req.session.playwalls = [];
    const dataCollection = db.collection('data');
    const data = dataCollection.find({});
    console.log(typeof data)
    const today = new Date();
    data.forEach(function(d) {
      console.log(data.length)
      if(d.admin == req.session.userData.username || req.session.userData.role == 'admin') {
        const expireDate = new Date(d.date);
        if(expireDate < today && d.status == 'actief') {
          playwalls.editStatus(d, req, res, next);
        }
        req.session.playwalls.push(d);
      }
    })
    setTimeout(function() {
      next();
    }, 1000)
  },
  // Here the rest of the methods
}

My question is; how can I make sure that the next() function is only fired if the data.forEach loop has processed all the documents that are returned by the query? So I can get rid of the setTimeout() function.

Hope my question is clear and someone can help me. I've looked around on the internet but couldn't find a proper answer.

Thanks in advance!

Answer 1

You should use Cursor.toArray , so you can iterate over your documents in sync way, then if you need to do some async job inside forEach callback you can use Promise 's

const playwalls = {
    /* GET ALL PLAYWALLS FOR THAT USER, FOR ADMIN GET THEM ALL
    -------------------------------------------------------------- */
    get(req, res, next) {
        req.session.playwalls = [];
        const dataCollection = db.collection('data');
        const data = dataCollection.find({});
        console.log(typeof data)
        const today = new Date();
        data.toArray(function (err, documents) {
            if (err) {
                return next(err);
            }
            documents.forEach(function (d) {
                console.log(data.length)
                if (d.admin == req.session.userData.username || req.session.userData.role == 'admin') {
                    const expireDate = new Date(d.date);
                    if (expireDate < today && d.status == 'actief') {
                        playwalls.editStatus(d, req, res, next);
                    }
                    req.session.playwalls.push(d);
                }
            })
            next();
        })
    }, // Here the rest of the methods
}

Answer 2

How about defining a function that returns a promise .

So, pull the forEach out into a separate function:

function promisedForEach(data) {
    return new Promise(resolve => {
        // your code
        resolve('ready')
     }
 }

Then invoke the function in place of your forEach:

promisedForEach(data).then( /* carry on */ ) 

Answer 3

I don't think setTimeout will work since everything has to happen in the same thread. So setTimeout will just simply freeze when it is it's turn to be executed.

Another way is to use rxjs observers. https://github.com/ReactiveX/RxJS This can be useful when you want to wait for things in other parts of the applications such as middleware before rendering the page.

You could do something like the following. I have not tested this. I will just give you some idea;

global.Rx = require('rxjs/Rx');

router.get('/playwall', function(req, res) {
  const myProcesses = {
    ready:  new Rx.Subject(false),
    loading:  false,
    readiness: {
      process1: {
        ready: false
      },
      process2: {
        ready: false
      },
      process3: {
        ready: false
      }
    },
    checkReadiness: function (){
      if (
        this.readiness.process1==true
        && this.readiness.process2==true
        && this.readiness.process3==true
      ) {
        this.loading = false;
        this.ready.next(true);
      }
    },
  };
  //asynchronous process
  (function (){
    anObject.fetchAll().then(function (results){
      myProcesses.readiness.process1 = true;
      myProcesses.checkReadiness();
    })
  })();

  //another asynchronous process
  (function (){
    anObject2.fetchAll().then(function (results){
      myProcesses.readiness.process2 = true;
      myProcesses.checkReadiness();
    })
  })();

  //another asynchronous process
  (function (){
    anObject2.fetchAll().then(function (results){
      myProcesses.readiness.process3 = true;
      myProcesses.checkReadiness();
    })
  })();

  //When all asynchronous processes have been completed
  myProcesses.ready.subscribe({next: (v) => {
    if (v==true){
      res.render('ejs/products/checkout', {});
    }
  }});
});