Node.js mysql - Nested queries and async/await

Question

I couldn't find any answer on Google or Stackoverflow for this, maybe you can help me. I am using Node.js with mysql and express.

Since Node.js is asynchronous, I work with async/await.

This application should return a nested object like this:

    [{
      "name":"Name1",
      "additionalData": [
       {
        "data1": "data", 
        "data2": "data"
       }
      ]
    },
      {"name":"Name2",
      "additionalData": [
       {
        "data1": "data", 
        "data2": "data"
       }
      ]
    }]

This is the code (simplified):

var express = require('express');
var mysql = require('mysql');
var app = express();

var con = mysql.createConnection({
    host: "localhost",
    user: "root",
    password: "password",
    database: "database"
  });

con.connect(function(err) {
    if (err) throw err;
    console.log("Connected!");
  });

async function getData() { 
      var sql = 'Select name from users' 

      let promise = new Promise((resolve, reject) => {

        con.query(sql, async (err, resultSet) => { 
          if (err) reject(err); 

          let promisesub = new Promise((resolvesub, rejectsub) => {
            con.query('Select data from othertable', (err, rs) => { 
              resolvesub(rs)       
            }) 
          })

          resultSet.forEach(async (row) => { 
            row.additionalData = await promisesub;
            console.log(row)
           }); 

          resolve(resultSet);
        }) 

      })

      let result = await promise;

      return result;
} 

app.get('/', function (req, res) { 

  getData().then(function(rows) { 
    res.send(rows) 
  }).catch(function(err) { 
    console.log(err) 
  }) 
}); 

app.listen(3000, function () {
  console.log('Backend listening on port 3000!');
});

So if I call http://localhost:3000/ I only the get names (without the additionaldata from the second query):

[{"name":"Name1"},
{"name":"Name2"}]

The problem is that only the result of the first promise is returned. The code in the forEach loop should assign the additionalData to each row. The code is working (the log gives the correct output) but is performed after the first promise is sent back.

How can i make the first promise wait until the additionalData is assigned to the object?

Thank you very much!

Best regards

Answer 1

Select data from othertable

Fetches all the data from that table without any regard for what the associated "name" is .

Also you are doing a lot of work that is 'trivial' in SQL. This does all the work of that inner loop for you:

SELECT  n.name, a.data1, a.data2
    FROM  Names AS n
    JOIN  AdditionalData AS a  ON a.name = n.name 

(Caveat: Without seeing SHOW CREATE TABLE , I cannot be sure I guessed at the schema for the data.)

In general, if you are tempted to "nest queries", you are doing SQL 'wrong'.

Answer 2

Below i did some changes into your code in a way we can see the behavior happening (take a look into http://jsfiddle.net and throw this code below there to see, or other playing ground)

            var con = {
        connect : function (fnc) { fnc()},
        query : function (sql, pr) {
            if (sql === 'Select name from users') {
              setTimeout(
                  () => pr(null,[{id:1},{id:2}]), 2000
              )
            } else {
              setTimeout(
                  () => pr(null,[{id:3},{id:4}]), 2000
              )
            }
        }
      }

      con.connect(function(err) {
          if (err) throw err;
          console.log("Connected!");
        });

      async function getData() { 
            var sql = 'Select name from users' 

            let promise = new Promise((resolve, reject) => {

              con.query(sql, async (err, resultSet) => { 
                if (err) reject(err); 

                let promisesub = new Promise((resolvesub, rejectsub) => {
                  con.query('Select data from othertable', (err, rs) => { 
                    resolvesub(rs)       
                  }) 
                })

                resultSet.forEach(async (row) => {
                  console.log('for each');
                  row.additionalData = await promisesub;
                  console.log(' step 1', row);
                 }); 

                resolve(resultSet);
              }) 

            })

            let result = await promise;

            return result;
      } 

      getData().then(function(rows) { 
          console.log('step 2 ',  rows); 
        }).catch(function(err) { 
          console.log(err) 
        });

The return of it is exactly in this order

Connected!
(index):61 for each
(index):77 step 2  (2) [{…}, {…}]
(index):63  step 1 {id: 1, additionalData: Array(2)}
(index):63  step 1 {id: 2, additionalData: Array(2)}

First thing i notice, is that the problem isnt into the libs you are using. Its on the second async enviroment you created, that holds the process but only inside the function forEach. So, it will pass over the forEach, even tho the sequence of the forEach will be correct... For this scenario, use other auxiliary promise:

        let auxProm = new Promise(resolve => {
            resultSet.forEach(async (row) => {
              console.log('for each');
            row.additionalData = await promisesub;
              console.log(' step 1', row);
            }); 
          resolve(resultSet)
        })
        auxProm.then(auxResultSet => resolve(auxResultSet));

Answer 3

Ok I found the solution after reading this:

Link

The article says that the forEach function doesn't wait for the callback to be done, so I implemented an own asyncForEach function.

Here is the code that works:

var express = require('express');
var mysql = require('mysql');
var app = express();

var con = mysql.createConnection({
    host: "localhost",
    user: "root",
    password: "admin",
    database: "db"
  });

  con.connect(function(err) {
    if (err) throw err;
    console.log("Connected!");
  });


async function getData() { 
      var sql = 'Select name from user' 

      let promise = new Promise((resolve, reject) => {

        con.query(sql, async (err, resultSet) => { 
          if (err) reject(err); 

          // asyncForEach
            async function asyncForEach(array, callback) {
              for (let index = 0; index < array.length; index++) {
                await callback(array[index], index, array)
              }
            } 

            //Subdata
            let promisesub = new Promise((resolvesub, rejectsub) => {
              con.query('Select data from othertable', (err, rs) => { 
                  resolvesub(rs)       
          }) 
        })

          const start = async () => {
              await asyncForEach(resultSet, async (row) => {
                row.additionalData = await promisesub;
              })
          resolve(resultSet)
          //console.log('ready')
          };

          start()
        }) 
      })

      let result = await promise;

      return result;
} 

app.get('/', function (req, res) {

    getData().then(function(rows) { 
        //console.log('step 3 ',  rows); 
        res.send(JSON.stringify(rows))
      }).catch(function(err) { 
        console.log(err) 
      });
});

app.listen(3000, function () {
    console.log('Example app listening on port 3000!');
  });  

Answer 4

Using the below code to get category and subcategory. How to get Subcategory by category?

async function getData() {
var sql1 = 'Select name from category'
let promise = new Promise((resolve, reject) => {

    sql.query(sql1, async (err, resultSet) => {
        if (err) reject(err);

        // asyncForEach
        async function asyncForEach(array, callback) {
            for (let index = 0; index < array.length; index++) {
                await callback(array[index], index, array)
            }
        }

        //Subdata
        

        const start = async () => {
            await asyncForEach(resultSet, async (row) => {
            let promisesub = new Promise((resolvesub, rejectsub) => {
              sql.query(`Select * from subcategory cat_id=${row.id}`, (err, rs) => {
                  resolvesub(rs)
              })
            })
                row.subcategory = await promisesub;
            })
            resolve(resultSet)
            //console.log('ready')
        };

        start()
    })
})

let result = await promise;
return result;
}