AWS Lambda 节点处理问题

Question

Using ASW Lambda (Node 12.x) I am trying to achieve the below use case

1. Call Database
2. Parse the database results
3. Call HTTPS Rest Endpoints one after another
4. Call Database for Insertion
5. Call Database for Query
6. Return Results

The code to do all this is something like this

 var mysql = require('/opt/node_modules/mysql');
    var axios = require('axios');
    var qs = require('qs');
    
    var config = require('/opt/config.json');
    
    var con = mysql.createConnection({
        host: config.dbhost,
        user: config.dbuser,
        password: config.dbpassword,
        database: config.dbname
    });
    
    exports.handler = function(event, context, callback) {
        context.callbackWaitsForEmptyEventLoop = false;
        let errors = [];
        try {
            let userId = event.queryStringParameters.userId;
            let servicesType = event.queryStringParameters.servicesType;
            servicesType = servicesType.split(",");
            
            const sqlQuery = `SELECT * FROM USER_PRODUCT p where p.userId = ? and p.active= 1 
            and p.serviceType in (?)`;
            con.query(sqlQuery, [userId, servicesType], (error, data) => {
                if (error) {
                    throw error;
                }
                if (data != null && data.length > 0) {
                    data.forEach(getData);
                }
    
                const getSQLQuery = "select * from DATA p where p.userId = ? and p.serviceType 
                in (?) order by p.serviceType";
                con.query(getSQLQuery, [userId, servicesType], (error, data) => {
                    if (error) {
                        throw error;
                    }
                    callback(null, {
                        statusCode: 200,
                        headers: {
                            "Access-Control-Allow-Origin": "*"
                        },
                        body: JSON.stringify({ status: 'success', data: data }),
                    });
                });
    
            });
        }
        catch (error) {
            callback(null, {
                statusCode: 400,
                headers: {
                    "Access-Control-Allow-Origin": "*"
                },
                body: JSON.stringify({ status: 'error', message: "Invalid Request", error: errors }),
            })
        }
    };
    
    function getData(pObject) {
        switch (pObject.serviceType) {
            case 'SAMPLE1':
                sample1Processing(pObject);
                break;
            case 'SAMPLE2':
                sample2Processing(pObject);
                break;
            default:
                // code
        }
    }
    
    async function sample1Processing(pObject) {
        let token = await getAuthToken(pObject);
        console.log("Auth token  :" + token);
        if (typeof token !== 'undefined' && token) {
            let ppData = await getSample1Data(pObject, token);
            storeSample1Data(ppData, pObject);
        }
        else {
            console.log("Did not get auth token");
        }
    
    }
    
    async function getAuthToken(pObject) {
        let responseData = null;
        let input = Buffer.from(pObject.client + ":" + paymentObject.secret);
        let encode = input.toString('base64');
        var data = qs.stringify({
            'grant_type': 'client_credentials'
        });
        var config = {
            method: 'post',
            url: paymentObject.baseURL + '/v1/oauth2/token',
            headers: {
                'Authorization': 'Basic ' + encode,
                'Content-Type': 'application/x-www-form-urlencoded'
            },
            data: data
        };
    
        await axios(config)
            .then(function(response) {
                console.log(JSON.stringify(response.data));
                responseData = response.data.access_token;
            })
            .catch(function(error) {
                console.log(error);
            });
        console.log(`Returning ${responseData} from auth token call`);
        return responseData;
    }
    
    async function getSample1Data(pObject, token) {
        let responseData = null;
        var config = {
            method: 'get',
            url: paymentObject.baseURL + '/v1/reporting/transactions?start_date=2020-07-01T00:00:00Z&end_date=2020-07-30T23:59:59Z',
            headers: {
                'Authorization': 'Bearer ' + token
            }
        };
    
        await axios(config)
            .then(function(response) {
                console.log(JSON.stringify(response.data));
                responseData = response.data;
            })
            .catch(function(error) {
                console.log(error);
            });
        return responseData;
    }
    
    async function storeSample1Data(data, pObject) {
        let customObjectArray = [];
        let customObject = [];
        if (data.transaction_details.length > 0) {
            for (let i = 0; i < data.transaction_details.length; i++) {
                customObject = [];
                customObject.push(.....);
                customObjectArray.push(customObject);
            }
            var sql = "INSERT INTO DATA (colum names) VALUES ? on duplicate key update updatedDate = ?";
            await con.query(sql, [customObjectArray, new Date()], function(err) {
                if (err) throw err;
            });
            console.log('end');
        }
    }

  async function sample2Processing(pObject) {
        var data = qs.stringify({});
        var config = {
            method: 'get',
            url: paymentObject.baseURL + '/v1/payment_intents',
            headers: {
                'Authorization': 'Bearer ' + paymentObject.secret
            },
            data: data
        };
    
        let res = await axios(config);
        let data1 = res.data;
        storeSample2Data(data1, pObject);
        return data1;
    }

    
    async function storeSample2Data(data, pObject) {
        let customObjectArray = [];
        let customObject = [];
        if (data.data.length > 0) {
            for (let i = 0; i < data.data.length; i++) {
                customObject = [];
                customObject.push(.....)
                customObjectArray.push(customObject);
            }
            var sql = "INSERT INTO DATA (column names) VALUES ? on duplicate key update updatedDate = ?";
            await con.query(sql, [customObjectArray, new Date()], function(err) {
                if (err) throw err;
            });
            console.log('end');
        }
    }

The issue is sometimes the API calls don't happen and sometimes only some of them go through. Sometimes everything is done as required and I get the output.

I understand its async nature of JS which causes this and hence all my async calls r in a promise or by async/await.

Answer 1

You should await the call to sample1Processing and sample2Processing :

    async function getData(pObject) {
        switch (pObject.serviceType) {
            case 'SAMPLE1':
                await sample1Processing(pObject);
                break;
            case 'SAMPLE2':
                await sample2Processing(pObject);
                break;
            default:
                // code
        }
    }

and the calls to storeSample1Data and storeSample2Data made in sample1Processing and sample2Processing respectively.

The loop that invokes the getData call needs to be changed to handle async calls so instead of

con.query(sqlQuery, [userId, servicesType], (error, data) => {
   if (error) {
      throw error;
   }
   if (data != null && data.length > 0) {
      data.forEach(getData);
   }
   ...

do something like

con.query(sqlQuery, [userId, servicesType], async (error, data) => {
   if (error) {
      throw error;
   }
   if (data != null && data.length > 0) {
      for (const d of data) {
        await getData(d);
      }
   }
   ...