Node.js + OracleDb - Insert the last date many times

Question

I received this JSON:

{
  "TOTAL_RECORDS": 1029,
  "REGISTROS": [
    {
      "CODIGO": "1",
      "ATENDIMENTO": "1",
      "PAGAMENTO": "1",
      "VENCIMENTO": "2016-12-17 00:00:00",
      "PROCESSAMENTO": "2016-12-10 00:00:00",
      "VALOR": "1800.00000",
      "NOSSO_NUMERO": "xxxxxxx",
      "NUMERO_DOCUMENTO": "xxxxx",
      "CODIGO_BANCO": "123",
      "LINHA_DIGITAVEL": "XXX70110000180000",
      "CODIGO_BARRAS": "XXX90940"
    },
    {
      "CODIGO": "2",
      "ATENDIMENTO": "2",
      "PAGAMENTO": "2",
      "VENCIMENTO": "2016-12-17 00:00:00",
      "PROCESSAMENTO": "2016-12-10 00:00:00",
      "VALOR": "2700.00000",
      "NOSSO_NUMERO": "xxxxxxx",
      "NUMERO_DOCUMENTO": "xxxxx",
      "CODIGO_BANCO": "123",
      "LINHA_DIGITAVEL": "XXX70110000180000",
      "CODIGO_BARRAS": "XXX90940"
    },...

Then I need to catch this info and save in DB Oracle, so I do this:

module.exports = function (object, callback) {

     var oracledb = require('oracledb');    
     for(const prop in object['REGISTROS']){                    

        codigo = object['REGISTROS'][prop]['CODIGO'];               

        atendimento = object['REGISTROS'][prop]['ATENDIMENTO'];

        pagamento = object['REGISTROS'][prop]['PAGAMENTO'];

        vencimento = object['REGISTROS'][prop]['VENCIMENTO'];

        processamento = object['REGISTROS'][prop]['PROCESSAMENTO'];

        valor = parseInt(object['REGISTROS'][prop]['VALOR']);

        nossoNumero = object['REGISTROS'][prop]['NOSSO_NUMERO'];

        numeroDocumento = object['REGISTROS'][prop]['NUMERO_DOCUMENTO'];

        codigoBanco = object['REGISTROS'][prop]['CODIGO_BANCO'];

        linhaDigitavel = object['REGISTROS'][prop]['LINHA_DIGITAVEL'];              

        codigoBarras = object['REGISTROS'][prop]['CODIGO_BARRAS'];

        oracledb.getConnection({            
            user: "x",
            password:"xxx",
            connectString: "mycon/string"  
        },
        function(err, connection){  
            if (err){  
                console.error(err.message);  
                return;  
            }

            connection.execute(
                "INSERT INTO USU_TBOLETO VALUES (:USU_CODIGO, :USU_ATEND, :USU_PAGAMENTO, " +
                ":USU_VENCIMENTO, :USU_PROCESSA, :USU_VALOR, :USU_NOSSONUM, :NUMERODOC, :USU_CODBANCO, " +
                ":USU_LINHADIG , :USU_CODBARRAS)",

            [codigo, atendimento, pagamento, vencimento, processamento, valor, nossoNumero, 
            numeroDocumento, codigoBanco, linhaDigitavel, codigoBarras], 
            { autoCommit: true},            
            function(err, result){

                if (err){  
                    console.error(err.message);            
                    doRelease(connection);  
                    return;  
                }
                console.log(codigo + ' - ' + atendimento + ' - ' + pagamento + ' - ' + vencimento);
                ///console.log(result.metaData);  
                ///console.log(result.rows);  
                doRelease(connection);  
            });                  
        }); 
    }   


    function doRelease(connection) {        
        connection.release(  
            function(err){  
                if (err) { console.error(err.message); }  
            }  
        );  
    }
}

And the problem is it's inserted in my database just the last record, 1029 times, like the total records. Why? I don't understand why. The INSERT code is into the FOR condition.

The correct would be insert 1029 times, starting codigo 1 until 1029.

Att. Diogo

Answer 1

When you run the code as you currently have it structured, it will send many async operations to libuv's work queue and your JavaScript code looses control. I'd recommend using the async module's eachSeries method to maintain control or a promise chain (or async/await in Node.js 7.6+).

Also, getting a new connection for each iteration of the loop is going to slow things down a lot! You only need one connection for this operation. You should get the connection, process the data, close the connection.

Another thing to reconsider is the use of autoCommit: true. If you do this for each and every row, you're treating each insert as its own transaction. If a failure were to happen half way through, figuring out what went wrong and fixing it (inserting the remaining rows) will be difficult and manual. I recommend using the commit method on the connection object after everything has been inserted (or use autoCommit: true on the last iteration of the loop).

Here's an example:

const oracledb = require('oracledb');
const config = require('./dbConfig.js');

module.exports = function(object, callback) {
    let conn;

    function insertRegistro(registro, commit) {
        if (commit) {
            console.log('Last iteration of the loop, committing with this one');
        }

        return conn.execute(
            `insert into usu_tboleto (
               usu_codigo, usu_atend, usu_pagamento, usu_vencimento, usu_processa, usu_valor, 
               usu_nossonum, numerodoc, usu_codbanco, usu_linhadig, usu_codbarras
             ) values (
               :usu_codigo, :usu_atend, :usu_pagamento, :usu_vencimento, :usu_processa, :usu_valor, 
               :usu_nossonum, :numerodoc, :usu_codbanco, :usu_linhadig , :usu_codbarras
            )`,
            {
                usu_codigo: registro.CODIGO,
                usu_atend: registro.ATENDIMENTO,
                usu_pagamento: registro.PAGAMENTO,
                usu_vencimento: registro.VENCIMENTO,
                usu_processa: registro.PROCESSAMENTO,
                usu_valor: registro.VALOR,
                usu_nossonum: registro.NOSSO_NUMERO,
                numerodoc: registro.NUMERO_DOCUMENTO,
                usu_codbanco: registro.CODIGO_BANCO,
                usu_linhadig: registro.LINHA_DIGITAVEL,
                usu_codbarras: registro.CODIGO_BARRAS
            },
            {
                autoCommit: commit
            }
        );
    }

    oracledb.getConnection(config)
        .then(function(c) {
            conn = c;

            console.log('Got connection, starting loop');

            let promiseChain = Promise.resolve();

            object['REGISTROS'].forEach(function(registro, index) {
                promiseChain = promiseChain
                    .then(function() {
                        return insertRegistro(registro, object['REGISTROS'].length === index + 1);
                    });
            });

            return promiseChain;
        })
        .catch(function(err) {
            console.log(err);
            console.log('Encountered error, rolling back transaction');

            return conn.rollback()
                .then(function() {
                    console.log('Transaction rolled back');
                })
                .catch(function(err) {
                    console.log('Error rolling back', err);
                });
        })
        .then(function() {
            return conn.close();
        })
        .then(function() {
            console.log('Connection closed');
        })
        .catch(function(err) {
            console.log(err);
        });
} 

You could optimize this further by cutting the number of roundtrips from 1029 to 1 (by doing all at once) or 3 (if you do sets of 500 at a time). This will greatly improve performance. Here's an example that does batch inserts in groups of 500.

const oracledb = require('oracledb');
const config = require('./dbConfig.js');

module.exports = function(object, callback) {
  let conn;

  function insertRegistros(opts) {
    if (opts.commit) {
        console.log('Last iteration of the loop, committing with this one');
    }

    return conn.execute(
     `declare
        type varchar2_aat is table of varchar2(50)
          index by pls_integer;

        l_usu_codigo_vals varchar2_aat;

      begin

        l_usu_codigo_vals := :usu_codigo_vals;

        forall x in 1 .. l_usu_codigo_vals.count
        insert into usu_tboleto (
          usu_codigo, usu_atend, usu_pagamento, usu_vencimento, usu_processa, usu_valor, 
          usu_nossonum, numerodoc, usu_codbanco, usu_linhadig, usu_codbarras
        ) values (
          :usu_codigo_vals(x), :usu_atend_vals(x), :usu_pagamento_vals(x), :usu_vencimento_vals(x), :usu_processa_vals(x), :usu_valor_vals(x), 
          :usu_nossonum_vals(x), :numerodoc_vals(x), :usu_codbanco_vals(x), :usu_linhadig_vals(x) , :usu_codbarras_vals(x)
        );

      end;`,
      {
        usu_codigo_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_codigo_vals},
        usu_atend_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_atend_vals},
        usu_pagamento_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_pagamento_vals},
        usu_vencimento_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_vencimento_vals},
        usu_processa_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_processa_vals},
        usu_valor_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_valor_vals},
        usu_nossonum_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_nossonum_vals},
        numerodoc_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.numerodoc_vals},
        usu_codbanco_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_codbanco_vals},
        usu_linhadig_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_linhadig_vals},
        usu_codbarras_vals: {type: oracledb.STRING, dir: oracledb.BIND_IN, val: opts.usu_codbarras_vals}
      },
      {
        autoCommit: opts.commit
      }
    );
  }

  oracledb.getConnection(config)
    .then(function(c) {
      conn = c;

      console.log('Got connection, starting loop');

      const batchRowCount = 500;
      const loops = Math.ceil(object['REGISTROS'].length / batchRowCount);

      let promiseChain = Promise.resolve();
      let registrosIdx = 0;

      for (let outerIndex = 0; outerIndex < loops; outerIndex += 1) {
        (function() {
          const usu_codigo_vals = [];
          const usu_atend_vals = [];
          const usu_pagamento_vals = [];
          const usu_vencimento_vals = [];
          const usu_processa_vals = [];
          const usu_valor_vals = [];
          const usu_nossonum_vals = [];
          const numerodoc_vals = [];
          const usu_codbanco_vals = [];
          const usu_linhadig_vals = [];
          const usu_codbarras_vals = [];

          for (let idx = 0; idx < batchRowCount; idx += 1) {
            if (registrosIdx === object['REGISTROS'].length) {
              break;
            }

            usu_codigo_vals.push(object['REGISTROS'][registrosIdx].CODIGO);
            usu_atend_vals.push(object['REGISTROS'][registrosIdx].ATENDIMENTO);
            usu_pagamento_vals.push(object['REGISTROS'][registrosIdx].PAGAMENTO);
            usu_vencimento_vals.push(object['REGISTROS'][registrosIdx].VENCIMENTO);
            usu_processa_vals.push(object['REGISTROS'][registrosIdx].PROCESSAMENTO);
            usu_valor_vals.push(object['REGISTROS'][registrosIdx].VALOR);
            usu_nossonum_vals.push(object['REGISTROS'][registrosIdx].NOSSO_NUMERO);
            numerodoc_vals.push(object['REGISTROS'][registrosIdx].NUMERO_DOCUMENTO);
            usu_codbanco_vals.push(object['REGISTROS'][registrosIdx].CODIGO_BANCO);
            usu_linhadig_vals.push(object['REGISTROS'][registrosIdx].LINHA_DIGITAVEL);
            usu_codbarras_vals.push(object['REGISTROS'][registrosIdx].CODIGO_BARRAS);

            registrosIdx += 1;
          }

          promiseChain = promiseChain
            .then(function() {
              return insertRegistros({
                usu_codigo_vals: usu_codigo_vals, 
                usu_atend_vals: usu_atend_vals,
                usu_pagamento_vals: usu_pagamento_vals,
                usu_vencimento_vals: usu_vencimento_vals,
                usu_processa_vals: usu_processa_vals,
                usu_valor_vals: usu_valor_vals,
                usu_nossonum_vals: usu_nossonum_vals,
                numerodoc_vals: numerodoc_vals,
                usu_codbanco_vals: usu_codbanco_vals,
                usu_linhadig_vals: usu_linhadig_vals,
                usu_codbarras_vals: usu_codbarras_vals,
                commit: outerIndex + 1 === loops
              });
            });
        })();
      }

      return promiseChain;
    })
    .catch(function(err) {
      console.log(err);
      console.log('Encountered error, rolling back transaction');

      return conn.rollback()
        .then(function() {
          console.log('Transaction rolled back');
        })
        .catch(function(err) {
          console.log('Error rolling back', err);
        });
    })
    .then(function() {
        return conn.close();
    })
    .then(function() {
        console.log('Connection closed');
    })
    .catch(function(err) {
        console.log(err);
    });
} 

As you can see, the code gets a little tricky. However, this should become simpler in the future when the driver adds support for binding arrays of records rather than just strings or numbers.

You can read more about that and the work around I'm using on my blog here: https://jsao.io/2017/01/plsql-record-types-and-the-node-js-driver/

Answer 2

The for loop gets executed almost immediately whereas the database operation is asynchronous and slow. The for loop must've been completed at time the callback to async operation got executed and therefore it'll only see the values during the last iteration of the loop.

To avoid this you can declare variables using ES6s const which will create the block scoped variable. Change your code to this and it should work:

module.exports = function(object, callback) {

    var oracledb = require('oracledb');
    for (const prop in object['REGISTROS']) {

        const codigo = object['REGISTROS'][prop]['CODIGO'];

        const atendimento = object['REGISTROS'][prop]['ATENDIMENTO'];

        const pagamento = object['REGISTROS'][prop]['PAGAMENTO'];

        const vencimento = object['REGISTROS'][prop]['VENCIMENTO'];

        const processamento = object['REGISTROS'][prop]['PROCESSAMENTO'];

        const valor = parseInt(object['REGISTROS'][prop]['VALOR']);

        const nossoNumero = object['REGISTROS'][prop]['NOSSO_NUMERO'];

        const numeroDocumento = object['REGISTROS'][prop]['NUMERO_DOCUMENTO'];

        const codigoBanco = object['REGISTROS'][prop]['CODIGO_BANCO'];

        const linhaDigitavel = object['REGISTROS'][prop]['LINHA_DIGITAVEL'];

        const codigoBarras = object['REGISTROS'][prop]['CODIGO_BARRAS'];

        oracledb.getConnection({
                user: "x",
                password: "xxx",
                connectString: "mycon/string"
            },
            function(err, connection) {
                if (err) {
                    console.error(err.message);
                    return;
                }

                connection.execute(
                    "INSERT INTO USU_TBOLETO VALUES (:USU_CODIGO, :USU_ATEND, :USU_PAGAMENTO, " +
                    ":USU_VENCIMENTO, :USU_PROCESSA, :USU_VALOR, :USU_NOSSONUM, :NUMERODOC, :USU_CODBANCO, " +
                    ":USU_LINHADIG , :USU_CODBARRAS)",

                    [codigo, atendimento, pagamento, vencimento, processamento, valor, nossoNumero,
                        numeroDocumento, codigoBanco, linhaDigitavel, codigoBarras
                    ], {
                        autoCommit: true
                    },
                    function(err, result) {

                        if (err) {
                            console.error(err.message);
                            doRelease(connection);
                            return;
                        }
                        console.log(codigo + ' - ' + atendimento + ' - ' + pagamento + ' - ' + vencimento);
                        ///console.log(result.metaData);  
                        ///console.log(result.rows);  
                        doRelease(connection);
                    });
            });
    }


    function doRelease(connection) {
        connection.release(
            function(err) {
                if (err) {
                    console.error(err.message);
                }
            }
        );
    }
} 

More about loop and closures:

Javascript infamous Loop issue?

JavaScript closure inside loops – simple practical example

Asynchronous Process inside a javascript for loop