node.js looping through GETs with promise

Question

I'm new to promises and I'm sure there's an answer/pattern out there but I just couldn't find one that was obvious enough to me to be the right one. I'm using node.js v4.2.4 and https://www.promisejs.org/

This should be pretty easy I think...I need to do multiple blocks of async in a specific order, and one of the middle blocks will be looping through an array of HTTP GETs.

//New Promise = asyncblock1 - FTP List, resolve the returned list array //.then(asynchblock2(list)) - loop through list array and HTTP GET needed files //.then(asynchblock3(list)) - update local log

I tried creating a new Promise, resolving it, passing the list to the .then, doing the GET loop, then the file update. I tried using a nested promise.all inside asynchblock2, but it's actually going in reverse order, 3, 2, and 1 due to the timing of those events. Thanks for any help.

EDIT: Ok, this is the pattern that I'm using which works, I just need a GET loop in the middle one now.

var p = new Promise((resolve, reject) => {
  setTimeout(() => {
    console.log('2 sec');
    resolve(1);
  },
  2000);
}).then(() => {
  return new Promise((resolve) => {
    setTimeout(() => {
      console.log('1.5 sec');

// instead of this section, here I'd like to do something like:
// for(var i = 0; i < dynamicarray.length; i++){
//   globalvar[i] = ftpclient.getfile(dynamicarray[i])
// }
// after this loop is done, resolve

      resolve(1);
    },
    1500);
  });
}).then(() => {
  return new Promise((resolve) => {
    setTimeout(() => {
      console.log('1 sec');
      resolve(1);
    },
    1000);
  });
});

EDIT Here is the almost working code!

 var pORecAlert = (function(){

  var pa;
  var newans = [];
  var anstodownload = [];
  var anfound = false;//anfound in log file
  var nexttab;
  var lastchar;
  var po;
  var fnar = [];
  var antext = '';

//-->> This section works fine; it's just creating a JSON object from a local file
    try{
        console.log('trying');
        porfile = fs.readFileSync('an_record_files.json', 'utf8');
        if(porfile == null || porfile == ''){
            console.log('No data in log file - uploaded_files_data.json being initialized!');
            plogObj = [];
        }
        else{
            plogObj = JSON.parse(porfile);
        }
    }
    catch(jpfp){
        console.log('Error parsing log file for PO Receiving Alert: ' + jpfp);
        return endPORecAlertProgram();
    };
    if((typeof plogObj) === 'object'){
        console.log('an_record_files.json log file found and parsed for PO Receiving Alert!');
    }
    else{
        return mkError(ferror, 'pORecAlert');
    };
//finish creating JSON Object

    pa = new Client();
    pa.connect(ftpoptions);
    console.log('FTP Connection for FTP Check Acknowledgement begun...');
    pa.on('greeting', function(msg){
      console.log('FTP Received Greeting from Server for ftpCheckAcknowledgement: ' + msg);
    });
    pa.on('ready', function(){
      console.log('on ready');

      //START PROMISE LIST
      var listpromise = new Promise((reslp, rejlp) => {
            pa.list('/public_html/test/out', false, (cerr, clist) => {
              if(cerr){
                    return mkError(ferror, 'pORecAlert');
              }
              else{
                    console.log('Resolving clist');
                    reslp(clist);
              }
            });
      });
      listpromise.then((reclist) => {
          ftpplist:
          for(var pcl = 0; pcl < reclist.length; pcl++){
                console.log('reclist iteration: ' + pcl);
                console.log('checking name: ', reclist[pcl].name);
                if(reclist[pcl].name.substring(0, 2) !== 'AN'){
                  console.log('Not AN - skipping');
                  continue ftpplist;
                }
                else{//found an AN
                  for(var plc = 0; plc < plogObj.length; plc++){
                        if(reclist[pcl].name === plogObj[plc].anname){
                          //console.log('Found reclist[pcl].name in local log');
                          anfound = true;
                        };
                  };
                  if(anfound === false){
                        console.log('Found AN file to download: ', reclist[pcl].name);
                        anstodownload.push(reclist[pcl].name);
                  };
                };
          };
          console.log('anstodownload array:');
          console.dir(anstodownload);
          return anstodownload;
      }).then((fnar) => {
            //for simplicity/transparency, here is the array being overwritten
            fnar = new Array('AN_17650_37411.699.txt', 'AN_17650_37411.700', 'AN_17650_37411.701', 'AN_17650_37411.702.txt', 'AN_17650_37411.801', 'AN_17650_37411.802.txt');
        return Promise.all(fnar.map((gfname) => {
            var nsalertnames = [];
          console.log('Getting: ', gfname);
          debugger;
          pa.get(('/public_html/test/out/' + gfname), function(err, anstream){//THE PROBLEM IS THAT THIS GET GETS TRIGGERED AN EXTRA TIME FOR EVERY OTHER FILE!!!
                antext = '';
                console.log('Get begun for: ', gfname);
                debugger;
                if(err){
                  ferror.nsrest_trace = 'Error - could not download new AN file!';
                  ferror.details = err;
                  console.log('Error - could not download new AN file!');
                  console.log('************************* Exiting *************************')
                  logError(ferror, gfname);
                }
                else{
                  // anstream.on('data', (anchunk) => {
                  //   console.log('Receiving data for: ', gfname);
                  //   antext += anchunk;
                  // });
                  // anstream.on('end', () => {
                  //   console.log('GET end for: ', gfname);
                  //   //console.log('path to update - gfname ', gfname, '|| end text.');
                  //   fs.appendFileSync(path.resolve('test/from', gfname), antext);
                  //   console.log('Appended file');
                  //   return antext;
                  // });//end end
                };
          });//get end
        }));//end Promise.all and map
      }).then((res99) => {
        // pa.end();
        // return Promise(() => {
           console.log('end all. res99: ', res99);
        //   //res4(1);
        //   return 1;
        // });
      });
    });
})();

-->> What happens here: So I added the almost working code. What is happening is that for every other file, an additional Get request gets made (I don't know how it's being triggered), which fails with an "Unable to make data connection".

So for my iteration over this array of 6, there ends up being 9 Get requests. Element 1 gets requested (works and expected), then 2 (works and expected), then 2 again (fails and unexpected/don't know why it was triggered). Then 3 (works and expected), then 4 (works and expected), then 4 again (fails and unexpected) etc

Answer 1

what you need is Promise.all() , sample code for your app:

...
}).then(() => {
  return Promise.all(arry.map(item => ftpclient.getFile(item)))
}).then((resultArray) => {
...

Answer 2

So thanks for the help (and the negative votes with no useful direction!)

I actually reached out to a good nodejs programmer and he said that there seemed to be a bug in the ftp module I was using, and even when trying to use a blackbird .map, the quick succession of requests somehow kicked off an error. I ended up using promise-ftp, blackbird, and promiseTaksQueue - the kicker was that I needed interval . Without it the ftp would end up causing a strange illogical error in the ftp module.

Answer 3

You need the async library. Use the async.eachSeries in situations where you need to use asynchronous operations within a loop, then execute a function when all of those are complete. There are many variations depending on the flow you want but this library does it all.

https://github.com/caolan/async

async.each(theArrayToLoop, function(item, callback) {
 // Perform async operation on item here.
  doSomethingAsync(item).then(function(){
     callback();
  })
}, function(err){
   //All your async calls are finished continue along here
});