Node.js异步下载多个文件

Question

In trying to get a hang of node.js asynchronous coding style, I decided to write a program that would read a text file containing a bunch of URLS to download and download each file. I started out writing a function to download just one file (which works fine), but having trouble extending the logic to download multiple files.

Here's the code:

var http     = require("http"),
    fs       = require("fs"),
    input    = process.argv[2],
    folder   = "C:/Users/Wiz/Downloads/",
    regex    = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
    urls     = null,
    url      = "",
    filename = "";

fs.readFile(input, "utf8", function(e, data) {
    console.log("Reading file: " + input);
    if (e) console.log("Got error:" + e.message);
    urls = data.split("\n");
    for (var i = urls.length; i--;) {
        url = urls[i];
        if (!url.match(regex)) continue;
        filename = folder + url.substring(url.lastIndexOf('/') + 1);
        downloadQueue.addItem(url, filename);
    }
});

var downloadQueue = {
    queue: [],
    addItem: function(p_sSrc, p_sDest) {
        this.queue.push({
            src: p_sSrc,
            dest: p_sDest
        });
        if (this.queue.length === 1) {
            this.getNext();
        }
    },
    getNext: function() {
        var l_oItem = this.queue[0];
        http.get(l_oItem.src, function(response) {
            console.log("Downloading: " + l_oItem.dest);
            var file = fs.createWriteStream(l_oItem.dest);
            response.on("end", function() {
                file.end();
                console.log("Download complete.");
                downloadQueue.removeItem();
            }).on("error", function(error) {
                console.log("Error: " + error.message);
                fs.unlink(l_oItem.dest);
            });
            response.pipe(file);
        });
    },
    removeItem: function() {
        this.queue.splice(0, 1);
        if (this.queue.length != 0) {
            this.getNext();
        } else {
            console.log("All items downloaded");
        }
    }
};

How do I structure the code so that the completion of the first download can signal the initiation of the next one. Please note that this exercise is just for learning purposes, to understand how asynchronous coding works. In practice, I'm sure there are much better tools out there to download multiple files.

Answer 1

Try simple at first, it look like you copy paste codes and quite don't understand what they do.

Do a simple loop, that get the url, and print something.

var http = require('http');

URL = require('url').parse('http://www.timeapi.org/utc/now?format=%25F%20%25T%20-%20%25N')
URL['headers'] = {'User-Agent': 'Hello World'}


// launch 20 queries asynchronously
for(var i = 0; i < 20; i++) {
  (function(i) {
    console.log('Query ' + i + ' started');
    var req = http.request(URL, function(res) {
      console.log('Query ' + i + ' status: ' + res.statusCode + ' - ' + res.statusMessage);
      res.on('data', function(content){
        console.log('Query ' + i + ' ended - ' + content);
      });
    });

    req.on('error', function(err) {
      console.log('Query ' + i + ' return error: ' + err.message);
    });

    req.end();
  })(i);
}

All the urls will be fetched asynchronously. You can observe that the response does not arrive in order, but are still processed correctly.

The difficulty with async is not to do the things is parallel, because you just write like a single task, and execute multiple time. It becomes complicated when you need for instance to wait for all tasks to finished before continuing. And for that, have a look at promises

Answer 2

Here is what I started out with. Figuring that each download was invoked asynchronously, they would all be independent of each other.

var http     = require("http"),
    fs       = require("fs"),
    input    = process.argv[2],
    folder   = "C:/Users/Wiz/Downloads/",
    regex    = /(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?/,
    urls     = null,
    url      = "",
    filename = "";

fs.readFile(input, "utf8",
  function(e, data) {
    console.log("Reading file: " + input);
    if (e) console.log("Got error:" + e.message);
    urls = data.split("\n");
    for (var i = urls.length; i--;) {
      url = urls[i];
      if (!url.match(regex)) continue;
      filename = folder + url.substring(url.lastIndexOf('/') + 1);
      http.get(url, function(response) {
                      var file =  fs.createWriteStream(filename);
                      response.on("end", function() {
                        file.end();
                      });
                      response.pipe(file);
                    })
    }
  });