修改数据后的nodejs管道流

Question

I am learning about streaming with nodejs, I understand the examples shown in the request npm module ;

request(url).pipe(fs.createWriteStream('./filename.json'))

But there are two parts of my problem.

Case 1:

function fetchSitemaps() {
    return requestAsync(url).then(data => {
        const $ = cheerio.load(data);
        let urls = [];
        $("loc").each((i, e) => urls.push($(e).text()));
        fs.writeFileSync('./sitemaps.json', JSON.stringify(urls))
    })
}

I want to convert the above from writeFileSync to createWriteStream , but how do I keep appending data to an array which is in JSON format?

Case 2:

function fetchLyricUrls() {
    let sitemaps = JSON.parse(fs.readFileSync('./sitemaps.json'));

    sitemaps.forEach((sitemap, i) => {
        let fileName = i + '.json';

        if(url_pat.exec(sitemap)) {
            fileName = url_pat.exec(sitemap)[1] + '.json';
        }

        requestAsync(url).then(data => {
            const $ = cheerio.load(data);
            let urls = [];
            $("loc").each((i, e) => urls.push($(e).text()));    
            return urls;
        }).then(urls => {
            let allUrls = [];
            urls.map(u => {
                return requestAsync(u).then(sm => {
                    const $ = cheerio.load(sm);
                    $("loc").each((i, e) => allUrls.push($(e).text()))
                    fs.writeFileSync('./lyrics.json', JSON.stringify(allUrls))
                    return allUrls;
                });
            });
        });
    });
}

The first part of the problem is same, appending to a json data using writeStream, but this time, I want to parse the the html data and get some text, which I want to send using stream, not the html data as a whole.

Answer 1

So let's split up the answers

Case 1

First of all I'd try to keep the data as a stream and try not to accumulate it. So in essence, instead of loading the whole sitemap and then parsing it, I'd use something like the xml-nodes so that the nodes are a separate stream. Then my module scramjet would come to transform

    const request = require('request');
    const xmlNodes = require('xml-nodes');
    const writable = fs.createWritableStream('./sitemaps.json');
    const cheerio = require('cheerio');
    const scramjet = require('scramjet');

    writable.write('[');

    let first = 0;
    request('http://example.com/sitemap.xml')
            // this fetches your sitemap
        .on('end', () => writable.end("]"))
            // when the stream ends, this will end the sitemaps.json
        .pipe(xmlNodes('loc'))
            // this extracts your "loc" nodes
        .pipe(new scramjet.DataStream())
            // this creates a mappable stream
        .map((nodeString) => cheerio('loc', nodeString).text())
            // this extracts the text as in your question
        .map((url) => (first++ ? ',' : '') + JSON.stringify(url))
            // this makes sure that strings are nicely escaped
            // and prepends them with a comma on every node, but first one
        .pipe(writable, {end: false})
            // and this will push all your entries to the writable stream

Case 2

Here you'll need to do something similar, although if case 1 is an immediate step, then I'd suggest to store the files in lines of JSONs, not an array. It'd make easier to stream that way.