Node.JS——需要帮助理解原生流

Question

I'm trying to understand how the native implementation of streams works. Here's the code:

 const Stream = require('stream'); // define a custom class to read my data into the stream class SourceWrapper extends Stream.Readable { constructor(opt, content) { super(opt); this.content = content; this.len = content.length; this.index = 0; } _read() { let i = this.index++; if (i >= this.len) this.push(null); else { this.push(this.content[i]); } } } // generate some data const arr = (new Array(10000000)).fill(1); // declare the streams const firstStream = new SourceWrapper({objectMode: true}, arr); const transform = (x, enc, next) => next(undefined, x * Math.random(x, 10)); const firstMapStream = new Stream.Transform({objectMode: true}); firstMapStream._transform = transform; const secondMapStream = new Stream.Transform({objectMode: true}); secondMapStream._transform = transform; // create a promise to measure execution time const start = new Date(); new Promise((resolve, reject) => { firstStream .pipe(firstMapStream) .pipe(secondMapStream) .on('finish', () => resolve(new Date())); }) .then((end) => console.log('execTime', end - start));

The problem is that it works on small data sets (ie [1,2,3,4] ), but seems to terminate shortly after being run on a large set.

What am I missing? Has it something to do with the objectMode ?

Appreciate any help.

Answer 1

The reason is that someone should to read data from the streams with binding data event listener. I've rewrite your code to make it clear to understand the issue. Also i fixed wrong index counting which skipped zero index.

'use strict';
const Stream = require('stream');

// define a custom class to read my data into the stream
class SourceWrapper extends Stream.Readable {
  constructor(opt, content) {
    super(opt);
    this.content = content;
    this.len = content.length;
    this.index = 0;
  }

  _read() {
    let i = this.index;
    if (i >= this.len) {
      this.push(null);
    } else {
      this.push(this.content[i]);
    }
    this.index++;
  }
}


const transform = (x, enc, next) => next(undefined, x * Math.random(x, 10));

const transform1 = new Stream.Transform({objectMode: true});
transform1._transform = transform;

const transform2 = new Stream.Transform({objectMode: true});
transform2._transform = transform;


const write = new Stream.Writable({
    objectMode: true,
    write(value, enc, next) {
        // Do something like writing...
        next();
    }
});


// generate some data 
const arr = (new Array(1000000)).fill(1);
const read = new SourceWrapper({objectMode: true}, arr);

new Promise((resolve, reject) => {
    read
    .pipe(transform1)
    .pipe(transform2)
    .pipe(write)
    .on('finish', () => {
        resolve();
    });
})
.then(() => {
    console.log('Done');
});