Kafka节点：消费缓慢

Question

I am writing a kafka-node consumer and compared to the consumer in python, it is very slow and seems not to recieve all messages. I tried to adjust all possible parameters, but nothing works.

I use this API: https://github.com/SOHU-Co/kafka-node#consumer

and this is my code:

//--------------------Create a consumer---------------------------------------------------------------------------
var kafka = require('kafka-node');
var Consumer = kafka.Consumer;
var client = new kafka.Client('192.168.2.2:2181');
var consumer = new Consumer(client,
        [{ topic: 'clusterTest1', offset: 0}],
        {
            autoCommit: false,
            fromOffset: 'latest',
            outOfRangeOffset: 'latest',
            fetchMinBytes: 1,
            fetchMaxWaitMs: 50,
            fetchMaxBytes: 52428800

        }
    );

offset = new kafka.Offset(client);
    offset.fetch([
        { topic: 'clusterTest1', partition: 0, time: Date.now(), maxNum: 1 }
    ], function (err, data) {
        // data
        // { 't': { '0': [999] } }
    });


consumer.on('message', function (message) {
    console.log(message.value);
}); 

});

consumer.on('error', function (err) {
    console.log('Error:',err);
});

 consumer.on('offsetOutOfRange', function (topic) {
     console.log('---------Offset out of range--------');
  topic.maxNum = 2;
  offset.fetch([topic], function (err, offsets) {
    if (err) {
      return console.error(err);
    }
    var max= Math.max.apply(null, offsets[topic.topic][topic.partition]);
    consumer.setOffset(topic.topic, topic.partition, max);
  });
}); 

However, this python code works fine:

from kafka import KafkaConsumer

consumer = KafkaConsumer(bootstrap_servers=['192.168.2.2:9092','192.168.2.3:9092','192.168.2.4:9092','192.168.2.5:9092'])
print('Erfolgreich verbunden')
print(consumer.subscribe('clusterTest1'))
print(consumer.subscription())
#print(consumer.DEFAULT_CONFIG)
for msg in consumer:
    print(msg.value)

Thanks for any help in advance.

Answer 1

Solved. I wasn't aware that there are multiple partitions. Now, I am using a consumerGroup and I can receive all messages.