nodejs和kafka键分区程序

Question

I am trying to use keyed partitioning with kafka Highlevelproducer, on a 4 partitions topic I have the code below:

   var kafka = require('kafka-node'),
      HighLevelProducer = kafka.HighLevelProducer,
      client = new kafka.Client(Host+":"+Port,client_id),
      producer = new HighLevelProducer(client,{partitionerType: 3});
.
.
   theKey = theKey+1;
   if ( theKey > Nb_key ) {
      theKey = 0;
   }
   var payloads = [
        { topic: Topic, key: theKey, messages: JSON.stringify({"hello": "world", "Timestamp": +timestamp}) }
      ];
      producer.send(payloads, function (err, data) {
      });

I checked that messages have the correct key value, but still, all messages are sent to partition 1. Do you see anything wrong with what I am doing ?

Thanks

Answer 1

Looking at the code, I think I understood the problem. The baseProducer is doing a hash on the key as a string, and the result is always the same, so always same partition. I wrote my own custom partitioner, doing a simple modulo on the key with the nb of partitions, it works perfectly. See code below:

   var MyPartitioner = function (partitions, key) {
   key = key || '0';
   var index = parseInt(key) % partitions.length;
   return partitions[index];
   };
   var kafka = require('kafka-node'),
      HighLevelProducer = kafka.HighLevelProducer,
      client = new kafka.Client(Host+":"+Port,client_id),
      producer = new HighLevelProducer(client,{requireAcks: 0,partitionerType: 4},MyPartitioner );
      producer.on('ready', function () {
      timer.setInterval(send_data, '',SendIntervalInUsecs+'u');
   });