How to consume the latest message from Kafka-consumer using NodeJs?

Question

I have created a NodeJS application to insert data into a MongoDB collection. This database insertion is done by using a Kafka. Kafka-node is the plugin I have used to call Kafka.

I can create the topic and send a message to the consumer at the producer level. The message and topic are taken from the POST request.
This is how I call the Kafka. Parameters are topic and message.

Every time I call this API, the producer is creating a new message and sent it to the consumer. In each call, all previous messages will be returned to the consumer.

I have used the fromOffset: 'earliest' and fromOffset: 'latest' options to restrict the previous messages, not working.

Can anyone give me a suggestion? Version of Kafka-node

"kafka-node": "^5.0.0",

Code I have used

var kafka = require('kafka-node');
const {MongoClient} = require('mongodb');
var url = 'mongodb://127.0.0.1:27017/';
const mongoClient = new MongoClient(url);
var Producer = kafka.Producer,
    client = new kafka.KafkaClient(),
    offset = new kafka.Offset(client),
    Consumer = kafka.Consumer,
    producer = new Producer(client);
producer.on('ready', function () {
        console.log('Producer is ready');
    });
producer.on('error', function (err) {
        console.log('Producer is in error state');
        console.log(err);
    })
const createProducer = async(req,res,next) => {
  var topic = req.body.topic;
  var sentMessage = JSON.stringify(req.body.messages);
  producer.send(payloads, async function( err, data) {
        })
  client = new kafka.KafkaClient(),
  consumer = new Consumer(client,
            [
                { topic: topic, partition: 0 }
            ],
            {
                autoCommit: false,
                fromOffset: 'earliest'
            }
        );
   consumer.on('message', async function (message) {            

      console.log("Message : "+JSON.stringify(message))
      try {            
        var currentdate = new Date(); 
        var datetime = "Last Sync: " + currentdate.getDate() + "/"
            + (currentdate.getMonth()+1)  + "/" 
            + currentdate.getFullYear() + " @ "  
            + currentdate.getHours() + ":"  
            + currentdate.getMinutes() + ":" 
            + currentdate.getSeconds();
        var abb = await createListing(mongoClient,
            {
              topic: topic,
              message: sentMessage,
              time: datetime
             }
             );
             
      } catch (e) {
         console.error(":"+e);
      }
       finally {
                
       }
    });
     await mongoClient.close();
        res.send({
            message: 'Successfully send data from producer',
            payloads: payloads
        })

    async function createListing(client, newListing){
        await mongoClient.connect();
        const result = await 
       
      client.db("sample_airbnb").collection("listingsAndReviews").insertOne(newListing);
        console.log(`New listing created with the following id: ${result.insertedId}`);
        return result.insertedId;
      }
}

The Messages consumed at the consumer are

Thanks,

Answer 1

You consumer will always consume all offsets that have not been marked consumed by its consumer group before.

This means that after consuming a given message (or a batch of messages), you need to commit the highest consumed offset to your Kafka cluster, to effectively mark those messages as consumed. Only then will your consumer group not re-consume those messages on startup.

To commit your offsets, you can either use kafka.js's autoCommit feature (which you explicitly disabled in your implementation), or manually commit your offsets using the API provided by kafka.js.

You can find the documentation to both here: https://kafka.js.org/docs/consuming#a-name-auto-commit-a-autocommit

Answer 2

I made some changes in the code, Now I can retrieve the latest message from my topic.

I have created consumer inside the offset.fetchLatestOffsets([topics],cb) , and made some changes in the consumer options.

var payloads = [
            { topic: topicName, messages: messageTotopic, partition: 0}
        ];
producer.send(payloads, async function( err, data) {
        })
var client = new kafka.KafkaClient();
 offset.fetchLatestOffsets([topic], async function (error, offsets) {
            if (error)
            console.log(error);
            offsetA = JSON.stringify(offsets[topic][0])
            console.log('offset Value:: '+offsetA);
            var consumer = new Consumer(
                client,
                      [
                       { topic:topic,
                         partition:0,
                         offset: offsetA-1, // Offset value starts from 0
                        }
                        ],
                        {
                            autoCommit: false,
                            fromOffset: true,
                        }
            );
            consumer.on('message', async function (message) {
                console.log("Message from last offset:: "+JSON.stringify(message)); // will return the latest message.
                consumer.close();
            });
        });

Using this way I am able to overcome the memory leakage issue related to the event emitters in the KafkaClient .