Kafka 9 KafkaConsumer multiple doesn't seem to process messages in parallel

Question

I have a Kafka cluster on 3 servers, so i created a topic of partition 3 with a replication factor of 3.

bin/kafka-topics.sh --create --zookeeper tstaapp001:52181,ewdlxsrv283:52181,devcapp001:52181 --replication-factor 3 --partitions 3 --topic TEST_PARTITION

I have a sample program of multi-threaded consumer copied from a blog using Kafka 9.0 but it doesn't really receives messages in parallel when running on my 8-core PC. Can anyone tell me if there could be anything wrong with my setup or my code?

public class Consumer implements Runnable {
    private final KafkaConsumer<String, String> consumer;
    private final List<String> topics;
    private final int id;

    public Consumer(int id, String groupId, List<String> topics) {
        this.id = id;
        this.topics = topics;
        Properties props = new Properties();
        props.put("bootstrap.servers",
                "tstaapp001:59092,ewdlxsrv283:59092,devcapp001:59092");
        props.put("group.id", groupId);
        props.put("key.deserializer", StringDeserializer.class.getName());
        props.put("value.deserializer", StringDeserializer.class.getName());
        this.consumer = new KafkaConsumer<>(props);
    }

    @Override
    public void run() {
        try {
            consumer.subscribe(topics);

            while (true) {
                ConsumerRecords<String, String> records = consumer
                        .poll(Long.MAX_VALUE);
                for (ConsumerRecord<String, String> record : records) {
                    Map<String, Object> data = new HashMap<>();
                    data.put("partition", record.partition());
                    data.put("offset", record.offset());
                    data.put("value", record.value());
                    System.out.println(this.id + ": " + data);
                }
            }
        } catch (WakeupException e) {
            // ignore for shutdown
        } finally {
            consumer.close();
        }
    }

    public void shutdown() {
        consumer.wakeup();
    }

    public static void main(String[] args) {
        int numConsumers = 3;
        String groupId = "TEST-GROUP";
        List<String> topics = Arrays.asList("TEST_PARTITION");

        ExecutorService executor =     Executors.newFixedThreadPool(numConsumers);

        final List<Consumer> consumers = new ArrayList<>();
        for (int i = 0; i < numConsumers; i++) {
            Consumer consumer = new Consumer(i, groupId, topics);
            consumers.add(consumer);
            executor.submit(consumer);
        }

        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                for (Consumer consumer : consumers) {
                    consumer.shutdown();
                }
                executor.shutdown();
                try {
                    executor.awaitTermination(5000, TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        });
    }
}

The following is the output:

1: {partition=1, offset=141, value=TEST1}
1: {partition=1, offset=142, value=TEST4}
1: {partition=1, offset=143, value=TEST7}
1: {partition=1, offset=144, value=TEST10}
1: {partition=1, offset=145, value=TEST13}
1: {partition=1, offset=146, value=TEST16}
1: {partition=1, offset=147, value=TEST19}
1: {partition=1, offset=148, value=TEST22}
1: {partition=1, offset=149, value=TEST25}
1: {partition=1, offset=150, value=TEST28}
1: {partition=1, offset=151, value=TEST31}
1: {partition=1, offset=152, value=TEST34}
1: {partition=1, offset=153, value=TEST37}
1: {partition=1, offset=154, value=TEST40}
1: {partition=1, offset=155, value=TEST43}
1: {partition=1, offset=156, value=TEST46}
1: {partition=1, offset=157, value=TEST49}
0: {partition=0, offset=145, value=TEST2}
0: {partition=0, offset=146, value=TEST5}
0: {partition=0, offset=147, value=TEST8}
0: {partition=0, offset=148, value=TEST11}
0: {partition=0, offset=149, value=TEST14}
0: {partition=0, offset=150, value=TEST17}
0: {partition=0, offset=151, value=TEST20}
0: {partition=0, offset=152, value=TEST23}
0: {partition=0, offset=153, value=TEST26}
0: {partition=0, offset=154, value=TEST29}
0: {partition=0, offset=155, value=TEST32}
0: {partition=0, offset=156, value=TEST35}
0: {partition=0, offset=157, value=TEST38}
0: {partition=0, offset=158, value=TEST41}
0: {partition=0, offset=159, value=TEST44}
0: {partition=0, offset=160, value=TEST47}
2: {partition=2, offset=142, value=TEST0}
2: {partition=2, offset=143, value=TEST3}
2: {partition=2, offset=144, value=TEST6}
2: {partition=2, offset=145, value=TEST9}
2: {partition=2, offset=146, value=TEST12}
2: {partition=2, offset=147, value=TEST15}
2: {partition=2, offset=148, value=TEST18}
2: {partition=2, offset=149, value=TEST21}
2: {partition=2, offset=150, value=TEST24}
2: {partition=2, offset=151, value=TEST27}
2: {partition=2, offset=152, value=TEST30}
2: {partition=2, offset=153, value=TEST33}
2: {partition=2, offset=154, value=TEST36}
2: {partition=2, offset=155, value=TEST39}
2: {partition=2, offset=156, value=TEST42}
2: {partition=2, offset=157, value=TEST45}
2: {partition=2, offset=158, value=TEST48}

Answer 1

Martin,

Your consumer code looks good. Have you tried running multiple instances of your consumer. For example take a look at my consumer https://github.com/sdpatil/KafkaAPIClient/blob/master/src/main/java/com/spnotes/kafka/partition/Consumer.java

If you have say topic with 3 partitions and you want to consume messages in parallel for all 3 of them (You cannot consume one partition in parallel), start 3 instances of your consumer and you will notice that Kafka assigns one partition to each of the consumer and they will receive messages in parallel. In my sample consumer there is ConsumerRebalanceListener, that will help you debug what is happening

Sunil

Answer 2

I would suggest you to use rapids-kafka-client , a library to that parallelism stuff for you.

ConsumerConfig.<String, String>builder()
.prop(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName())
.prop(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName())
.prop(GROUP_ID_CONFIG, "stocks")
.topics("stock_changed")
.consumers(7)
.callback((ctx, record) -> {
  log.info("status=consumed, value={}", record.value());
})
.build()
.consume()
.waitFor();