监控 Kafka Consumer Lag 并生成警报

Question

How do I monitor Kafka consumer lag and generate emails/alerts?Below is my requirement

1. I want to trigger an email when a messages older than 1 day on the topic.

I am using Spring boot micro services,Java 8

@Configuration
public class KafkaConsumerConfig 
{
    @Value(value = "${kafka.bootstrapAddress}")
    private String bootstrapAddress;
 
    @Value(value = "${general.topic.group.id}")
    private String groupId;
 
    @Value(value = "${user.topic.group.id}")
    private String userGroupId;
 
    // 1. Consume string data from Kafka
 
    @Bean
    public ConsumerFactory<Integer, String>  consumerFactory() {
        Map<String, Object> props = new HashMap<>();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, 
                StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, 
                StringDeserializer.class);
        props.put(JsonDeserializer.TRUSTED_PACKAGES, "*");
        return new DefaultKafkaConsumerFactory<>(props);
    }
    
    
 
    @Bean
    KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<Integer, String>>
                        kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<Integer, String> factory =
                                new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        factory.setConcurrency(3);
        factory.getContainerProperties().setPollTimeout(3000);
        return factory;
    }
    
 //not compiling
    public KafkaMessageListenerContainer<Integer, String> m1()
    {
        ContainerProperties containerProps = new ContainerProperties("topic1", "topic2");
        containerProps.setMessageListener(new MessageListener<Integer, String>() { 
 
            @Override
            public void onMessage(ConsumerRecord<Integer, String> data) {
                // TODO Auto-generated method stub
                
            }
           
        });
        DefaultKafkaConsumerFactory<Integer, String> cf =
                                new DefaultKafkaConsumerFactory<>(consumerFactory()); //not compiling
        KafkaMessageListenerContainer<Integer, String> container =
                                new KafkaMessageListenerContainer<>(cf, containerProps);
        return container;
    }

Answer 1

You can use Burrow instead of trying to write your own solution.