重新發布kafka DLQ主題

Question

重試2次后，我的話題轉到了DLQ。

有什么方法可以按需重新發布dlq主題（手動使用api）？

在這里，我要添加我的代碼，它將包含三個API

1）將發送消息並成功消耗掉。

2）將發送主題並且將引發錯誤，最后將移至DLQ。

3） 將嘗試讀取和發布DLQ的主題（我住的地方）。

 public class KafkaProducerConfig {

private String bootstrapAddress="localhost:9092";

private String groupId = "dlq_topics_group";

@Bean
public ProducerFactory<String, Object> producerFactory() {
    Map<String, Object> configProps = new HashMap<String, Object>();
configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,bootstrapAddress);
    configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    configProps.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    return new  DefaultKafkaProducerFactory<String, Object>(configProps);
}
@Bean
public KafkaTemplate<String, Object> kafkaTemplate() {
return new KafkaTemplate<>(producerFactory());
}

}    

@EnableKafka
@Configuration
public class KafkaConsumerConfig {

private String bootstrapAddress="localhost:9092";

private String groupId = "dlq_topics_group";

@Autowired
KafkaTemplate<Object, Object> template;

private Integer maxRetry = 3;

private Integer concurrency=1;

private Integer pollTimeout=1000;

@Bean
public ConsumerFactory<Object, Object> consumerFactory() {
    Map<String, Object> props = new HashMap<String, Object>();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress);
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    return new DefaultKafkaConsumerFactory<>(props);
}

@Bean
public ConcurrentKafkaListenerContainerFactory<Object, Object> kafkaListenerContainerFactory(
    ConcurrentKafkaListenerContainerFactoryConfigurer configurer) {
  ConcurrentKafkaListenerContainerFactory<Object, Object> factory = new ConcurrentKafkaListenerContainerFactory<>();
  factory.setConsumerFactory(consumerFactory());
    factory.setConcurrency(concurrency);
    factory.getContainerProperties().setPollTimeout(pollTimeout);
    factory.getContainerProperties().setSyncCommits(true);
    factory.getContainerProperties().setAckOnError(false);
    factory.setErrorHandler(new SeekToCurrentErrorHandler(new DeadLetterPublishingRecovererHandler(template), maxRetry));
  configurer.configure(factory, consumerFactory());
  return factory;
}


@Bean
public KafkaConsumer<Object, Object> kafkaConsumer() {
    return new KafkaConsumer<(consumerFactory().getConfigurationProperties());
}
}    


@Controller
@RequestMapping(value="/kafka/controller")
public class KafkaController {

@Autowired
MessageService messageService;


@PostMapping(value="/send/{message}")
public void sendMessage1(@PathVariable String message) {
    messageService.sendMessage(message);
}

@PostMapping(value="/send-kafka-msg/{message}")
public void sendMessage(@PathVariable String message) {
    messageService.sendMessage(message);
}

@PostMapping(value="/resend-dlq-topic")
public void reprocessDLQTopics() {
**//LOGIC TO READ DLQ AND REPROCESS THE TOPICS**
    messageService.reprocessDLQTopics();
}
}


@Service
public class MessageService {

private static final Logger logger = LoggerFactory.getLogger(MessageService.class);

private static final String TOPIC_EX = "kafka-msg-topic-ex";
private static final String TOPIC = "kafka-msg-topic";
private static final String DLQ_TOPICS = "dql_topics";
private static final String DLQ_NOTIFICATION_TOPIC = "dlq_notification_topic";

@Autowired
KafkaTemplate<String, Object> kafkaTemplate;

@Autowired
KafkaAdmin kafkaAdmin;

@Autowired
KafkaConsumer<Object, Object> kafkaConsumer;

public void sendMessage(String message) {
    logger.info("sending message ", message);
    kafkaTemplate.send(TOPIC, message);

}

public void sendMessageEx(String message) {
    logger.info("sending message ", message);
    kafkaTemplate.send(TOPIC, message);
}

@KafkaListener(topics = { TOPIC, TOPIC_EX }, groupId = "group-id")
public void messageReciver(String message) {
    logger.info("message receiver" +kafkaAdmin.getConfig().entrySet().parallelStream().toString());
    throw new RuntimeException();
}

@KafkaListener(id = "DLQ_TOPICS_GROUP", topics = DLQ_TOPICS)
public void dltListen(String dlqMsg) {
    logger.info("Received from DLT: " + dlqMsg.toString());
}

public void reprocessDLQTopics() {
    **//LOGIC TO RE-PUBLISH DLQ TOPICS**
}
}    

Answer 1

不清楚“重新發布dlq主題”是什么意思，但是您可以使用KafkaTemplate將記錄發送到任何主題（包括DLQ）。

如果您是指從DLQ主題中消費並重新發布到原始主題，那么可以，您也可以這樣做。

編輯

這是一種解決方案：

@Component
class Config {

    @Autowired
    private KafkaTemplate<String, String> template;

    @Autowired
    private KafkaListenerEndpointRegistry registry;

    @KafkaListener(id = "so57157134", topics = "so57157134")
    public void listen(String in) {
        System.out.println(in);
    }

    @KafkaListener(id = "so57157134_DLT", topics = "so57157134_DLT", autoStartup = "false")
    public void listenDLT(String in) {
        System.out.println("Re-publish DLT");
        this.template.send("so57157134", in);
    }

    @EventListener
    public void listenForIdleContainer(ListenerContainerIdleEvent event) {
        System.out.println(event);
        this.registry.getListenerContainer("so57157134_DLT").stop();
    }

    public void reprocessDLQTopics() {
        this.registry.getListenerContainer("so57157134_DLT").start();
    }

    @Bean
    public NewTopic topic() {
        return new NewTopic("so57157134", 10, (short) 1);
    }

    @Bean
    public NewTopic topicDLT() {
        return new NewTopic("so57157134_DLT", 10, (short) 1);
    }

}

唯一的問題是，如果重播記錄仍然有故障，將會發生什么情況。 容器將永遠不會閑置，因此您將永遠循環。 一種解決方案是以某種方式檢測循環並在該條件下停止容器。