使用 Avro Schema 注冊表的 Kafka 消費者單元測試失敗

Question

我正在編寫一個消費者，它會收聽 Kafka 主題並在消息可用時使用消息。 我通過在本地運行 Kafka 測試了邏輯/代碼，它運行良好。

在編寫單元/組件測試用例時，它因 avro 架構注冊表 url 錯誤而失敗。 我嘗試了 inte.net 上可用的不同選項，但找不到任何可用的選項。 我不確定我的方法是否正確。 請幫忙。

聽眾 Class

@KafkaListener(topics = "positionmgmt.v1", containerFactory = "genericKafkaListenerFactory")
    public void receive(ConsumerRecord<String, GenericRecord> consumerRecord) {
        try {
            GenericRecord generic = consumerRecord.value();
            Object obj = generic.get("metadata");

            ObjectMapper mapper = new ObjectMapper();

            Header headerMetaData = mapper.readValue(obj.toString(), Header.class);

            System.out.println("Received payload :   " + consumerRecord.value());

            //Call backend with details in GenericRecord 

        }catch (Exception e){
            System.out.println("Exception while reading message from Kafka " + e );
        }

卡夫卡配置

@Bean
    public ConcurrentKafkaListenerContainerFactory<String, GenericRecord> genericKafkaListenerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, GenericRecord> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(genericConsumerFactory());
        return factory;
    }

public ConsumerFactory<String, GenericRecord> genericConsumerFactory() {
        Map<String, Object> config = new HashMap<>();

        config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:9092");
        config.put(ConsumerConfig.GROUP_ID_CONFIG, "group_id");
        config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaAvroDeserializer.class);
        config.put(KafkaAvroDeserializerConfig.SCHEMA_REGISTRY_URL_CONFIG,"http://localhost:8081");
        return new DefaultKafkaConsumerFactory<>(config);
    }

Avro 模式

{
   "type":"record",
   "name":"KafkaEvent",
   "namespace":"com.ms.model.avro",
   "fields":[
      {
         "name":"metadata",
         "type":{
            "name":"metadata",
            "type":"record",
            "fields":[
               {
                  "name":"correlationid",
                  "type":"string",
                  "doc":"this is corrleation id for transaction"
               },
               {
                  "name":"subject",
                  "type":"string",
                  "doc":"this is subject for transaction"
               },
               {
                  "name":"version",
                  "type":"string",
                  "doc":"this is version for transaction"
               }
            ]
         }
      },
      {
         "name":"name",
         "type":"string"
      },
      {
         "name":"dept",
         "type":"string"
      },
      {
         "name":"empnumber",
         "type":"string"
      }
   ]
}

這是我試過的測試代碼......

@ComponentTest
    @RunWith(SpringRunner.class)
    @EmbeddedKafka(partitions = 1, topics = { "positionmgmt.v1" })
    @SpringBootTest(classes={Application.class})
    @DirtiesContext
    public class ConsumeKafkaMessageTest {

      private static final String TEST_TOPIC = "positionmgmt.v1";

      @Autowired(required=true)
      EmbeddedKafkaBroker embeddedKafkaBroker;

      private Schema schema;

      private  SchemaRegistryClient schemaRegistry;
      private  KafkaAvroSerializer avroSerializer;
      private  KafkaAvroDeserializer avroDeserializer;

      private MockSchemaRegistryClient mockSchemaRegistryClient = new MockSchemaRegistryClient();
      private String registryUrl = "unused";

      private String avroSchema = string representation of avro schema

      @BeforeEach
      public void setUp() throws Exception {
        Schema.Parser parser = new Schema.Parser();
        schema = parser.parse(avroSchema);

        mockSchemaRegistryClient.register("Vendors-value", schema);
      }

      @Test
      public void consumeKafkaMessage_receive_sucess() {

        Schema metadataSchema = schema.getField("metadata").schema();
        GenericRecord metadata = new GenericData.Record(metadataSchema);
        metadata.put("version", "1.0");
        metadata.put("correlationid", "correlationid");
        metadata.put("subject", "metadata");

        GenericRecord record = new GenericData.Record(schema);
        record.put("metadata", metadata);
        record.put("name", "ABC");
        record.put("dept", "XYZ");

        Consumer<String, GenericRecord> consumer = configureConsumer();
        Producer<String, GenericRecord> producer = configureProducer();

        ProducerRecord<String, GenericRecord> prodRecord = new ProducerRecord<String, GenericRecord>(TEST_TOPIC, record);

        producer.send(prodRecord);

        ConsumerRecord<String, GenericRecord> singleRecord = KafkaTestUtils.getSingleRecord(consumer, TEST_TOPIC);
        assertNotNull(singleRecord.value());

        consumer.close();
        producer.close();

      }

      private Consumer<String, GenericRecord> configureConsumer() {
        Map<String, Object> consumerProps = KafkaTestUtils.consumerProps("groupid", "true", embeddedKafkaBroker);
        consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        Consumer<String, GenericRecord> consumer = new DefaultKafkaConsumerFactory<String, GenericRecord>(consumerProps).createConsumer();
        consumer.subscribe(Collections.singleton(TEST_TOPIC));
        return consumer;
      }

      private Producer<String, GenericRecord> configureProducer() {
        Map<String, Object> producerProps = new HashMap<>(KafkaTestUtils.producerProps(embeddedKafkaBroker));
        producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
        producerProps.put(KafkaAvroSerializerConfig.SCHEMA_REGISTRY_URL_CONFIG, mockSchemaRegistryClient);
        producerProps.put(KafkaAvroSerializerConfig.AUTO_REGISTER_SCHEMAS, "false");
        return new DefaultKafkaProducerFactory<String, GenericRecord>(producerProps).createProducer();
      }

}

錯誤

component.com.ms.listener.ConsumeKafkaMessageTest > consumeKafkaMessage_receive_sucess() FAILED
    org.apache.kafka.common.KafkaException: Failed to construct kafka producer
        at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:457)
        at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:289)
        at org.springframework.kafka.core.DefaultKafkaProducerFactory.createKafkaProducer(DefaultKafkaProducerFactory.java:318)
        at org.springframework.kafka.core.DefaultKafkaProducerFactory.createProducer(DefaultKafkaProducerFactory.java:305)
        at component.com.ms.listener.ConsumeKafkaMessageTest.configureProducer(ConsumeKafkaMessageTest.java:125)
        at component.com.ms.listener.ConsumeKafkaMessageTest.consumeKafkaMessage_receive_sucess(ConsumeKafkaMessageTest.java:97)

        Caused by:
        io.confluent.common.config.ConfigException: Invalid value io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient@20751870 for configuration schema.registry.url: Expected a comma separated list.
            at io.confluent.common.config.ConfigDef.parseType(ConfigDef.java:345)
            at io.confluent.common.config.ConfigDef.parse(ConfigDef.java:249)
            at io.confluent.common.config.AbstractConfig.<init>(AbstractConfig.java:78)
            at io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig.<init>(AbstractKafkaAvroSerDeConfig.java:105)
            at io.confluent.kafka.serializers.KafkaAvroSerializerConfig.<init>(KafkaAvroSerializerConfig.java:32)
            at io.confluent.kafka.serializers.KafkaAvroSerializer.configure(KafkaAvroSerializer.java:48)
            at org.apache.kafka.common.serialization.ExtendedSerializer$Wrapper.configure(ExtendedSerializer.java:60)
            at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:372)
            ... 5 more

Answer 1

我進行了一些調查，發現問題出在KafkaAvroSerializer / Deserializer使用的CashedSchemaRegistryClient中。 它用於從Confluent Schema Registry中獲取架構定義。

您已經在本地擁有架構定義，因此您無需為它們進入架構注冊表。 （至少在您的測試中）

我有一個類似的問題，我通過創建一個自定義的KafkaAvroSerializer / KafkaAvroDeserializer來解決了。

這是KafkaAvroSerializer的示例。 這很簡單。 您只需要擴展提供的KafkaAvroSerializer並告訴他使用MockSchemaRegistryClient。

public class CustomKafkaAvroSerializer extends KafkaAvroSerializer {
    public CustomKafkaAvroSerializer() {
        super();
        super.schemaRegistry = new MockSchemaRegistryClient();
    }

    public CustomKafkaAvroSerializer(SchemaRegistryClient client) {
        super(new MockSchemaRegistryClient());
    }

    public CustomKafkaAvroSerializer(SchemaRegistryClient client, Map<String, ?> props) {
        super(new MockSchemaRegistryClient(), props);
    }
}

這是KafkaAvroDeserializer的示例。 調用反序列化方法時，您需要告訴他要使用哪種模式。

public class CustomKafkaAvroDeserializer extends KafkaAvroDeserializer {
    @Override
    public Object deserialize(String topic, byte[] bytes) {
        this.schemaRegistry = getMockClient(KafkaEvent.SCHEMA$);  
        return super.deserialize(topic, bytes);
    }

    private static SchemaRegistryClient getMockClient(final Schema schema$) {
        return new MockSchemaRegistryClient() {
            @Override
            public synchronized Schema getById(int id) {
                return schema$;
            }
        };
    }
}

最后一步是告訴spring使用創建的Serializer / Deserializer

spring.kafka.producer.properties.schema.registry.url= not-used
spring.kafka.producer.value-serializer = CustomKafkaAvroSerializer
spring.kafka.producer.key-serializer = org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.group-id = showcase-producer-id

spring.kafka.consumer.properties.schema.registry.url= not-used
spring.kafka.consumer.value-deserializer = CustomKafkaAvroDeserializer
spring.kafka.consumer.key-deserializer = org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.group-id = showcase-consumer-id
spring.kafka.auto.offset.reset = earliest

spring.kafka.producer.auto.register.schemas= true
spring.kafka.properties.specific.avro.reader= true

我寫了一篇簡短的博客文章： https : //medium.com/@igorvlahek1/no-need-for-schema-registry-in-your-spring-kafka-tests-a5b81468a0e1?source=friends_link&sk=e55f73b86504e9f577e259181c8d0e23

鏈接到工作示例項目： https : //github.com/ivlahek/kafka-avro-without-registry

Answer 2

@ivlahek 的答案是有效的，但是如果你在 3 年后看這個例子，你可能想要對 CustomKafkaAvroDeserializer 做一些小的修改

private static SchemaRegistryClient getMockClient(final Schema schema) {
        return new MockSchemaRegistryClient() {

     @Override
     public ParsedSchema getSchemaBySubjectAndId(String subject, int id)
                    throws IOException, RestClientException {
         return new AvroSchema(schema);
     }            
 };
}

Answer 3

如錯誤所示，您需要在生產者配置中提供一個字符串給注冊表，而不是對象。

由於您使用的是Mock類，因此該字符串可以是任何...

但是，您需要在給定注冊表實例的情況下構造序列化程序

Serializer serializer = new KafkaAvroSerializer(mockSchemaRegistry);
 // make config map with ("schema.registry.url", "unused") 
serializer.configure(config, false);

否則，它將嘗試創建一個非模擬的客戶端

並將其放入屬性

producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, serializer);

Answer 4

如果您的@KafkaListener 在測試類中，那么您可以在 StringDeserializer 中讀取它，然后手動將其轉換為所需的類

    @Autowired
    private MyKafkaAvroDeserializer myKafkaAvroDeserializer;

    @KafkaListener( topics = "test")
    public void inputData(ConsumerRecord<?, ?> consumerRecord) {
        log.info("received payload='{}'", consumerRecord.toString(),consumerRecord.value());

        GenericRecord genericRecord = (GenericRecord)myKafkaAvroDeserializer.deserialize("test",consumerRecord.value().toString().getBytes(StandardCharsets.UTF_8));


        Myclass myclass = (Myclass) SpecificData.get().deepCopy(Myclass.SCHEMA$, genericRecord);
}

@Component
public class MyKafkaAvroDeserializer extends KafkaAvroDeserializer {
    @Override
    public Object deserialize(String topic, byte[] bytes) {

            this.schemaRegistry = getMockClient(Myclass.SCHEMA$);

        return super.deserialize(topic, bytes);
    }



    private static SchemaRegistryClient getMockClient(final Schema schema$) {
        return new MockSchemaRegistryClient() {
            @Override
            public synchronized org.apache.avro.Schema getById(int id) {
                return schema$;
            }
        };
    }
}

記住在 application.yml 中添加 schema registry 和 key/value 序列化器，雖然它不會被使用

    consumer:
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer

    properties:
      schema.registry.url :http://localhost:8080