Kafka订户客户端0.10.2-Spring集成-INVALID_TOPIC_EXCEPTION

Question

I'm trying to subscribe Kafka server using Spring integration. I'm able to see kafka connect log in application log. Given the below the same.

Please let me know if i'm missing any necessary configuration here.

16:56:02.959 [main] INFO org.apache.kafka.clients.consumer.ConsumerConfig 

    ConsumerConfig values: 
    auto.commit.interval.ms = 100
    auto.offset.reset = latest
    bootstrap.servers = [10.176.138.40:9092, 10.176.138.208:9092, 
         10.176.138.169:9092, 10.176.138.184:9092, 10.176.138.188:9092]
    check.crcs = true
    client.id = 
    connections.max.idle.ms = 540000
    enable.auto.commit = true
    exclude.internal.topics = true
    fetch.max.bytes = 52428800
    fetch.max.wait.ms = 500
    fetch.min.bytes = 1
    group.id = amqunpnotifyd-dev-thilak
    heartbeat.interval.ms = 1000
    interceptor.classes = null
    key.deserializer = class 
    org.apache.kafka.common.serialization.StringDeserializer
    max.partition.fetch.bytes = 1048576
    max.poll.interval.ms = 300000
    max.poll.records = 500
    metadata.max.age.ms = 300000
    metric.reporters = []
    metrics.num.samples = 2
    metrics.recording.level = INFO
    metrics.sample.window.ms = 30000
    partition.assignment.strategy = [class 
    org.apache.kafka.clients.consumer.RangeAssignor]
    receive.buffer.bytes = 65536
    reconnect.backoff.ms = 5000
    request.timeout.ms = 305000
    retry.backoff.ms = 100
    sasl.jaas.config = null
    sasl.kerberos.kinit.cmd = /usr/bin/kinit
    sasl.kerberos.min.time.before.relogin = 60000
    sasl.kerberos.service.name = null
    sasl.kerberos.ticket.renew.jitter = 0.05
    sasl.kerberos.ticket.renew.window.factor = 0.8
    sasl.mechanism = GSSAPI
    security.protocol = PLAINTEXT
    send.buffer.bytes = 131072
    session.timeout.ms = 10000
    ssl.cipher.suites = null
    ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
    ssl.endpoint.identification.algorithm = null
    ssl.key.password = null
    ssl.keymanager.algorithm = SunX509
    ssl.keystore.location = null
    ssl.keystore.password = null
    ssl.keystore.type = JKS
    ssl.protocol = TLS
    ssl.provider = null
    ssl.secure.random.implementation = null
    ssl.trustmanager.algorithm = PKIX
    ssl.truststore.location = null
    ssl.truststore.password = null
    ssl.truststore.type = JKS
    value.deserializer = class org.apache.kafka.common.serialization.StringDeserializer


16:56:02.967 [main] INFO org.apache.kafka.clients.consumer.ConsumerConfig ConsumerConfig values: 
    auto.commit.interval.ms = 100
    auto.offset.reset = latest
    bootstrap.servers = [10.176.138.40:9092, 10.176.138.208:9092, 
      10.176.138.169:9092, 10.176.138.184:9092, 10.176.138.188:9092]
    check.crcs = true
    client.id = consumer-1
    connections.max.idle.ms = 540000
    enable.auto.commit = true
    exclude.internal.topics = true
    fetch.max.bytes = 52428800
    fetch.max.wait.ms = 500
    fetch.min.bytes = 1
    group.id = amqunpnotifyd-dev-thilak
    heartbeat.interval.ms = 1000
    interceptor.classes = null
    key.deserializer = class 
    org.apache.kafka.common.serialization.StringDeserializer
    max.partition.fetch.bytes = 1048576
    max.poll.interval.ms = 300000
    max.poll.records = 500
    metadata.max.age.ms = 300000
    metric.reporters = []
    metrics.num.samples = 2
    metrics.recording.level = INFO
    metrics.sample.window.ms = 30000
    partition.assignment.strategy = [class 
    org.apache.kafka.clients.consumer.RangeAssignor]
    receive.buffer.bytes = 65536
    reconnect.backoff.ms = 5000
    request.timeout.ms = 305000
    retry.backoff.ms = 100
    sasl.jaas.config = null
    sasl.kerberos.kinit.cmd = /usr/bin/kinit
    sasl.kerberos.min.time.before.relogin = 60000
    sasl.kerberos.service.name = null
    sasl.kerberos.ticket.renew.jitter = 0.05
    sasl.kerberos.ticket.renew.window.factor = 0.8
    sasl.mechanism = GSSAPI
    security.protocol = PLAINTEXT
    send.buffer.bytes = 131072
    session.timeout.ms = 10000
    ssl.cipher.suites = null
    ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
    ssl.endpoint.identification.algorithm = null
    ssl.key.password = null
    ssl.keymanager.algorithm = SunX509
    ssl.keystore.location = null
    ssl.keystore.password = null
    ssl.keystore.type = JKS
    ssl.protocol = TLS
    ssl.provider = null
    ssl.secure.random.implementation = null
    ssl.trustmanager.algorithm = PKIX
    ssl.truststore.location = null
    ssl.truststore.password = null
    ssl.truststore.type = JKS
    value.deserializer = class 
    org.apache.kafka.common.serialization.StringDeserializer

I'm getting following Warn in the logger even though topic exists.

WARN org.apache.kafka.clients.NetworkClient Error while fetching metadata with correlation id 3 : {"notification.email.click"=INVALID_TOPIC_EXCEPTION}

Due to which topic is not asssigned to my consumer group.

Answer 1

As the documentation say this particular exception occurs when "The client has attempted to perform an operation on an invalid topic."

This generally happens when the client is not able to publish due to a host of reasons

• Topic not created.
• Firewall configuration between client and kafka server.
• Etc

Since in your case the your configuration is not know. You can start by debugging the above reasons.

Answer 2

Your topic name is "notification.email.click" (including quotes). Double quote ( " ) is not a legal character for a topic name.

According to Kafka Docs, legal characters for topic naming include:

1. ASCII alphanumerics
2. .
3. _
4. -

The source code for 0.10.2 defines the following legal characters:

val legalChars = "[a-zA-Z0-9\\._\\-]"
private val maxNameLength = 249
private val rgx = new Regex(legalChars + "+")