subsrcibe to many topics in kafka using python

Question

I am trying to subscribe to multiple topics in Kafka. I need to execute a function for each topic. here is my code

def consumer_connectionf():
for message1 in consumer:
    message1 = bool(message1.value)
    print(message1, "=this m1")
    drone = collection.find_one({"_id": ObjectId("626736322a7fdc5c8eb486fe")})
    if drone:
     collection.update_one({"_id": ObjectId("626736322a7fdc5c8eb486fe")},{'$set':{'connected':message1}})  
     print('connect ',message1)
     
def consumer_armedg():
for message2 in consumer:
    message2 = bool(message2.value)
    drone = collection.find_one({"_id": ObjectId("626736322a7fdc5c8eb486fe")})
    if drone:
     collection.update_one({"_id": ObjectId("626736322a7fdc5c8eb486fe")},{'$set':{'armed':message2}})  
     print('armed ',message2)
     print('here')
while True:
consumer = KafkaConsumer(bootstrap_servers=['localhost:9092'])
consumer.subscribe(['k_connectin_status','k_armed_status'])
msg = consumer.subscription()

if msg == 'k_connectin_statuds':
    consumer_connectionf(),
#consumer_armed = KafkaConsumer(bootstrap_servers=['localhost:9092'])
#consumer_armed.subscribe('k_armed_status')
#if topic:

#if msg == 'k_armed_statuds':
consumer_armedg();

so, any suggestions!!

Answer 1

If youre using kafka-python you could do something like this:

consumer.subscribe(['k_connectin_status','k_armed_status'])

while True:
    # poll messages each certain ms
    raw_messages = consumer.poll(
        timeout_ms=100, max_records=200
    )
    # for each messages batch
    for topic_partition, messages in raw_messages.items():
        # if message topic is k_connectin_status
        if topic_partition.topic == 'k_connectin_status':
            process_k_connectin_status(messages)
        # if message topic is k_armed_status
        elif topic_partition.topic == 'k_armed_status':
            process_k_armed_status(messages)

More info about consumer poll method is available here https://kafka-python.readthedocs.io/en/master/apidoc/KafkaConsumer.html#kafka.KafkaConsumer.poll