python threads and Queue messages between them

Question

I have a program that contains two parts, they work at the same time using threads,they communicate using a Queue.

import kafka
import time
from queue import Queue 
from threading import Thread 

# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q): 
    consumer = kafka.KafkaConsumer('mytopic',bootstrap_servers=['myserver'],enable_auto_commit=True,group_id='30',auto_offset_reset='earliest',consumer_timeout_ms=40000)
    for message in consumer:    
        messageStr=message.value.decode("utf-8") 
        in_q.put(messageStr)
        print(messageStr)
        print(message.offset)
    print("consumer is closed ")
 
# A thread that modify data 
def modifier (out_q): 
        while True:
            if(out_q.empty()==False):
                    data=out_q.get() 
                    print('data after some modification',data)

# Create the shared queue and launch both threads 
message = Queue() 

consumeMessgae = Thread(target = consumer, args =(message, )) 
modifyMessage = Thread(target = modifier , args =(message, )) 
consumeMessgae.start() 
modifyMessage.start() 

I want to update my modifier function to be able:

• change the while loop because it is CPU consuming and instead keep listening to the Queue
• I want to be able to close the modifier function when the consumer thread is closed (consumer function will automatically close after 40s if no new messages) how can I achieve this?

Answer 1

You can achieve that using a threading.Event to notify the modifier thread to abort execution. the final code would be as follows:

import kafka
import time
from queue import Queue, Empty
from threading import Thread, Event


# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q, event):
    consumer = kafka.KafkaConsumer('mytopic', bootstrap_servers=['myserver'], enable_auto_commit=True, group_id='30',
                                   auto_offset_reset='earliest', consumer_timeout_ms=40000)
    try:
        for message in consumer:
            messageStr = message.value.decode("utf-8")
            in_q.put(messageStr)
            print(messageStr)
            print(message.offset)
    except StopIteration:
        # This exception is raised by the kafka.KafkaConsumer iterator, after be waiting 40000 without any new message:
        # Notify the modifier thread, that he should abort execution:
        event.set()
    print("consumer is closed ")


# A thread that modify data
def modifier(out_q, event):
    while True:
        try:
            # Block in the queue only for 1 second waiting for a pending item,
            # to be able to check if the event was signaled, at most after 1 second lapsed:
            data = out_q.get(timeout=1)
        except Empty:
            print("Queue 'out_q' is empty")
        else:
            # Will be executed only if there were no Exception.
            # Add you additional logic here, but take care of handle any possible Exception that could be generated
            # by your data processing logic:
            print('data after some modification', data)

        # Wait for 1 second before next while loop iteration, but it will shot-circuit before 1 second lapsed, if the
        # event is signaled by the consumer thread:
        if event.wait(1):
            # If we're here is because event was signaled by consumer thread, so we must abort execution:
            break
    print("modifier is closed ")



# Create the shared queue and launch both threads
message = Queue()

# Create an Event, it will be used by the consumer thread, to notify the modifier thread that he must abort execution:
alert_event = Event()

consumeMessgae = Thread(target=consumer, args=(message, alert_event,))
modifyMessage = Thread(target=modifier, args=(message, alert_event,))
consumeMessgae.start()
modifyMessage.start()