python 线程和它们之间的队列消息

Question

我有一个包含两个部分的程序，它们使用线程同时工作，它们使用队列进行通信。

import kafka
import time
from queue import Queue 
from threading import Thread 

# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q): 
    consumer = kafka.KafkaConsumer('mytopic',bootstrap_servers=['myserver'],enable_auto_commit=True,group_id='30',auto_offset_reset='earliest',consumer_timeout_ms=40000)
    for message in consumer:    
        messageStr=message.value.decode("utf-8") 
        in_q.put(messageStr)
        print(messageStr)
        print(message.offset)
    print("consumer is closed ")
 
# A thread that modify data 
def modifier (out_q): 
        while True:
            if(out_q.empty()==False):
                    data=out_q.get() 
                    print('data after some modification',data)

# Create the shared queue and launch both threads 
message = Queue() 

consumeMessgae = Thread(target = consumer, args =(message, )) 
modifyMessage = Thread(target = modifier , args =(message, )) 
consumeMessgae.start() 
modifyMessage.start() 

我想更新我的修饰符 function 以便能够：

• 更改 while 循环，因为它消耗 CPU，而是继续侦听队列
• 我希望能够在消费者线程关闭时关闭修饰符 function（如果没有新消息，消费者 function 将在 40 秒后自动关闭）我该如何实现？

Answer 1

您可以使用 threading.Event 来通知修改器线程中止执行来实现这一点。 最终代码如下：

import kafka
import time
from queue import Queue, Empty
from threading import Thread, Event


# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q, event):
    consumer = kafka.KafkaConsumer('mytopic', bootstrap_servers=['myserver'], enable_auto_commit=True, group_id='30',
                                   auto_offset_reset='earliest', consumer_timeout_ms=40000)
    try:
        for message in consumer:
            messageStr = message.value.decode("utf-8")
            in_q.put(messageStr)
            print(messageStr)
            print(message.offset)
    except StopIteration:
        # This exception is raised by the kafka.KafkaConsumer iterator, after be waiting 40000 without any new message:
        # Notify the modifier thread, that he should abort execution:
        event.set()
    print("consumer is closed ")


# A thread that modify data
def modifier(out_q, event):
    while True:
        try:
            # Block in the queue only for 1 second waiting for a pending item,
            # to be able to check if the event was signaled, at most after 1 second lapsed:
            data = out_q.get(timeout=1)
        except Empty:
            print("Queue 'out_q' is empty")
        else:
            # Will be executed only if there were no Exception.
            # Add you additional logic here, but take care of handle any possible Exception that could be generated
            # by your data processing logic:
            print('data after some modification', data)

        # Wait for 1 second before next while loop iteration, but it will shot-circuit before 1 second lapsed, if the
        # event is signaled by the consumer thread:
        if event.wait(1):
            # If we're here is because event was signaled by consumer thread, so we must abort execution:
            break
    print("modifier is closed ")



# Create the shared queue and launch both threads
message = Queue()

# Create an Event, it will be used by the consumer thread, to notify the modifier thread that he must abort execution:
alert_event = Event()

consumeMessgae = Thread(target=consumer, args=(message, alert_event,))
modifyMessage = Thread(target=modifier, args=(message, alert_event,))
consumeMessgae.start()
modifyMessage.start()