[英]RabbitMQ Python PIKA - using add_callback_threadsafe lost acknowledgement when stopping program
我正在使用 pika 小批量處理 RabitMQ 消息，並為每個批次使用一個線程。 在線程中的 function 結束時，我通過add_callback_threadsafe向通道發送消息確認。
在 parallele 中，我正在捕獲 SIGINT 信號以正確停止程序，方法是等待thread.join()所有線程在停止通道消耗和關閉連接之前完成。
但是一旦發送 CtrlC 以生成 SIGINT，事件如果程序等待所有線程完成，將不會處理確認。
==> 有沒有辦法強制通道/連接在關閉連接之前處理等待的add_callback_threadsafe ？
# import packages
# connect to Rabbit MQ
import pika
# intercept stop signal
import signal
# print exception
import traceback
# threading
import functools
import threading
from queue import Queue
# logs time
import datetime
import time
# Function Message Acknowledgement
def ack_message(ch, delivery_tag):
"""Note that `ch` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
print(f'DEBUG ack_message : begining of ack_message function')
if ch.is_open:
ch.basic_ack(delivery_tag)
print(f'DEBUG ack_message : Acknowledgement delivered')
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'ERROR Channel Closed when trying to Acknowledge')
pass
# Function Process multiple messages in separate thread
def block_process():
# list global variables to be changed
global channel
# init local variables
body_list = list()
tag_list = list()
print(f'DEBUG block_process : start of block_process function')
# cancel the timer if exist, as we will proces all elements in the queue here
if event and event.isAlive():
event.cancel()
# extract all queued messages fom internal python queue and rebuild individual listes body and tag from tupple
for i in range(list_Boby_Tag.qsize()):
myTuppleBodyTag = list_Boby_Tag.get()
body_list += [myTuppleBodyTag[0]]
tag_list += [myTuppleBodyTag[1]]
# that also empty the queue
# do something that take time with the block of nessage in body_list
time.sleep(10)
for body in body_list:
body_str = body.decode()
print(f'DEBUG block_process : message processed is {body_str}')
# acknowledging all tags in tag_list by using the channel thread safe function .connection.add_callback_threadsafe
for tag in tag_list:
print(f'DEBUG preprare delivering Acknowledgement from thread')
cb = functools.partial(ack_message, channel, tag)
channel.connection.add_callback_threadsafe(cb)
print(f'DEBUG block_process : end of block_process function')
return
# Function Process message by message and call
def process_message(ch, method, properties, body):
# list global variables to be changed
global list_Boby_Tag
global event
global threads
# do nothing if this flag is on, as the program is about to close
if PauseConsume == 1:
return
# cancel the timer if exist as we are going to process a block or restart a new timer
if event and event.isAlive():
event.cancel()
# put in the queue the data from the body and tag as tupple
list_Boby_Tag.put((body,method.delivery_tag))
# if a max queue size is reached (here 500), immmediately launch a new thread to process the queue
if list_Boby_Tag.qsize() == 500 :
#print(f'DEBUG thread count before {len(threads)}')
# keep in the threads list only the thread still running
threads = [x for x in threads if x.is_alive()]
#print(f'DEBUG thread count after {len(threads)}')
# start the inference in a separated thread
t = threading.Thread(target=block_process)
t.start()
# keep trace of the thread so it can be waited at the end if still running
threads.append(t)
#print(f'DEBUG thread count after add {len(threads)}')
elif list_Boby_Tag.qsize() > 0 :
# if the queue is not full create a thread with a timer to do the process after sometime, here 10 seconds for test purpose
event = threading.Timer(interval=10, function=block_process)
event.start()
# also add this thread to the list of threads
threads.append(event)
# PARAMETERS
RabbitMQ_host = '192.168.1.190'
RabbitMQ_port = 5672
RabbitMQ_queue = 'test_ctrlC'
RabbitMQ_cred_un = 'xxxx'
RabbitMQ_cred_pd = 'xxxx'
# init variables for batch process
list_Boby_Tag = Queue()
threads = list()
event = None
PauseConsume = 0
init_time = time.time()
# connect to RabbitMQ via Pika
cred = pika.credentials.PlainCredentials(RabbitMQ_cred_un,RabbitMQ_cred_pd)
connection = pika.BlockingConnection(pika.ConnectionParameters(host=RabbitMQ_host, port=RabbitMQ_port, credentials=cred))
channel = connection.channel()
channel.queue_declare(queue=RabbitMQ_queue,durable=True)
# tell rabbitMQ to don't dispatch a new message to a worker until it has processed and acknowledged the previous one :
channel.basic_qos(prefetch_count=1)
# define the comsumer
channel.basic_consume(queue=RabbitMQ_queue,
auto_ack=False, # false = need message acknowledgement : basic_ack in the callback
on_message_callback=process_message)
# empty queue and generate test data
channel.queue_purge(queue=RabbitMQ_queue)
# wait few second so the purge can be check in the RabbitMQ ui
print(f'DEBUG main : queue {RabbitMQ_queue} purged')
connection.sleep(10)
# generate 10 test messages
for msgId in range(10):
channel.basic_publish(exchange='',
routing_key=RabbitMQ_queue,
body=f'message{msgId}',
properties=pika.BasicProperties(
delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE
))
print(f'DEBUG main : test messages created in {RabbitMQ_queue}')
# Function clean stop of pika connection in case of interruption or exception
def cleanClose():
# tell the on_message_callback to do nothing
PauseConsume = 1
# Wait for all threads to complete
for thread in threads:
thread.join()
# stop pika connection after a short pause
connection.sleep(3)
channel.stop_consuming()
connection.close()
return
# Function handle exit signals
def exit_handler(signum, frame):
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'Exit signal received ({signum})')
cleanClose()
exit(0)
signal.signal(signal.SIGINT, exit_handler) # send by a CTRL+C or modified Docker Stop
#signal.signal(signal.SIGTSTP, exit_handler) # send by a CTRL+Z Docker Stop
print(' [*] Waiting for messages. To exit press CTRL+C')
try:
channel.start_consuming()
except Exception:
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'Exception received within start_consumming')
traceback.print_exc()
cleanClose()
Luke 在這里找到了解決方法： https://github.com/lukebakken/pika-1402/blob/lukebakken/pika-1402/test_pika_blockthread.py
他更改了示例代碼：
通過隊列添加批處理的示例代碼，與原始代碼匹配在這里：
# from https://github.com/lukebakken/pika-1402/blob/lukebakken/pika-1402/test_pika_blockthread.py
# import packages
# connect to Rabbit MQ
import pika
import pika.credentials
import pika.spec
# intercept stop signal
import signal
# print exception
# import traceback
# threading
import functools
import threading
from queue import Queue
# logs time
import datetime
import time
# PARAMETERS
RabbitMQ_host = "192.168.1.190"
RabbitMQ_port = 5672
RabbitMQ_queue = "test_ctrlC"
RabbitMQ_cred_un = "xxxx"
RabbitMQ_cred_pd = "xxxx"
nbTest = 100000
nbBatch = 1000
nbPrefetch = 10000
# note, prefecth always >= nbBatch
timerSec = 60 # timer wait
workSec = 5 # nbr of sec for simulating batch work
# init variables for batch process
init_time = time.time()
exiting = False
work_threads = list()
event = None
list_Boby_Tag = Queue()
# Function Message Acknowledgement
def ack_message(ch, delivery_tag):
"""Note that `ch` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if ch.is_open:
ch.basic_ack(delivery_tag)
#print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG ack_message : begining of ack_message function, tag: {delivery_tag}")
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),"Channel Closed when trying to Acknowledge")
pass
return
# Function Process multiple messages in separate thread
def do_work(channel, list_Boby_Tag):
# init local variables
body_list = list()
tag_list = list()
# cancel the timer if exist, as we will proces all elements in the queue here
if event and event.is_alive():
event.cancel()
# extract all queued messages fom internal python queue and rebuild individual listes body and tag from tupple
for i in range(list_Boby_Tag.qsize()):
myTuppleBodyTag = list_Boby_Tag.get()
body_list += [myTuppleBodyTag[0]]
tag_list += [myTuppleBodyTag[1]]
# that also empty the queue
# do something that take time with the block of nessage in body_list
time.sleep(workSec)
for body in body_list:
body_str = body.decode()
#print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'DEBUG block_process : message processed is {body_str}')
# acknowledging all tags in tag_list by using the channel thread safe function .connection.add_callback_threadsafe
for tag in tag_list:
cb = functools.partial(ack_message, channel, tag)
channel.connection.add_callback_threadsafe(cb)
return
# Function Process message by message and call thread by block or timer
def process_message(ch, method, body):
global work_threads
global list_Boby_Tag
global event
# cancel the timer if exist as we are going to process a block or restart a new timer
if event and event.is_alive():
event.cancel()
# put in the queue the data from the body and tag as tupple
list_Boby_Tag.put((body,method.delivery_tag))
# if a max queue size is reached (here 500), immmediately launch a new thread to process the queue
if list_Boby_Tag.qsize() == nbBatch :
#print(f'DEBUG thread count before {len(threads)}')
# keep in the threads list only the thread still running
work_threads = [x for x in work_threads if x.is_alive()]
#print(f'DEBUG thread count after {len(threads)}')
# start the inference in a separated thread
t = threading.Thread(target=do_work, args=(ch, list_Boby_Tag))
t.start()
# keep trace of the thread so it can be waited at the end if still running
work_threads.append(t)
#print(f'DEBUG thread count after add {len(threads)}')
elif list_Boby_Tag.qsize() > 0 :
# if the queue is not full create a thread with a timer to do the process after sometime, here 10 seconds for test purpose
event = threading.Timer(interval=timerSec, function=do_work, args=(ch, list_Boby_Tag))
event.start()
# also add this thread to the list of threads
work_threads.append(event)
return
# Function to start pika channel and stopping it
def pika_runner():
# connect to RabbitMQ via Pika
cred = pika.credentials.PlainCredentials(RabbitMQ_cred_un, RabbitMQ_cred_pd)
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=RabbitMQ_host, port=RabbitMQ_port, credentials=cred
)
)
channel = connection.channel()
channel.queue_declare(queue=RabbitMQ_queue, durable=True)
# tell rabbitMQ to don't dispatch a new message to a worker until it has processed and acknowledged the previous one :
channel.basic_qos(prefetch_count=nbPrefetch)
# empty queue and generate test data
channel.queue_purge(queue=RabbitMQ_queue)
# wait few second so the purge can be check in the RabbitMQ ui
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG main : queue {RabbitMQ_queue} purged, sleeping 5 seconds")
connection.sleep(5)
# generate test messages
for msgId in range(nbTest):
channel.basic_publish(
exchange="",
routing_key=RabbitMQ_queue,
body=f"message-{msgId+1}",
properties=pika.BasicProperties(
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
),
)
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG main : test messages created in {RabbitMQ_queue}")
# loop forever to retrieve messahes
for method_frame, properties, body in channel.consume(
queue=RabbitMQ_queue, inactivity_timeout=1, auto_ack=False
):
if exiting:
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : stopping consuming")
#channel.stop_consuming()
channel.cancel()
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : joining work threads")
for thread in work_threads:
thread.join()
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : all work threads done, sleeping 5 seconds to let acks be delivered")
connection.sleep(5)
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : closing connections and channels")
channel.close()
connection.close()
else:
if method_frame is not None:
process_message(channel, method_frame, body)
return
# Function handle exit signals
def exit_handler(signum, _):
global exiting
if exiting:
return
exiting = True
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),"Exit signal received")
pika_thread.join()
exit(0)
# launch the thread that will connect and listen to Pika
pika_thread = threading.Thread(target=pika_runner)
pika_thread.start()
# catch interuption signal to exit gracefully
signal.signal(signal.SIGINT, exit_handler) # send by a CTRL+C or modified Docker Stop
print(" [*] Waiting for messages. To exit press CTRL+C")
# wait for all threads to finish
for thread in work_threads:
thread.join()
pika_thread.join()
如何在 python 中使用 pika (RabbitMQ) 向消費者添加多處理
[英]How to add multiprocessing to consumer with pika (RabbitMQ) in python
使用pika的python中的SparkStreaming，RabbitMQ和MQTT
[英]SparkStreaming, RabbitMQ and MQTT in python using pika
Python - RabbitMQ Pika 消費者 - 如何使用異步函數作為回調
[英]Python - RabbitMQ Pika consumer - How to use async function as callback
使用pika python模塊時使用SSL連接到遠程rabbitmq實例的錯誤
[英]Connection errors to remote rabbitmq instance using SSL when using pika python module
當我使用pika（python）向RabbitMQ嘗試ack消息時出現錯誤“未知傳遞標記”
[英]Error “unknown delivery tag” occurs when i try ack messages to RabbitMQ using pika (python)
如何在 Python 中使用 RabbitMQ pika.basic_consume 更改超時
[英]How to change timeout using RabbitMQ pika.basic_consume in Python
使用 Python 鼠兔庫合並 RabbitMQ 隊列的最佳方法是什么？
[英]What is the best way to merge RabbitMQ queues using Python pika library?
聲明:本站的技術帖子網頁，遵循CC BY-SA 4.0協議，如果您需要轉載，請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.