[英]How do I remove all Celery results from a redis backend when a user's Flask session ends?
[英]How do you integrate celery into flask + redis?
我最近下載了一個docker 應用程序並開始修改它。 我正在嘗試實現一些線程函數,但它們似乎阻止了服務器的初始化。 我四處詢問,似乎 celery 可能是我最好的選擇,因為我可以設置一個隊列並讓它執行我的功能而不會中斷應用程序本身。
我修改了 bootstrap/Firehouse.py 以運行 celery function 因為它是在服務器初始化后調用的。 所以現在 Firehouse.py 看起來像這樣
import datetime
import json
from flask.json import jsonify
from sqlalchemy import desc
import cache
from shared import app, db
from model.Board import Board
from model.BoardListCatalog import BoardCatalog
from model.Post import render_for_catalog
from model.Thread import Thread
from model.ThreadPosts import _datetime_handler
class Firehose:
def get(self):
return jsonify(self._get_threads())
def get_impl(self):
threads = self._get_threads()
for thread in threads:
board_id = thread["board"]
board = db.session.query(Board).get(board_id)
thread["board"] = board.name
render_for_catalog(threads)
return threads
def _get_threads(self):
firehose_cache_key = "firehose-threads"
cache_connection = cache.Cache()
cached_threads = cache_connection.get(firehose_cache_key)
if cached_threads:
deserialized_threads = json.loads(cached_threads)
for thread in deserialized_threads:
thread["last_updated"] = datetime.datetime.utcfromtimestamp(thread["last_updated"])
return deserialized_threads
firehose_limit = app.config["FIREHOSE_LENGTH"]
raw_threads = db.session.query(Thread).order_by(desc(Thread.last_updated)).limit(firehose_limit).all()
threads = BoardCatalog()._to_json(raw_threads)
for thread in threads:
db_thread = db.session.query(Thread).get(thread["id"])
thread["board"] = db_thread.board
cache_friendly = json.dumps(threads, default=_datetime_handler)
cache_connection.set(firehose_cache_key, cache_friendly)
return threads
from bootstrap import runMonitor
runMonitor.delay()
shared.py 也被修改為指向設置 celery url 所以現在 shared.py 看起來像這樣
import os
import random
import ipaddress
from customjsonencoder import CustomJSONEncoder
from flask import Flask
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from flask_restful import Api
from werkzeug.datastructures import ImmutableDict
from celery import Celery
import jinja_cache
class ManiwaniApp(Flask):
jinja_options = ImmutableDict(extensions=["jinja2.ext.autoescape", "jinja2.ext.with_"],
bytecode_cache=jinja_cache.KeystoreCache())
app = ManiwaniApp(__name__, static_url_path='')
app.config['TEMPLATES_AUTO_RELOAD'] = True
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///test.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["UPLOAD_FOLDER"] = "~/maniwani/uploads"
app.config["THUMB_FOLDER"] = os.path.join(app.config["UPLOAD_FOLDER"], "thumbs")
app.config["SERVE_STATIC"] = True
app.config["SERVE_REST"] = True
app.config["USE_RECAPTCHA"] = False
app.config["FIREHOSE_LENGTH"] = 10
app.config['CELERY_BROKER_URL'] = 'redis://redis:6397/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://redis:6397/0'
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
if os.getenv("MANIWANI_CFG"):
app.config.from_envvar("MANIWANI_CFG")
app.url_map.strict_slashes = False
app.json_encoder = CustomJSONEncoder
db = SQLAlchemy(app)
migrate = Migrate(app, db)
rest_api = Api(app)
SECRET_FILE = "./deploy-configs/secret"
def get_secret():
return open(SECRET_FILE).read()
if os.path.exists(SECRET_FILE):
app.secret_key = get_secret()
def gen_poster_id():
return '%04X' % random.randint(0, 0xffff)
def ip_to_int(ip_str):
# The old version of ip_to_int had a logical bug where it would always shift the
# final result to the left by 8. This is preserved with the `<< 8`.
return int.from_bytes(
ipaddress.ip_address(ip_str).packed,
byteorder="little"
) << 8
最后 bootstrap.py 被修改並包含我的自定義編寫腳本,腳本的代碼並不重要,但我認為相關的是定義的 celery.task ,我將在下面包括:
@celery.task
def runMonitor():
# script that does stuff
但是問題是我無法連接到 redis 服務器。 它在 docker 中定義為具有“redis”主機,這應該允許它指向在服務器初始化時為 docker 生成的任何 ip 地址。
誰能幫我弄清楚:a)最好在哪里調用 runMonitor.delay() b)如何連接到 redis 服務器(當前連接被拒絕,但服務器在線並且端口打開)
我得到的錯誤是:
maniwani_1 | Traceback (most recent call last):
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 559, in connect
maniwani_1 | sock = self._connect()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 615, in _connect
maniwani_1 | raise err
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 603, in _connect
maniwani_1 | sock.connect(socket_address)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/gevent/_socket3.py", line 400, in connect
maniwani_1 | raise error(err, strerror(err))
maniwani_1 | ConnectionRefusedError: [Errno 111] Connection refused
maniwani_1 |
maniwani_1 | During handling of the above exception, another exception occurred:
maniwani_1 |
maniwani_1 | Traceback (most recent call last):
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 127, in reconnect_on_error
maniwani_1 | yield
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1 | self._pubsub.subscribe(key)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 3580, in subscribe
maniwani_1 | ret_val = self.execute_command('SUBSCRIBE', *iterkeys(new_channels))
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 3466, in execute_command
maniwani_1 | self.connection = self.connection_pool.get_connection(
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1 | connection.connect()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1 | raise ConnectionError(self._error_message(e))
maniwani_1 | redis.exceptions.ConnectionError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1 |
maniwani_1 | During handling of the above exception, another exception occurred:
maniwani_1 |
maniwani_1 | Traceback (most recent call last):
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 559, in connect
maniwani_1 | sock = self._connect()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 615, in _connect
maniwani_1 | raise err
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 603, in _connect
maniwani_1 | sock.connect(socket_address)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/gevent/_socket3.py", line 400, in connect
maniwani_1 | raise error(err, strerror(err))
maniwani_1 | ConnectionRefusedError: [Errno 111] Connection refused
maniwani_1 |
maniwani_1 | During handling of the above exception, another exception occurred:
maniwani_1 |
maniwani_1 | Traceback (most recent call last):
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 451, in _reraise_as_library_errors
maniwani_1 | yield
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 779, in send_task
maniwani_1 | self.backend.on_task_call(P, task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 344, in on_task_call
maniwani_1 | self.result_consumer.consume_from(task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 169, in consume_from
maniwani_1 | return self.start(task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 147, in start
maniwani_1 | self._consume_from(initial_task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1 | self._pubsub.subscribe(key)
maniwani_1 | File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1 | self.gen.throw(type, value, traceback)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 130, in reconnect_on_error
maniwani_1 | self._ensure(self._reconnect_pubsub, ())
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 355, in ensure
maniwani_1 | return retry_over_time(
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/kombu/utils/functional.py", line 344, in retry_over_time
maniwani_1 | return fun(*args, **kwargs)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 115, in _reconnect_pubsub
maniwani_1 | metas = self.backend.client.mget(self.subscribed_to)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 1671, in mget
maniwani_1 | return self.execute_command('MGET', *args, **options)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 898, in execute_command
maniwani_1 | conn = self.connection or pool.get_connection(command_name, **options)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1 | connection.connect()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1 | raise ConnectionError(self._error_message(e))
maniwani_1 | redis.exceptions.ConnectionError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1 |
maniwani_1 | During handling of the above exception, another exception occurred:
maniwani_1 |
maniwani_1 | Traceback (most recent call last):
maniwani_1 | File "app.py", line 9, in <module>
maniwani_1 | from blueprints.main import main_blueprint
maniwani_1 | File "./blueprints/main.py", line 9, in <module>
maniwani_1 | from model.Firehose import Firehose
maniwani_1 | File "./model/Firehose.py", line 50, in <module>
maniwani_1 | runMonitor.delay()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/app/task.py", line 425, in delay
maniwani_1 | return self.apply_async(args, kwargs)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/app/task.py", line 565, in apply_async
maniwani_1 | return app.send_task(
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 780, in send_task
maniwani_1 | amqp.send_task_message(P, name, message, **options)
maniwani_1 | File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1 | self.gen.throw(type, value, traceback)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 455, in _reraise_as_library_errors
maniwani_1 | reraise(ConnectionError, ConnectionError(text_t(exc)),
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/vine/five.py", line 194, in reraise
maniwani_1 | raise value.with_traceback(tb)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 451, in _reraise_as_library_errors
maniwani_1 | yield
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 779, in send_task
maniwani_1 | self.backend.on_task_call(P, task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 344, in on_task_call
maniwani_1 | self.result_consumer.consume_from(task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 169, in consume_from
maniwani_1 | return self.start(task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 147, in start
maniwani_1 | self._consume_from(initial_task_id)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1 | self._pubsub.subscribe(key)
maniwani_1 | File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1 | self.gen.throw(type, value, traceback)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 130, in reconnect_on_error
maniwani_1 | self._ensure(self._reconnect_pubsub, ())
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 355, in ensure
maniwani_1 | return retry_over_time(
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/kombu/utils/functional.py", line 344, in retry_over_time
maniwani_1 | return fun(*args, **kwargs)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 115, in _reconnect_pubsub
maniwani_1 | metas = self.backend.client.mget(self.subscribed_to)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 1671, in mget
maniwani_1 | return self.execute_command('MGET', *args, **options)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 898, in execute_command
maniwani_1 | conn = self.connection or pool.get_connection(command_name, **options)
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1 | connection.connect()
maniwani_1 | File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1 | raise ConnectionError(self._error_message(e))
maniwani_1 | kombu.exceptions.OperationalError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1 | unable to load app 0 (mountpoint='') (callable not found or import error)
maniwani_1 | *** no app loaded. going in full dynamic mode ***
maniwani_1 | *** uWSGI is running in multiple interpreter mode ***
redis 啟動並運行:
sudo docker container ls
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d4614c9b1013 nginx "/docker-entrypoint.…" 25 seconds ago Up 16 seconds 0.0.0.0:80->80/tcp maniwani_nginx_1
95503d1e7a28 minio/minio "/usr/bin/docker-ent…" 28 seconds ago Up 23 seconds 9000/tcp maniwani_minio_1
0da8e7a9a573 postgres "docker-entrypoint.s…" 30 seconds ago Up 25 seconds 5432/tcp maniwani_postgres_1
143064a00cb8 maniwani_maniwani "sh ./docker-entrypo…" 30 seconds ago Up 24 seconds 3031/tcp, 5000/tcp maniwani_maniwani_1
5b51b76c4471 redis "docker-entrypoint.s…" 30 seconds ago Up 26 seconds 6379/tcp maniwani_redis_1
54d637f6afe1 maniwani_captchouli "sh ./entrypoint.sh" 30 seconds ago Up 23 seconds 8512/tcp maniwani_captchouli_1
您的 redis 似乎在端口6379/tcp
上運行
您應該更新配置信息以設置好端口
app.config['CELERY_BROKER_URL'] = 'redis://redis:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://redis:6379/0'
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.