如何将 celery 集成到 flask + redis 中？

Question

I recently downloaded a docker application and began modifying it. I'm trying to implement some threaded functions but they seem to hold up the initialization of the server. I asked around and it seems that celery might be my best option in that I could set a queue up and have it execute my functions without disrupting the application itself.

I modified bootstrap/Firehouse.py to run the celery function since it is called after the initialization of the server. So now Firehouse.py looks like this

import datetime
import json

from flask.json import jsonify
from sqlalchemy import desc

import cache
from shared import app, db
from model.Board import Board
from model.BoardListCatalog import BoardCatalog
from model.Post import render_for_catalog
from model.Thread import Thread
from model.ThreadPosts import _datetime_handler



class Firehose:
    def get(self):
        return jsonify(self._get_threads())

    def get_impl(self):
        threads = self._get_threads()
        for thread in threads:
            board_id = thread["board"]
            board = db.session.query(Board).get(board_id)
            thread["board"] = board.name
        render_for_catalog(threads)
        return threads

    def _get_threads(self):
        firehose_cache_key = "firehose-threads"
        cache_connection = cache.Cache()
        cached_threads = cache_connection.get(firehose_cache_key)
        if cached_threads:
            deserialized_threads = json.loads(cached_threads)
            for thread in deserialized_threads:
                thread["last_updated"] = datetime.datetime.utcfromtimestamp(thread["last_updated"])
            return deserialized_threads
        firehose_limit = app.config["FIREHOSE_LENGTH"]
        raw_threads = db.session.query(Thread).order_by(desc(Thread.last_updated)).limit(firehose_limit).all()
        threads = BoardCatalog()._to_json(raw_threads)
        for thread in threads:
            db_thread = db.session.query(Thread).get(thread["id"])
            thread["board"] = db_thread.board
        cache_friendly = json.dumps(threads, default=_datetime_handler)
        cache_connection.set(firehose_cache_key, cache_friendly)
        return threads

from bootstrap import runMonitor
runMonitor.delay()

shared.py was also modified to point to set up the celery url So now shared.py looks like this

import os
import random
import ipaddress
from customjsonencoder import CustomJSONEncoder
from flask import Flask
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from flask_restful import Api
from werkzeug.datastructures import ImmutableDict
from celery import Celery
import jinja_cache


class ManiwaniApp(Flask):
    jinja_options = ImmutableDict(extensions=["jinja2.ext.autoescape", "jinja2.ext.with_"],
                                  bytecode_cache=jinja_cache.KeystoreCache())


app = ManiwaniApp(__name__, static_url_path='')
app.config['TEMPLATES_AUTO_RELOAD'] = True
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///test.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config["UPLOAD_FOLDER"] = "~/maniwani/uploads"
app.config["THUMB_FOLDER"] = os.path.join(app.config["UPLOAD_FOLDER"], "thumbs")
app.config["SERVE_STATIC"] = True
app.config["SERVE_REST"] = True
app.config["USE_RECAPTCHA"] = False
app.config["FIREHOSE_LENGTH"] = 10
app.config['CELERY_BROKER_URL'] = 'redis://redis:6397/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://redis:6397/0'

celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)


if os.getenv("MANIWANI_CFG"):
    app.config.from_envvar("MANIWANI_CFG")
app.url_map.strict_slashes = False
app.json_encoder = CustomJSONEncoder
db = SQLAlchemy(app)
migrate = Migrate(app, db)
rest_api = Api(app)

SECRET_FILE = "./deploy-configs/secret"
def get_secret():
    return open(SECRET_FILE).read()

if os.path.exists(SECRET_FILE):
    app.secret_key = get_secret()


def gen_poster_id():
    return '%04X' % random.randint(0, 0xffff)


def ip_to_int(ip_str):
    # The old version of ip_to_int had a logical bug where it would always shift the
    # final result to the left by 8. This is preserved with the `<< 8`.
    return int.from_bytes(
        ipaddress.ip_address(ip_str).packed,
        byteorder="little"
    ) << 8

Lastly bootstrap.py was modified and includes my custom written script, the code for the script is not important but what I think is relevant is the celery.task that was define which I will include below:

@celery.task
def runMonitor():
    # script that does stuff

However the problem is that I cannot connect to the redis server. It is defined in docker as having a host of 'redis' and this should allow it to point to whatever ip address is generated for docker at the initialization of the server.

Can anyone help me figure out: a) where to best call runMonitor.delay() b) how to connect to the redis server (currently the connections are rejected but the server is online and port open)

The error I get is:

maniwani_1    | Traceback (most recent call last):
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 559, in connect
maniwani_1    |     sock = self._connect()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 615, in _connect
maniwani_1    |     raise err
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 603, in _connect
maniwani_1    |     sock.connect(socket_address)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/gevent/_socket3.py", line 400, in connect
maniwani_1    |     raise error(err, strerror(err))
maniwani_1    | ConnectionRefusedError: [Errno 111] Connection refused
maniwani_1    |
maniwani_1    | During handling of the above exception, another exception occurred:
maniwani_1    |
maniwani_1    | Traceback (most recent call last):
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 127, in reconnect_on_error
maniwani_1    |     yield
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1    |     self._pubsub.subscribe(key)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 3580, in subscribe
maniwani_1    |     ret_val = self.execute_command('SUBSCRIBE', *iterkeys(new_channels))
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 3466, in execute_command
maniwani_1    |     self.connection = self.connection_pool.get_connection(
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1    |     connection.connect()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1    |     raise ConnectionError(self._error_message(e))
maniwani_1    | redis.exceptions.ConnectionError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1    |
maniwani_1    | During handling of the above exception, another exception occurred:
maniwani_1    |
maniwani_1    | Traceback (most recent call last):
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 559, in connect
maniwani_1    |     sock = self._connect()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 615, in _connect
maniwani_1    |     raise err
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 603, in _connect
maniwani_1    |     sock.connect(socket_address)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/gevent/_socket3.py", line 400, in connect
maniwani_1    |     raise error(err, strerror(err))
maniwani_1    | ConnectionRefusedError: [Errno 111] Connection refused
maniwani_1    |
maniwani_1    | During handling of the above exception, another exception occurred:
maniwani_1    |
maniwani_1    | Traceback (most recent call last):
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 451, in _reraise_as_library_errors
maniwani_1    |     yield
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 779, in send_task
maniwani_1    |     self.backend.on_task_call(P, task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 344, in on_task_call
maniwani_1    |     self.result_consumer.consume_from(task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 169, in consume_from
maniwani_1    |     return self.start(task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 147, in start
maniwani_1    |     self._consume_from(initial_task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1    |     self._pubsub.subscribe(key)
maniwani_1    |   File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1    |     self.gen.throw(type, value, traceback)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 130, in reconnect_on_error
maniwani_1    |     self._ensure(self._reconnect_pubsub, ())
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 355, in ensure
maniwani_1    |     return retry_over_time(
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/kombu/utils/functional.py", line 344, in retry_over_time
maniwani_1    |     return fun(*args, **kwargs)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 115, in _reconnect_pubsub
maniwani_1    |     metas = self.backend.client.mget(self.subscribed_to)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 1671, in mget
maniwani_1    |     return self.execute_command('MGET', *args, **options)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 898, in execute_command
maniwani_1    |     conn = self.connection or pool.get_connection(command_name, **options)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1    |     connection.connect()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1    |     raise ConnectionError(self._error_message(e))
maniwani_1    | redis.exceptions.ConnectionError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1    |
maniwani_1    | During handling of the above exception, another exception occurred:
maniwani_1    |
maniwani_1    | Traceback (most recent call last):
maniwani_1    |   File "app.py", line 9, in <module>
maniwani_1    |     from blueprints.main import main_blueprint
maniwani_1    |   File "./blueprints/main.py", line 9, in <module>
maniwani_1    |     from model.Firehose import Firehose
maniwani_1    |   File "./model/Firehose.py", line 50, in <module>
maniwani_1    |     runMonitor.delay()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/app/task.py", line 425, in delay
maniwani_1    |     return self.apply_async(args, kwargs)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/app/task.py", line 565, in apply_async
maniwani_1    |     return app.send_task(
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 780, in send_task
maniwani_1    |     amqp.send_task_message(P, name, message, **options)
maniwani_1    |   File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1    |     self.gen.throw(type, value, traceback)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 455, in _reraise_as_library_errors
maniwani_1    |     reraise(ConnectionError, ConnectionError(text_t(exc)),
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/vine/five.py", line 194, in reraise
maniwani_1    |     raise value.with_traceback(tb)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/kombu/connection.py", line 451, in _reraise_as_library_errors
maniwani_1    |     yield
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/app/base.py", line 779, in send_task
maniwani_1    |     self.backend.on_task_call(P, task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 344, in on_task_call
maniwani_1    |     self.result_consumer.consume_from(task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 169, in consume_from
maniwani_1    |     return self.start(task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 147, in start
maniwani_1    |     self._consume_from(initial_task_id)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 177, in _consume_from
maniwani_1    |     self._pubsub.subscribe(key)
maniwani_1    |   File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__
maniwani_1    |     self.gen.throw(type, value, traceback)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 130, in reconnect_on_error
maniwani_1    |     self._ensure(self._reconnect_pubsub, ())
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 355, in ensure
maniwani_1    |     return retry_over_time(
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/kombu/utils/functional.py", line 344, in retry_over_time
maniwani_1    |     return fun(*args, **kwargs)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/celery/backends/redis.py", line 115, in _reconnect_pubsub
maniwani_1    |     metas = self.backend.client.mget(self.subscribed_to)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 1671, in mget
maniwani_1    |     return self.execute_command('MGET', *args, **options)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/client.py", line 898, in execute_command
maniwani_1    |     conn = self.connection or pool.get_connection(command_name, **options)
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 1192, in get_connection
maniwani_1    |     connection.connect()
maniwani_1    |   File "/usr/local/lib/python3.8/dist-packages/redis/connection.py", line 563, in connect
maniwani_1    |     raise ConnectionError(self._error_message(e))
maniwani_1    | kombu.exceptions.OperationalError: Error 111 connecting to redis:6397. Connection refused.
maniwani_1    | unable to load app 0 (mountpoint='') (callable not found or import error)
maniwani_1    | *** no app loaded. going in full dynamic mode ***
maniwani_1    | *** uWSGI is running in multiple interpreter mode ***

redis is up and running:

sudo docker container ls
CONTAINER ID        IMAGE                 COMMAND                  CREATED             STATUS              PORTS                NAMES
d4614c9b1013        nginx                 "/docker-entrypoint.…"   25 seconds ago      Up 16 seconds       0.0.0.0:80->80/tcp   maniwani_nginx_1
95503d1e7a28        minio/minio           "/usr/bin/docker-ent…"   28 seconds ago      Up 23 seconds       9000/tcp             maniwani_minio_1
0da8e7a9a573        postgres              "docker-entrypoint.s…"   30 seconds ago      Up 25 seconds       5432/tcp             maniwani_postgres_1
143064a00cb8        maniwani_maniwani     "sh ./docker-entrypo…"   30 seconds ago      Up 24 seconds       3031/tcp, 5000/tcp   maniwani_maniwani_1
5b51b76c4471        redis                 "docker-entrypoint.s…"   30 seconds ago      Up 26 seconds       6379/tcp             maniwani_redis_1
54d637f6afe1        maniwani_captchouli   "sh ./entrypoint.sh"     30 seconds ago      Up 23 seconds       8512/tcp             maniwani_captchouli_1

Answer 1

Your redis seems to run on port 6379/tcp

You should update your config informations to set the good port

app.config['CELERY_BROKER_URL'] = 'redis://redis:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://redis:6379/0'