Celery logs getting written in old log file

Question

import logging
from datetime import date
import time
from datetime import time
from datetime import datetime

from configuration import get_config


class Logger:
    """
    Convenience class to read the service.ini and set the logger.
    """
    _logger = None

    @staticmethod
    def get_file_path():
        """
        Gets the current log path.

        :return: log path
        :rtype: str
        """
        return 'logs//' + str(datetime.now().minute) + '.log'
        #return 'logs//' + date.today().strftime("%b-%d-%Y") + '.log'

    @staticmethod
    def get_log_level():
        """
        Returns the log level for the application

        :return: log level
        :rtype: int
        """
        return {
            'debug': logging.DEBUG,
            'info': logging.INFO,
            'error': logging.ERROR,
            'warning': logging.WARNING,
            'critical': logging.CRITICAL,
        }[get_config('logger', 'level').lower()]

    @classmethod
    def get_logger(cls):
        """
        Get the configured singleton logger. logging is a thread safe class.

        :return: returns logging object
        :rtype: logging
        """
        name = cls.get_file_path()
        if not cls._logger or name != cls._logger.__dict__['name']:
            cls._logger = logging.getLogger(name)
            cls._logger.setLevel(cls.get_log_level())
            fh = logging.FileHandler(name)
            #fh.setLevel(cls.get_log_level())
            formatter = logging.Formatter('%(asctime)s [%(filename)s - %(lineno)s] -%(funcName)s '
                                          '- %(levelname)s - %(message)s')
            fh.setFormatter(formatter)
            cls._logger.addHandler(fh)
        return cls._logger











"""
Provides the logging system for celery task scope.
"""
import logging
from celery.app.log import TaskFormatter
from celery.signals import after_setup_logger

from log import Logger_new


@after_setup_logger.connect
def setup_loggers(logger, *args, **kwargs):
    """
    Sent after the setup of every global logger. Used to augment logging configuration.

    :param logger: logger from the celery system
    :type logger: logging.RootLogger
    :param args: celery arguments
    :type args: tuple
    :param kwargs: celery environment details
    :type kwargs: dict
    :return: no return
    """
    file_handle = logging.FileHandler(Logger_new.get_file_path())
    file_handle.setFormatter(TaskFormatter('%(asctime)s - %(task_id)s - %(task_name)s - %(levelname)s - %(message)s'))
    logger.addHandler(file_handle)
    logger.setLevel(Logger_new.get_log_level())


def get_task_logger():
    """
    This function will return a logger used for logging task details.

    :return: a logger object
    :rtype: logging.Logger
    """
    return logging.getLogger('task_logger')

So, this is my code, the first file has the general logging settings and the second part has the celery task-logger configuration. Each minute a new file should get generated as per the code, everytime I hit the api-s. But that doesn't happen. All the celery task log information gets written in the file which gets generated right after the celery is started. But I want to have the information stored in a new file everytime I hit the API-s, ie every minute a new file should generate containing the recent task details.

Answer 1

As far as I know Celery is removing all the loggers and installs its own. If you want to add your custom logger, you need to do it in the setup_logging signal handler, something like:

import celery.signals

@celery.signals.setup_logging.connect
def on_celery_setup_logging(**kwargs):
    # setup logging here
    pass

I never needed this so I do not know all the necessary details. More about this can be found in the Logging section of the Celery documentation.