Python logging with dictConfig using two streamhandlers posting to stdout and stderr at different message levels

Question

I'm trying to achieve what's done in these posts Python logging split between stdout and stderr

Python logging split between stdout and stderr

but with dictConfig, so far without success. Here is my code and config:

This is the config I use for generating stdout log

# logging.json
{
    "version": 1,
    "disable_existing_loggers": false,
    "formatters": {
        "console": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: \n%(message)s\n"
        },
        "file": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: %(lineno)d: \n%(message)s\n"
        }
    },
    "handlers": {
            "console": {
                "level": "INFO",
                "formatter": "console",
                "class": "logging.StreamHandler",
                "stream": "ext://sys.stdout"
            },
            "file": {
                "level": "DEBUG",
                "formatter": "file",
                "class": "logging.FileHandler",
                "encoding": "utf-8",
                "filename": "app.log"
            }
    },
    "loggers": {
        "": {
            "handlers": ["console", "file"],
            "level": "INFO",
            "propagate": false
        },
        "default": {
            "handlers": ["console", "file"],
            "level": "DEBUG",
            "propagate": false
        }

    }
}

And this one is for stderr logs

# logging_stderr.json

{
    "version": 1,
    "disable_existing_loggers": false,
    "formatters": {
        "console": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: \n%(message)s\n"
        },
        "file": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: %(lineno)d: \n%(message)s\n"
        }
    },
    "handlers": {
            "console": {
                "level": "WARN",
                "formatter": "console",
                "class": "logging.StreamHandler",
                "stream": "ext://sys.stderr"
            },
            "file": {
                "level": "DEBUG",
                "formatter": "file",
                "class": "logging.FileHandler",
                "encoding": "utf-8",
                "filename": "wusync.log"
            }
    },
    "loggers": {
        "": {
            "handlers": ["console", "file"],
            "level": "INFO",
            "propagate": false
        },
        "default": {
            "handlers": ["console", "file"],
            "level": "DEBUG",
            "propagate": false
        }

    }
}

Then in my code, I have a helper function.

import logging
import logging.config
import os
from os.path import abspath, basename, dirname, exists, isfile, isdir, join, split, splitext
import sys

_script_dir = abspath(dirname(__file__))


def build_default_logger(logdir, name=None, cfgfile=None):
    """
    Create per-file logger and output to shared log file.
    - If found config file under script folder, use it;
    - Otherwise use default config: save to /project_root/project_name.log.
    - 'filename' in config is a filename; must prepend folder path to it.
    :logdir: directory the log file is saved into.
    :name: basename of the log file,
    :cfgfile: config file in the format of dictConfig.
    :return: logger object.
    """
    try:
        os.makedirs(logdir)
    except:
        pass

    cfg_file = cfgfile or join(_script_dir, 'logging.json')
    logging_config = None
    try:
        if sys.version_info.major > 2:
            with open(cfg_file, 'r', encoding=TXT_CODEC, errors='backslashreplace', newline=None) as f:
                text = f.read()
        else:
            with open(cfg_file, 'rU') as f:
                text = f.read()
        # Add object_pairs_hook=collections.OrderedDict hook for py3.5 and lower.
        logging_config = json.loads(text, object_pairs_hook=collections.OrderedDict)
        logging_config['handlers']['file']['filename'] = join(logdir, logging_config['handlers']['file']['filename'])
    except Exception:
        filename = name or basename(basename(logdir.strip('\\/')))
        log_path = join(logdir, '{}.log'.format(filename))
        logging_config = {
            "version": 1,
            "disable_existing_loggers": False,
            "formatters": {
                "console": {
                    "format": "%(asctime)s: %(levelname)s: %(pathname)s: \n%(message)s\n"
                },
                "file": {
                    "format": "%(asctime)s: %(levelname)s: %(pathname)s: %(lineno)d: \n%(message)s\n"
                }
            },
            "handlers": {
                    "console": {
                        "level": "INFO",
                        "formatter": "console",
                        "class": "logging.StreamHandler",
                        "stream": "ext://sys.stdout"
                    },
                    "file": {
                        "level": "DEBUG",
                        "formatter": "file",
                        "class": "logging.FileHandler",
                        "encoding": "utf-8",
                        "filename": log_path
                    }
            },
            "loggers": {
                "": {
                    "handlers": ["console", "file"],
                    "level": "INFO",
                    "propagate": True
                },
                "default": {
                    "handlers": ["console", "file"],
                    "level": "WARN",
                    "propagate": True
                }
            }
        }
    if name:
        logging_config['loggers'][name] = logging_config['loggers']['default']
    logging.config.dictConfig(logging_config)
    return logging.getLogger(name or 'default')

Finally in my main routine

# main.py

_script_dir = abspath(dirname(__file__))
_logger = util.build_default_logger(logdir='temp', cfgfile=abspath(join(_script_dir, 'logging_stderr.json')))
_stderr_logger = util.build_default_logger(logdir='temp', name='myerrorlog', cfgfile=abspath(join(_script_dir, 'logging_stderr.json')))


...

_logger.info('my info')
_stderr_logger.warning('my warning')

I expect that the info will show through stdout, and warning through stderr. But in the result, there are only warnings and the info is completely gone.

If using only _logger , then everything goes through stdout.

Where was I wrong? Is it that dictconfig only supports one streamhandler?

Answer 1

I solved my own problem by using filters.

This post helped me:

install filter on logging level in python using dictConfig

I basically want to send INFO to stdout, and WARNING-to-CRITICAL to stderr. This implies having a scope with both ends defined for handlers. The level attribute of handlers defines the low end only.

Now filter to the rescue. I ended up using this config:

{
    "version": 1,
    "disable_existing_loggers": false,
    "filters": {
        "infofilter": {
          "()": "util.LowPassFilter",
          "level": 20
        },
        "warnfilter": {
          "()": "util.HighPassFilter",
          "level": 30
        }
    },
    "formatters": {
        "console": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: \n%(message)s\n"
        },
        "file": {
            "format": "%(asctime)s: %(levelname)s: %(pathname)s: %(lineno)d: \n%(message)s\n"
        }
    },
    "handlers": {
            "console": {
                "level": "INFO",
                "formatter": "console",
                "class": "logging.StreamHandler",
                "stream": "ext://sys.stdout",
                "filters": ["infofilter"]
            },
            "console_err": {
                "level": "WARN",
                "formatter": "console",
                "class": "logging.StreamHandler",
                "stream": "ext://sys.stderr",
                "filters": ["warnfilter"]
            },
            "file": {
                "level": "DEBUG",
                "formatter": "file",
                "class": "logging.FileHandler",
                "encoding": "utf-8",
                "filename": "app.log"
            }
    },
    "loggers": {
        "": {
            "handlers": ["console", "console_err", "file"],
            "level": "INFO",
            "propagate": false
        },
        "default": {
            "handlers": ["console", "console_err", "file"],
            "level": "DEBUG",
            "propagate": true
        }
    }
}

and the filters

class LowPassFilter(object):
    """
    Logging filter: Show log messages below input level.
    - CRITICAL = 50
    - FATAL = CRITICAL
    - ERROR = 40
    - WARNING = 30
    - WARN = WARNING
    - INFO = 20
    - DEBUG = 10
    - NOTSET = 0
    """
    def __init__(self, level):
        self.__level = level

    def filter(self, log):
        return log.levelno <= self.__level


class HighPassFilter(object):
    """Logging filter: Show log messages above input level."""
    def __init__(self, level):
        self.__level = level

    def filter(self, log):
        return log.levelno >= self.__level