Use Loguru logging library to log requests made in an imported file

Question

So I have program written in a python file (main.py) that uses classes in an api wrapper file (bluerev.py). I want to use the loguru logger in main.py to collect all exceptions from the program + all requests made in the api wrapper. So the logging set up in the bluerev.py api wrapper looks like this:

import logging

#Logging setup
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class BluerevApiRequestHandler:

def __init__(self):
    self.date_str_format = "%Y-%m-%dT%H:%M:%S.%f"

@staticmethod
def setup_logger_for_requests():
    """
    Sets up the requests library to log api requests
    """
    logger.setLevel(logging.DEBUG)
    requests_log = logging.getLogger("requests.packages.urllib3")
    requests_log.setLevel(logging.DEBUG)
    requests_log.propagate = True

THe main.py logging code looks like this:

from blurev import *
from loguru import logger
#more imports and code

@logger.catch
def main():
    # associated file and credential locations
    gmail_creds_file = "gmail_creds.json"
    revu_creds_file = r"revu_credentials.json"
    token_file = r"revu_refresh_token.json"
    files_location = os.path.join(os.getcwd(), "rev_dist_files")
    review_files_location = os.path.join(os.getcwd(), "files_to_distribute")

    # Set up Logging
    logging_file = r"plan_review_distributor.log"
    #logging_format_str = "%(levelname)s - %(asctime)-15s - %(filename)s - line %(lineno)d --> %(message)s"
    logger.add(os.path.join(files_location, logging_file),  level="WARNING")
    logger.add(os.path.join(files_location, logging_file), filter="blurev", level="DEBUG")

    #call the requests logging function here in main()
   request_handler = BluerevApiRequestHandler()
   request_handler.setup_logger_for_requests()

   #more code

So I am wondering how this should be changed so that requests being called in the blurev.py file by functions and code in main.py are being logged into the logging_file. Right now they are not.

Answer 1

The problem is that loguru uses a completely different mechanism for logging than the classic logging library. The classic logging library constructs a hierarchy of loggers, and log records are propagate up to the root (see the Advanced logging tutorial from the Python docs for reference). But loguru does not use at all this hierarchy, it operates in a totally disjointed way from it.
So if you want the logs emitted using the classic logging library to be ultimately handled by loguru , you have to intercept them.

Here is a Minimal Reproducible Example of my solution:

# main.py

import logging as classic_logging
import os

from blurev import BluerevApiRequestHandler
from loguru import logger as loguru_logger


@loguru_logger.catch
def main():
    logging_file = r"plan_review_distributor.log"
    loguru_logger.add(os.path.join(".", logging_file), filter=lambda rec: rec["name"] != "blurev", level="WARNING")
    loguru_logger.add(os.path.join(".", logging_file), filter="blurev", level="DEBUG")

    root_logger = classic_logging.getLogger("")
    root_logger.handlers = [InterceptHandler()]  # replace ANY pre-existing handler
    root_logger.setLevel("DEBUG")

    request_handler = BluerevApiRequestHandler()
    request_handler.do_something_using_requests()


########################################################################################################################
# recipe from https://loguru.readthedocs.io/en/stable/overview.html#entirely-compatible-with-standard-logging
class InterceptHandler(classic_logging.Handler):
    def emit(self, record):
        # Get corresponding Loguru level if it exists
        try:
            level = loguru_logger.level(record.levelname).name
        except ValueError:
            level = record.levelno

        # Find caller from where originated the logged message
        frame, depth = classic_logging.currentframe(), 2
        while frame.f_code.co_filename == classic_logging.__file__:
            frame = frame.f_back
            depth += 1

        loguru_logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
########################################################################################################################


if __name__ == "__main__":
    main()

# bluerev.py

import logging as classic_logging

import requests

bluerev_logger = classic_logging.getLogger(__name__)


class BluerevApiRequestHandler:

    def do_something_using_requests(self):
        print(requests.get("http://example.com").text.splitlines()[3])  # expecting: "    <title>Example Domain</title>"
        classic_logging.getLogger("requests.packages.urllib3").warning("warning from requests")  # simulating
        bluerev_logger.debug("debug from bluerev")
        bluerev_logger.critical("critical from bluerev")

The idea is to add an handler to the classic root logger, which will transmit the log records to the loguru logger. We don't need to setup_logger_for_requests , we just let the log record propagate naturally from "requests.packages.urllib3" up the hierarchy.
I added a filter to the first loguru sink so that a "blurev" log record does not get written twice to the file (because caught by the two sinks).

This is the output I get:

# stdout

2022-01-07 11:58:02.480 | DEBUG    | urllib3.connectionpool:_new_conn:227 - Starting new HTTP connection (1): example.com:80
2022-01-07 11:58:02.653 | DEBUG    | urllib3.connectionpool:_make_request:452 - http://example.com:80 "GET / HTTP/1.1" 200 648
2022-01-07 11:58:02.654 | WARNING  | blurev:do_something_using_requests:14 - warning from requests
2022-01-07 11:58:02.654 | DEBUG    | blurev:do_something_using_requests:15 - debug from bluerev
2022-01-07 11:58:02.654 | CRITICAL | blurev:do_something_using_requests:16 - critical from bluerev
    <title>Example Domain</title>

# plan_review_distributor.log

2022-01-07 11:58:02.654 | WARNING  | blurev:do_something_using_requests:14 - warning from requests
2022-01-07 11:58:02.654 | DEBUG    | blurev:do_something_using_requests:15 - debug from bluerev
2022-01-07 11:58:02.654 | CRITICAL | blurev:do_something_using_requests:16 - critical from bluerev

See Loguru's documentation section "Entirely compatible with standard logging" .
Also, you sometimes wrote "bluerev", some other times "bluerev", better watch out !