多处理 + 日志记录 + tqdm 进度条闪烁

Question

有tqdm经验的人可以帮我使用以下复制器吗？

import itertools
import logging
import multiprocessing
import random
import sys
import time

import tqdm
from tqdm.contrib import DummyTqdmFile

log: logging.Logger = logging.getLogger(__name__)


DEFAULT_FORMAT = "[%(asctime)s.%(msecs)06d][%(processName)s][%(threadName)s][%(levelname)s][%(module)s] %(message)s"
VERBOSITY = 1


def configure_logger(log: logging.Logger, verbosity: int, *, format: str = DEFAULT_FORMAT, dry_run: bool = False):
    """Configures the logger instance based on verbosity level"""

    # can't use force=true as it requires Python >= 3.8
    root = logging.getLogger()
    for handler in root.handlers:
        root.removeHandler(handler)

    if dry_run:
        format = format.replace(" %(message)s", "[DRY RUN] %(message)s")
    logging.basicConfig(format=format, datefmt="%Y-%m-%d %H:%M:%S", stream=DummyTqdmFile(sys.stdout))

    verbosity_to_loglevel = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG}
    log.setLevel(verbosity_to_loglevel[min(verbosity, 2)])

    if verbosity >= 4:
        logging.getLogger().setLevel(logging.INFO)
    if verbosity >= 5:
        logging.getLogger().setLevel(logging.DEBUG)

def by_n(iterable, n):
    """Iterate by chunks of n items"""
    return (tuple(filter(lambda x: x is not None, x)) for x in itertools.zip_longest(*[iter(iterable)] * n))


def _worker(batch):
    log.info("Let's go!")
    for i in batch:
        log.info("Processing item %d", i)
        time.sleep(random.uniform(0.1, 0.2))
    log.info("Done!")

    return len(batch)

if __name__ == "__main__":
    configure_logger(log, VERBOSITY)

    log.info("Let's go!")

    foos = list(range(1000))
    kwargs = {
        "desc": "Processing...",
        "total": len(foos),
        "leave": True,
        "mininterval": 1,
        "maxinterval": 5,
        "unit": "foo",
        "dynamic_ncols": True,
    }
    pbar = tqdm.tqdm(**kwargs)

    with multiprocessing.Pool(processes=5) as workers_pool:
        for batch_length in workers_pool.imap_unordered(
            _worker,
            by_n(foos, min(10 + int((len(foos) - 10) / 100), 1000)),
        ):
            pbar.update(batch_length)

    log.info("Done!")

当我设置VERBOSITY = 0时，来自工作进程的log调用不输出任何内容，并且进度条显示正确。

但是，一旦我通过设置VERBOSITY = 1从工作进程登录，那么这就是发生的事情

• 当进度条正确定位并粘在屏幕底部时
• 大多数时候显示进度为0% ，只是偶尔会以正确的进度和百分比闪烁

Answer 1

也许这会部分帮助。 这涉及多处理的进展。 在做了一些研究之后，我写了一个名为parallelbar的小模块。 它允许您分别显示池的整体进度和每个核心的进度。 它易于使用并且有很好的描述。

例如：

from parallelbar import progress_imapu
from parallelbar.tools import cpu_bench


if __name__=='__main__':
    # create list of task
    tasks = [1_000_000 + i for i in range(100)]
    progress_imapu(cpu_bench, tasks, core_progress=True)