python asyncio從3.4遷移到3.5+

Question

大家晚上好，我試圖創建互聯網機器人，但在將腳本從python 3.4遷移到3.5或3.6+時遇到了問題。 它使用asyncio並在3.4 python上運行良好，但是當我使用python3.5 +啟動它時出現錯誤： RuntimeError: Cannot run the event loop while another loop is running

這是代碼方案：

import multiprocessing as mp
import asyncio
import concurrent.futures
import aiohttp

def create_proccesses(separate_loop_creator, coro):
    proccesses = []
    for n in range(2):
        proc = mp.Process(target=separate_loop_creator, args=(coro,))
        proc.start()
        proccesses.append(proc)
    for p in proccesses:
        p.join()

def separate_loop_creator(coro):
    sep_loop = asyncio.new_event_loop()
    asyncio.set_event_loop(sep_loop)
    tasks = [asyncio.async(coro(sep_loop)) for _ in range(100)]
    try:
        sep_loop.run_until_complete(asyncio.wait(tasks))
        sep_loop.close()
    except Exception as err:
        print(err)
        for task in tasks:
            task.cancel()
        sep_loop.close()


@asyncio.coroutine
def manager(exe, loop):
    # some calculations and start coros in several processes
    loop.run_in_executor(
        exe,
        create_proccesses,
        separate_loop_creator,
        some_coro
    )

@asyncio.coroutine
def some_work_in_mainloop():
    while True:
        print('Some server dealing with connections here...')
        yield from asyncio.sleep(1)

@asyncio.coroutine
def some_coro(loop):
    with aiohttp.ClientSession(loop=loop) as session:
        response = yield from session.get('http://google.com')
        yield from asyncio.sleep(2)
        print(response.status)

if __name__ == '__main__':
    mainloop = asyncio.get_event_loop()
    executor = concurrent.futures.ProcessPoolExecutor(5)
    asyncio.async(some_work_in_mainloop())
    asyncio.async(manager(executor, mainloop))
    try:
        mainloop.run_forever()
    finally:
        mainloop.close()

異常在eparate_loop_creator separate_loop_creator()協程中引發，它是RuntimeError: Cannot run the event loop while another loop is running 。 我認為這是因為更改了get_event_loop() ，但我不明白我的代碼有什么問題。

這是我想做的：

                       +--------------+
               +-------+other service |
    +----------+       +--------------+
    | mainloop |
    +----------+
               |     +------------+
               +-----+   executor |
                     +------+-----+
                            |
                     +------+--------+
                     |start proccess |
                     +---+-------+---+
+-----------------+      |       |      +---------------+
|start new loop   +------+       +------+ start new loop|
+--------+--------+                     +-------+-------+
         |                                      |
 +-------+-------+                       +------v-------+
 |   run coro    |                       | run coro     |
 +---------------+                       +--------------+

這是我在python3.5.3上獲得的跟蹤：

Traceback (most recent call last):
  File "tst.py", line 21, in separate_loop_creator
    sep_loop.run_until_complete(asyncio.wait(tasks))
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete
    self.run_forever()
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever
    'Cannot run the event loop while another loop is running')
RuntimeError: Cannot run the event loop while another loop is running
Cannot run the event loop while another loop is running
Traceback (most recent call last):
  File "tst.py", line 21, in separate_loop_creator
    sep_loop.run_until_complete(asyncio.wait(tasks))
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete
    self.run_forever()
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever
    'Cannot run the event loop while another loop is running')
RuntimeError: Cannot run the event loop while another loop is running

Python 3.4.3結果：

...
200
Some server dealing with connections here...
200
200
Some server dealing with connections here...
200
200
Some server dealing with connections here...
200
...

Answer 1

這實際上是CPython 3.6.0中asyncio中的錯誤。 有一個PR可以解決此問題，以便3.6.1可以正常工作。

解決方法是，可以在項目中添加以下代碼：

import sys

if sys.version_info[:3] == (3, 6, 0):
    import asyncio.events as _ae
    import os as _os

    _ae._RunningLoop._pid = None

    def _get_running_loop():
        if _ae._running_loop._pid == _os.getpid():
            return _ae._running_loop._loop

    def _set_running_loop(loop):
        _ae._running_loop._pid = _os.getpid()
        _ae._running_loop._loop = loop

    _ae._get_running_loop = _get_running_loop
    _ae._set_running_loop = _set_running_loop

Answer 2

最好的解決方案，如果可能，請嘗試從multiprocessing完全刪除multiprocessing ，並僅使用一個事件循環（對於獨立的CPU密集型任務，可以選擇使用ProcessPoolExecutor ）。

截至2017年3月2日，此問題有一個打開的python錯誤，影響到非Windows平台： https : //bugs.python.org/issue22087 。

這是觸發相同問題的較短程序：

import asyncio
import multiprocessing as mp


def create_another_loop():
    loop = asyncio.new_event_loop()
    loop.run_forever()


async def create_process():
    proc = mp.Process(target=create_another_loop)
    proc.start()
    proc.join()


if __name__ == '__main__':
    main_loop = asyncio.get_event_loop()
    main_loop.run_until_complete(create_process())
    main_loop.close()

受https://github.com/python/asyncio/pull/497此處建議的修復啟發，一個變黑的解決方法（警告！使用后果自負！）是將此代碼添加到新創建的Process ：

if asyncio.events._running_loop:
    asyncio.events._running_loop._loop = None

例：

import asyncio
import multiprocessing as mp
import time
from concurrent.futures.process import ProcessPoolExecutor


async def clock(label, n=5, sleep=1):
    print(label, "start")
    for i in range(n):
        await asyncio.sleep(sleep)
        print(label, i + 1)
    print(label, "end")
    return label


def create_another_loop():
    # HACK START
    if asyncio.events._running_loop:
        asyncio.events._running_loop._loop = None
    # HACK END

    loop = asyncio.new_event_loop()
    loop.run_until_complete(clock("sub"))
    loop.close()


def create_process():
    time.sleep(2)
    proc = mp.Process(target=create_another_loop)
    proc.start()
    proc.join()
    return "ok"


async def create_process_in_pool():
    return await main_loop.run_in_executor(ProcessPoolExecutor(), create_process)


if __name__ == '__main__':
    main_loop = asyncio.get_event_loop()
    tasks = (
        clock("main"),
        create_process_in_pool(),
    )
    print(main_loop.run_until_complete(asyncio.gather(*tasks)))
    main_loop.close()

其他可能的解決方法：在開始循環之前創建進程，或使用asyncio.create_subprocess_exec甚至允許通過流與子進程進行通信 。