如何将多处理与 asyncio 结合起来

Question

I have url checker code to check its response 200 or not but I want make it asynchronous along with multiprocessing but i stuck in the code plz help me build the code.

import aiohttp
import asyncio
from aiomultiprocess import Pool
# ================================================================================


async def fetch(session,url):
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                if response.status != 200:
                    response.raise_for_status()
                return {'url':url, 'action':'success', 'status_code':response.status}
    except:
        return {'url':url, 'action':'error', 'status_code':404}    

async def fetch_all(session, urls):
    tasks = []
    for url in urls:
        task = asyncio.create_task(fetch(session, url))
        tasks.append(task)
    results = await asyncio.gather(*tasks)
    return results

calling the function

async with aiohttp.ClientSession() as session:
    dataset = await fetch_all(session, urls)

Answer 1

The easiest way would be to use Pool.starmap() :

async def fetch_all(session, urls):
    async with Pool() as pool:
        starargs = [(session, url) for url in urls]
        results = await pool.starmap(fetch, starargs)
        return results