在 AsyncIO 中处理超时异常

Question

import asyncio
import aiohttp
from time import perf_counter
import csv

path = "*******************"
domains = []
total_count=0

with open(path, 'r') as file:
    csvreader = csv.reader(file)
    for row in csvreader:
        try:
            website = row[4].split("//")[-1].split("www.")[-1].split('/')[0]
            if website == "":
                continue
            domains.append(website)
        except:
            continue
sample = domains[0:50]

async def fetch(s, body):
    async with s.post('https://****************', json=body) as r:
        if r.status!= 200:
            pass
        enrich_response = await r.json()
        #print(enrich_response)
        employees = enrich_response['employees']
        for employee in employees:
            if(employee['job_title'] == "Owner"):
                print(employee)
                print("************************************************")
                global total_count
                total_count += 1
                print("Total Count:", total_count)
                continue
            elif(employee['job_title'] == "CEO"):
                print(employee)
                print("***************************************************")
                total_count+=1
                print("Total Count:", total_count)
                continue
            else:
               continue

async def fetch_all(s,bodies):
    tasks = []
    for body in bodies:
        task = asyncio.create_task(fetch(s, body))
        tasks.append(task)
    res = await asyncio.gather(*tasks)
    return res

async def main():
    # apikeys = list(apikeysone.keys.values())
    bodies = []
    for domain in sample:
        body = {
        "api_key": "********************************",
        "domain" : "{}".format(domain)
        }
        bodies.append(body)
    async with aiohttp.ClientSession() as session:
        data = await fetch_all(session, bodies)
        print(data[0])

if __name__ == '__main__':
    start = perf_counter()
    try:
        asyncio.run(main())
    except Exception as e:
        print(e)
        pass
    stop = perf_counter()
    print("Time taken:", stop - start)

你好！

我正在尝试使用 asyncio 连接到抓取服务提供商，而不是简单的同步 api 调用。

但是我收到超时错误。 我如何使用异常处理等待几秒钟再重试一次？ 或者如果它失败了就跳过那个任务？

提前谢谢各位码友！

尝试添加到某些地方继续/通过

Answer 1

尝试探索asyncio.wait_for()函数。 它需要一个等待值和一个超时值。 如果任务在超时值之前没有完成，它会引发asyncio.exceptions.TimeoutError ，您可以在except子句中以任何您想要的方式处理它。

一个典型的例子（来自 Python 文档）如下：

async def eternity():
    # Sleep for one hour
    await asyncio.sleep(3600)
    print('yay!')

async def main():
    # Wait for at most 1 second
    try:
        await asyncio.wait_for(eternity(), timeout=1.0)
    except TimeoutError:
        print('timeout!')

asyncio.run(main())

# Expected output:
#
#     timeout!