为什么我的第二个 python 异步（抓取）function（使用第一个异步（抓取）函数的结果）没有返回结果？

Question

程序应该做什么的总结：

第 1 步（同步）：准确确定需要抓取的页面数量。

第 2 步（同步）：创建指向要在 for 循环中抓取的页面的链接。

第 3 步（异步）：使用第 2 步中的链接列表从每个页面获取指向所需详细信息页面的链接。

第 4 步（异步）：使用第 3 步的结果提取每个 hofladen 的详细信息。 此信息存储在每个农场商店的列表中，并且每个列表都附加到全局列表。

我哪里有问题？

从第 3 步到第 4 步的转换似乎无法正常工作。

Traceback (most recent call last):
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 108, in <module>
    asyncio.run(main())
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 96, in main
    await asyncio.gather(*tasks_detail_infos)
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 61, in scrape_detail_infos
    data = JsonLdExtractor().extract(body_d)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/extruct/jsonld.py", line 21, in extract
    tree = parse_html(htmlstring, encoding=encoding)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/extruct/utils.py", line 10, in parse_html
    return lxml.html.fromstring(html, parser=parser)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/lxml/html/__init__.py", line 873, in fromstring
    doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/lxml/html/__init__.py", line 761, in document_fromstring
    raise etree.ParserError(
lxml.etree.ParserError: Document is empty

Process finished with exit code 1

我做了什么来隔离问题？

在第一次尝试中，我重写了异步 function append_detail_infos，这样它就不再尝试创建列表和 append 值，而只打印数据 [0][“名称”]。 这导致了错误消息

Traceback (most recent call last):
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 108, in <module>
    asyncio.run(main())
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 96, in main
    await asyncio.gather(*tasks_detail_infos)
  File "/Users/REPLACED_MY_USER/PycharmProjects/PKI-Projekt/test_ttt.py", line 61, in scrape_detail_infos
    data = JsonLdExtractor().extract(body_d)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/extruct/jsonld.py", line 21, in extract
    tree = parse_html(htmlstring, encoding=encoding)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/extruct/utils.py", line 10, in parse_html
    return lxml.html.fromstring(html, parser=parser)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/lxml/html/__init__.py", line 873, in fromstring
    doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
  File "/Users/REPLACED_MY_USER/miniconda3/envs/scrapy/lib/python3.10/site-packages/lxml/html/__init__.py", line 761, in document_fromstring
    raise etree.ParserError(
lxml.etree.ParserError: Document is empty

Process finished with exit code 1

在下一次尝试中，我将 detail_links 中的链接导出为 .csv 并目视检查并打开其中一些以查看它们是否有效。 情况也是如此。

程序代码：

import asyncio
import time
import aiohttp
import requests
import re
from selectolax.parser import HTMLParser
from extruct.jsonld import JsonLdExtractor
import pandas as pd

BASE_URL = "https://hofladen.info"
FIRST_PAGE = 1


def get_last_page(url: str) -> int:
    res = requests.get(url).text
    html = HTMLParser(res)
    last_page = int(re.findall("(\d+)", html.css("li.page-last > a")[0].attributes["href"])[0])

    return last_page


def build_links_to_pages(start: int, ende: int) -> list:
    lst = []
    for i in range(start, ende + 1):
        url = f"https://hofladen.info/regionale-produkte?page={i}"
        lst.append(url)

    return lst


async def scrape_detail_links(url: str):
    async with aiohttp.ClientSession() as session:
        async with session.get(url, allow_redirects=True) as resp:
            body = await resp.text()
            html = HTMLParser(body)

            for node in html.css(".sp13"):
                detail_link = BASE_URL + node.attributes["href"]
                detail_links.append(detail_link)


async def append_detail_infos(data):
    my_detail_lst = []
    # print(data[0]["name"])  # name for debugging purpose
    my_detail_lst.append(data[0]["name"])  # name
    my_detail_lst.append(data[0]["address"]["streetAddress"])  # str
    my_detail_lst.append(data[0]["address"]["postalCode"])  # plz
    my_detail_lst.append(data[0]["address"]["addressLocality"])  # ort
    my_detail_lst.append(data[0]["address"]["addressRegion"])  # bundesland
    my_detail_lst.append(data[0]["address"]["addressCountry"])  # land
    my_detail_lst.append(data[0]["geo"]["latitude"])  # breitengrad
    my_detail_lst.append(data[0]["geo"]["longitude"])  # längengrad

    detail_infos.append(my_detail_lst)


async def scrape_detail_infos(detail_link: str):
    async with aiohttp.ClientSession() as session_detailinfos:
        async with session_detailinfos.get(detail_link) as res_d:
            body_d = await res_d.text()
            data = JsonLdExtractor().extract(body_d)
            await append_detail_infos(data)


async def main() -> None:
    start_time = time.perf_counter()

    # Beginn individueller code
    # ----------
    global detail_links, detail_infos
    detail_links, detail_infos = [], []

    tasks = []
    tasks_detail_infos = []
    # extrahiere die letzte zu iterierende Seite
    last_page = get_last_page("https://hofladen.info/regionale-produkte")

    # scrape detail links
    links_to_pages = build_links_to_pages(FIRST_PAGE, last_page)
    for link in links_to_pages:
        task = asyncio.create_task(scrape_detail_links(link))
        tasks.append(task)



    print("Saving the output of extracted information.")
    await asyncio.gather(*tasks)

    pd.DataFrame(data=detail_links).to_csv("detail_links.csv")

    # scrape detail infos
    for detail_url in detail_links:
        task_detail_infos = asyncio.create_task(scrape_detail_infos(detail_url))
        tasks_detail_infos.append(task_detail_infos)

    await asyncio.gather(*tasks_detail_infos)

    # Ende individueller Code
    # ------------

    time_difference = time.perf_counter() - start_time
    print(f"Scraping time: {time_difference} seconds.")
    print(len(detail_links))
    # print(detail_infos[])



asyncio.run(main())

该问题的有效解决方案：

添加python allow_redirects=True到python async with session_detailinfos.get(detail_link, allow_redirects=True) as res_d:

添加python return_exceptions=True到python await asyncio.gather(*tasks_detail_infos, return_exceptions=True)

Answer 1

该问题的有效解决方案：

添加python allow_redirects=True到python async with session_detailinfos.get(detail_link, allow_redirects=True) as res_d:

添加python return_exceptions=True到python await asyncio.gather(*tasks_detail_infos, return_exceptions=True)