如何使用BeautifulSoup遍歷在多個網頁上抓取多個文檔？

Question

我有興趣從“自然語言處理”項目的醫療文檔網頁中獲取文本。 我要抓取的網頁文檔文本沒有設計任何語義標記，只是帶有粗體標題的一大塊文本。 獲得一些幫助並從第一頁開始，我感興趣的是，我實現了以下代碼以從網頁中獲取文檔文本：

import requests
import re
from bs4 import BeautifulSoup, Tag, NavigableString, Comment

url = 'https://www.mtsamples.com/site/pages/sample.asp?Type=24- Gastroenterology&Sample=2332-Abdominal%20Abscess%20I&D'
res = requests.get(url)
res.raise_for_status()
html = res.text
soup = BeautifulSoup(html, 'html.parser')

title_el = soup.find('h1')
page_title = title_el.text.strip()
first_hr = title_el.find_next_sibling('hr')

description_title = title_el.find_next_sibling('b', 
text=re.compile('description', flags=re.I))
description_text_parts = []
for s in description_title.next_siblings:
    if s is first_hr:
        break
    if isinstance(s, Tag):
        description_text_parts.append(s.text.strip())
    elif isinstance(s, NavigableString):
        description_text_parts.append(str(s).strip())
description_text = '\n'.join(p for p in description_text_parts if p.strip())

# titles are all bold and uppercase
titles = [b for b in first_hr.find_next_siblings('b') if b.text.strip().isupper()]

docs = []
for t in titles:
    text_parts = []
    for s in t.next_siblings:
        # go until next title
        if s in titles:
            break
        if isinstance(s, Comment):
            continue
        if isinstance(s, Tag):
            if s.name == 'div':
                break
            text_parts.append(s.text.strip())
        elif isinstance(s, NavigableString):
            text_parts.append(str(s).strip())
    text = '\n'.join(p for p in text_parts if p.strip())
    docs.append({
        'title': t.text.strip(),
        'text': text
    })

這會將我的文檔文本作為字典添加，由title字段鍵和text值分隔為一個字典，該列表名為docs 。 此時，在上面的示例中抓取的網頁將是docs列表中的唯一元素。

我感興趣的是創建一個循環，以從位於https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1的網頁上的Gastroenterology部分添加所有醫療文檔記錄。 有23個單獨的頁面，每個頁面按字母順序包含許多不同的醫學文檔，總共包含230個醫學文檔。 我想知道執行此循環的最佳方法是什么？ 同樣，我的目標是將每個醫療文檔附加到docs列表中，如之前代碼中第一個示例所示。 任何幫助將非常感激！

Answer 1

對於可靠的解決方案，只要存在“下一個”（ > ）按鈕，就可以使用while循環連續迭代鏈接。 在每次迭代時，您都可以抓取下一頁並提取帶有鏈接和描述的標題，以供以后使用：

import requests, re
from bs4 import BeautifulSoup as soup
def parse_page(_d):
   data = filter(lambda x:x != '\n', [i for i in _d.find('table', {'id':'Browse'}).td.contents if isinstance(i, str) or i.name == 'a'])
   _next = next(data, None)
   while _next is not None:
      new_d, _n = {'title':_next.text, 'link':_next['href']}, next(data, None)
      if hasattr(_n, 'text'):
         yield new_d
         yield {'title':_n.text, 'link':_n['href'], 'about':next(data)}
      else:
         yield {**new_d, 'about':_n}   
      _next = next(data, None)  

d, r = soup(requests.get('https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1').text, 'html.parser'), []
r.append(list(parse_page(d)))
_c = [i for i in d.find('div', {'class':'Contrast'}).find_all('a') if i.text == '>']
while _c:
   d = soup(requests.get(f'https://www.mtsamples.com{_c[0]["href"]}').text, 'html.parser')
   r.append(list(parse_page(d)))
   _c = [i for i in d.find('div', {'class':'Contrast'}).find_all('a') if i.text == '>']

輸出（ r第一個元素，由於SO的字符限制）：

[{'title': 'Abdominal Abscess I&D;', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2332-Abdominal Abscess I&D', 'about': 'Incision and drainage (I&D;) of abdominal abscess, excisional debridement of nonviable and viable skin, subcutaneous tissue and muscle, then removal of foreign body.'}, {'title': 'Abdominal Exploration', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2038-Abdominal Exploration', 'about': 'Congenital chylous ascites and chylothorax and rule out infradiaphragmatic lymphatic leak.   Diffuse intestinal and mesenteric lymphangiectasia.  '}, {'title': 'Abdominal Pain - Consult', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1921-Abdominal Pain - Consult', 'about': 'The patient presented to the emergency room last evening with approximately 7- to 8-day history of abdominal pain which has been persistent.'}, {'title': 'Abscess Excision', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2330-Abscess Excision', 'about': 'Excision of abscess, removal of foreign body.  Repair of incisional hernia.  Recurrent re-infected sebaceous cyst of abdomen.  Abscess secondary to retained foreign body and incisional hernia.'}, {'title': 'Admission History & Physical - Nausea', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1152-Admission History & Physical - Nausea', 'about': 'Patient status post gastric bypass surgery, developed nausea and right upper quadrant pain.'}, {'title': 'Adrenalectomy & Umbilical Hernia Repair', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2464-Adrenalectomy & Umbilical Hernia Repair', 'about': 'Laparoscopic hand-assisted left adrenalectomy and umbilical hernia repair.  Patient with a 5.5-cm diameter nonfunctioning mass in his right adrenal.'}, {'title': 'Air Under Diaphragm - Consult', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2166-Air Under Diaphragm - Consult', 'about': 'Possible free air under the diaphragm.  On a chest x-ray for what appeared to be shortness of breath she was found to have what was thought to be free air under the right diaphragm.  No intra-abdominal pathology.'}, {'title': 'Appendectomy', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=1975-Appendectomy', 'about': 'Appendicitis, nonperforated.  Appendectomy.  A transverse right lower quadrant incision was made directly over the point of maximal tenderness. '}, {'title': 'Appendectomy - 1', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2298-Appendectomy - 1', 'about': 'Acute appendicitis, gangrenous.  Appendectomy.'}, {'title': 'Appendectomy - Laparoscopic ', 'link': '/site/pages/sample.asp?Type=24-Gastroenterology&Sample=2728-Appendectomy - Laparoscopic ', 'about': 'Laparoscopic appendectomy and peritoneal toilet and photos.  Pelvic inflammatory disease and periappendicitis.'}]

Answer 2

只需找到所有分頁網址，然后遍歷所有這些頁面，找到文檔網址並提取文檔即可。 這是為您提供的完整解決方案。

這將同時瀏覽頁面並從所有頁面中批量提取文檔

import requests
from bs4 import BeautifulSoup, Tag, Comment, NavigableString
from urllib.parse import urljoin
from pprint import pprint
import itertools
import concurrent
from concurrent.futures import ThreadPoolExecutor

BASE_URL = 'https://www.mtsamples.com'


def make_soup(url: str) -> BeautifulSoup:
    res = requests.get(url)
    res.raise_for_status()
    html = res.text
    soup = BeautifulSoup(html, 'html.parser')
    return soup


def make_soup_parallel(urls: list) -> list:
    workers = min(10, len(urls))
    with ThreadPoolExecutor(max_workers=workers) as e:
        return list(e.map(make_soup, urls))


def find_pagination_urls(soup: BeautifulSoup) -> list:
    urls = set()
    for a in soup.select('.Contrast a'):
        if not a.text.isnumeric():
            continue
        url = urljoin(BASE_URL, a['href'])
        urls.add(url)
    return sorted(list(urls), key=lambda u: int(u.split('page=')[1]))


def find_document_urls(soup: BeautifulSoup) -> list:
    urls = []
    for a in soup.select('#Browse a'):
        url = urljoin(BASE_URL, a['href'])
        urls.append(url)
    return urls


def find_all_doc_urls() -> list:
    index_url = 'https://www.mtsamples.com/site/pages/browse.asp?type=24-Gastroenterology&page=1'
    index_soup = make_soup(index_url)

    next_pages = find_pagination_urls(index_soup)
    doc_urls = []
    for soup in make_soup_parallel(next_pages):
        doc_urls.extend(find_document_urls(index_soup))
    return doc_urls


def extract_docs(soup: BeautifulSoup) -> list:
    title_el = soup.find('h1')
    first_hr = title_el.find_next_sibling('hr')

    # titles are all bold and uppercase
    titles = [b for b in first_hr.find_next_siblings('b') if b.text.strip().isupper()]

    docs = []
    for t in titles:
        text_parts = []
        for s in t.next_siblings:
            # go until next title
            if s in titles:
                break
            if isinstance(s, Comment):
                continue
            if isinstance(s, Tag):
                if s.name == 'div':
                    break
                text_parts.append(s.text.strip())
            elif isinstance(s, NavigableString):
                text_parts.append(str(s).strip())
        text = '\n'.join(p for p in text_parts if p.strip())
        docs.append({
            'title': t.text.strip(),
            'text': text
        })
    return docs


def batch(it, n: int):
    it = [iter(it)] * n
    return itertools.zip_longest(*it, fillvalue=None)


docs = []
doc_urls = find_all_doc_urls()

for b in batch(doc_urls, 5):
    batch_urls = list(filter(bool, b))
    for soup in make_soup_parallel(batch_urls):
        docs.extend(extract_docs(soup))
pprint(docs)

輸出：

[{'text': 'Abdominal wall abscess.', 'title': 'PREOPERATIVE DIAGNOSIS:'},
 {'text': 'Abdominal wall abscess.', 'title': 'POSTOPERATIVE DIAGNOSIS:'},
 {'text': 'Incision and drainage (I&D) of abdominal abscess, excisional '
          'debridement of nonviable and viable skin, subcutaneous tissue and '
          'muscle, then removal of foreign body.',
  'title': 'PROCEDURE:'},
 {'text': 'LMA.', 'title': 'ANESTHESIA:'},
...