试图用 beautifulsoup 抓取其他类别

Question

这是我要抓取的网站：[https://www.jurongpoint.com.sg/store-directory/]

这是我的代码，如您所见，我不知道如何为 url 变量填充两个 {} 作为我想要抓取的 4 类别，尤其是 url 的服务非常不同。 url 变量上面的注释显示了点击进入时 4 类别的链接。感谢任何帮助，谢谢！

from bs4 import BeautifulSoup
import requests

def parse():


    cate=["Service","Food & Beverage","Fashion & Accessories","Electronics & Technology"]

    #cate=Food+%26+Beverage
    #cate=Electronics+%26+Technology
    #cate=Fashion+%26+Accessories
    #cate=Services


    url="https://www.jurongpoint.com.sg/store-directory/?level=&cate={}+%26+{}"

    for cat in cate:
    
    
        for page in range(1,14):
            print(page)

            soup = BeautifulSoup(requests.get(url).text ,"html.parser")


            for link in soup.find_all('div',class_='entry-content'):

                try:
                    shops=soup.find_all('div',class_="col-9")
                    names=soup.find_all('tr',class_="clickable")

                    for n, k in zip(names, shops):
                        name = n.find_all('td')[1].text.replace(' ','')
                        desc = k.text.replace(' ','')
                        print(name + "\n")
                        print(desc)

                except AttributeError as e:
                    print(e)


                next_button = soup.select_one('.PagedList-skipToNext a')
                if next_button:
                    url = next_button.get('href')
                else:
                    break


parse() 

Answer 1

使用您请求的参数并避免管理转义字符（如 %26）

url = "https://www.jurongpoint.com.sg/store-directory"

for cat in cate:
    for page in range(1, 14):
        print(f'Scraping category {cat} page {page}')
        payload = {
            'level': '',
            'cate': cat,
            'page': page
        }
        resp = requests.get(url, params=payload)
        soup = BeautifulSoup(resp.text, 'html.parser')
        # your code here

>>> resp.url
'https://www.jurongpoint.com.sg/store-directory/?level=&cate=Electronics+%26+Technology&page=8'