移至下一頁以使用BeautifulSoup進行抓取

Question

我無法自動執行以下代碼以轉到下一頁並從Indeed.com抓取數據。 請讓我知道如何處理此問題。

import requests 
import bs4 
from bs4 import BeautifulSoup 
import pandas as pd 
import time

URL = "https://www.indeed.com/jobs?q=Amazon&l="

# Get the html info of the page 
page = requests.get(URL) 
soup = BeautifulSoup(page.text, "html.parser")

# Get the job title 
def extract_job_title_from_result(soup):    
    jobs = []   
    for div in soup.find_all(name="div",attrs={"class":"row"}):
        for a in div.find_all(name="a", attrs={"data-tn-element":"jobTitle"}):
            jobs.append(a["title"])   
        return(jobs) 

extract_job_title_from_result(soup)

# Get company name 
def extract_company_from_result(soup):    
    companies = []   
    for div in soup.find_all(name="div", attrs={"class":"row"}):
        company = div.find_all(name="span", attrs={"class":"company"})
            if len(company) > 0:
                for b in company:
                    companies.append(b.text.strip())
                else:
                    sec_try = div.find_all(name="span", attrs={"class":"result-link-source"})
                    for span in sec_try:
                        companies.append(span.text.strip())
    return(companies) 

extract_company_from_result(soup)

ocations = extract_location_from_result(soup)
jobs = extract_job_title_from_result(soup)
companies = extract_company_from_result(soup) 
summary = extract_summary_from_result(soup)

columns = {'company_name': companies, 'job_title': jobs}
df = pd.DataFrame.from_dict(columns, orient='index')
df = df.transpose()

我試圖將參數添加到url並使用for循環，但是它不起作用。 我真的很感激一個有效的解決方案。

Answer 1

使用頁碼移至下一頁。嘗試以下代碼讓我知道這是否適合您。

from bs4 import BeautifulSoup
import pandas as pd
import re
headers = {'User-Agent':
       'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}

page = "https://www.indeed.com/jobs?q=Amazon&l="
company_name = []
job_title = []
page_num = 10
session = requests.Session()
while True:
    pageTree = session.get(page, headers=headers)
    pageSoup = BeautifulSoup(pageTree.content, 'html.parser')
    jobs= pageSoup.find_all("a", {"data-tn-element": "jobTitle"})
    Companys = pageSoup.find_all("span", {"class": "company"})
    for Company, job in zip(Companys, jobs):
        companyname=Company.text
        company_name.append(companyname.replace("\n",""))
        job_title.append(job.text)
    if pageSoup.find("span", text=re.compile("Next")):
        page = "https://www.indeed.com/jobs?q=Amazon&start={}".format(page_num)
        page_num +=10
    else:
        break

df = pd.DataFrame({"company_name":company_name,"job_title":job_title})

print(df.head(1000))

輸出：

                                          company_name                                          job_title
0                                           Amazon HVH  Warehouse Team Member (Part-Time, Full-Time, F...
1                                           Amazon HVH  Warehouse Team Member (Seasonal, Part-Time, Fu...
2                                           Amazon HVH  Warehouse/Shopper Team Member (Seasonal, Part-...
3                                           Amazon.com  Amazon Go Retail Associate - Full-time & Part-...
4                                           Amazon HVH  Warehouse Team Member (Seasonal, Part-Time, Fl...
5                                           Amazon HVH                      Warehouse/Shopper Team Member
6                                           Amazon HVH             Amazon Warehouse Fulfillment Associate
7                                           Amazon.com       Amazon Go Retail Associate - Overnight Shift
8                                           Amazon HVH                              Warehouse Team Member
9                                           Amazon HVH  Shopper Team Member (Seasonal, Part-Time, Full...
10                                          Amazon HVH  Warehouse/Shopper Team Member (Seasonal, Part-...
11                                          Amazon HVH        Warehouse Team Member (Seasonal, Full-Time)
12           ISS Allentown - Hiring for Amazon Fulf...                                        Help Wanted
13                                          Amazon HVH    Warehouse (Seasonal, Part-Time, Flexible Hours)
14                           Amazon.com Services, Inc.                                  Process Assistant
15                                          Amazon HVH  Warehouse Shopper/Team Member- Moonachie, Tete...
16                                          Amazon HVH  Warehouse/Shopper Team Member (Seasonal, Part-...
17                           Amazon.com Services, Inc.                         Lead Fulfillment Associate
18                                          Amazon HVH  Warehouse Team Member (Seasonal, Part Time, Fl...
19                                          Amazon HVH            Part-Time Amazon Fresh Pickup Associate
20                                          Amazon.com        Amazon Go Lead Retail Associate - Overnight
21                           Amazon.com Services, Inc.                          Full Time Shift Assistant
22                                          Amazon.com                    Amazon Go Lead Retail Associate
23                           Amazon.com Services, Inc.                                Receiving Associate
24                                          Amazon.com                               Packager - Amazon Go
25                                   Amazon Retail LLC                    Warehouse Associate - Amazon Go
26                                          Amazon.com             Retail Sales Associate - Woodridge, IL
27                           Amazon.com Services, Inc.                         Operations Admin Assistant
28                                          Amazon HVH                     Amazon Warehouse - Milford, MA
29                                          Amazon.com                        Seasonal Delivery Associate
..                                                 ...                                                ...
970                                         Amazon.com                            Optimization Specialist
971                          Amazon.com Services, Inc.  Operations Program Manager, Social Responsibility
972                          Amazon.com Services, Inc.                                 Paid Media Manager
973                          Amazon.com Services, Inc.           Amazon S3, Software Development Engineer
974                                         Amazon.com                             Sr. Facilities Manager
975                                         Amazon.com     Software Development Engineer - Amazon Devices
976                          Amazon.com Services, Inc.           Senior HR Specialist- Work Authorization
977                                         Amazon.com             Media Software Engineer - Amazon Chime
978                          Amazon.com Services, Inc.                          Senior Designer - Digital
979                          Amazon.com Services, Inc.                                 Knowledge Engineer
980                          Amazon.com Services, Inc.                                  Research Engineer
981                          Amazon.com Services, Inc.         Data Engineer, Talent Management Analytics
982                          Amazon.com Services, Inc.                         AWS TRANSPORTATION MANAGER
983                                         Amazon.com  Strategic Partner Development Manager, Retail ...
984                          Amazon.com Services, Inc.  Software Development Engineer, Localization - ...
985                                Amazon Services LLC                         Email Marketing Specialist
986                          Amazon.com Services, Inc.                             Event Producer Manager
987                                         Amazon.com                                 Content Strategist
988                                Amazon Robotics LLC                       Commodity Management Analyst
989                          Amazon Web Services, Inc.     AWS Institute Operations and Relations Manager
990                          Amazon.com Services, Inc.                            Marketing Manager, Cleo
991                          Amazon.com Services, Inc.              Manager, Programmatic Partner Manager
992                                         Amazon.com  GSOC Program Manager (Amazon Business Assuranc...
993                          Amazon.com Services, Inc.  Sr. HR Assistant - Military Spouse Preferred -...
994                                 Amazon Studios LLC  Sr. Development and Programming Executive - Ge...
995                          Amazon.com Services, Inc.                    Financial Analyst II, AGFS FP&A
996                      Amazon Capital Services, Inc.        Principal Enterprise Sales - Amazon Connect
997                        Amazon Digital Services LLC                  Sr Product Manager, Amazon Photos
998                          Amazon.com Services, Inc.                                Prime Air Site Lead
999                          Amazon.com Services, Inc.  Applied Scientist Winter/Fall Internship - Nat...