在导航到下一页并使用网站上的 selenium 抓取所有数据时出错？

Question

Hi Here I am trying to scrape all the teacher jobs from https://www.naukri.com/ this url I want all the pages data but I am getting only one page data and getting this error

Traceback (most recent call last):
  File "naukri.py", line 48, in <module>
    driver.execute_script("arguments.click();", next_page)
  File "/home/nyros/Documents/mypython/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 636, in execute_script
    'args': converted_args})['value']
  File "/home/nyros/Documents/mypython/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "/home/nyros/Documents/mypython/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.JavascriptException: Message: javascript error: arguments.click is not a function
  (Session info: chrome=80.0.3987.116)

The code which I wrote is:

import selenium.webdriver

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

url ='https://www.naukri.com/'
driver = webdriver.Chrome(r"mypython/bin/chromedriver_linux64/chromedriver")


driver.get(url)
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#qsbClick > span.blueBtn'))).click()
driver.find_element_by_xpath('//*[@id="skill"]/div[1]/div[2]/input').send_keys("teacher")
driver.find_element_by_xpath('//*[@id="qsbFormBtn"]').click()

data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
result = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
for r in result:
    data = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
    result = WebDriverWait(data, 10).until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
    for r in result:
        try:
            title=r.find_element_by_class_name("desig").text
            print('title:',title)
            school=r.find_element_by_class_name("org").text
            print('school:',school)
            location=r.find_element_by_class_name("loc").text
            print("location:",location)
            salary=r.find_element_by_class_name("salary").text
            print("salary:",salary)
        except:
            pass
            print('-------')
    next_page = r.find_elements_by_xpath("/html/body/div[5]/div/div[3]/div[1]/div[59]/a/button")
    driver.execute_script("arguments.click();", next_page)

Please help me anyone Thanks in advance!

Answer 1

Since the element index of the 'next' button changes from 59 in the first page to 60 in the next pages, you can just find all elements on the page which have class "grayBtn" , and click on index [-1] of the list returned, as this will always provide the next button. I removed some unnecessary parts of your code too, like repeated importations as well as unnecessary button clicks. I instantly directed to the page containing the list of results for teachers, instead of entering "teacher" into the search field on the home page. I was left with the following:

from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re    

Category = input("Category?")
Category = re.sub(" ", "%20", Category)
Type = re.sub(" ", "-", Category.lower())

url ='https://www.naukri.com/' + Type + '-jobs?k=' + Category
driver = webdriver.Chrome(r"mypython/bin/chromedriver_linux64/chromedriver")
driver.get(url)

data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
result = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
for res in result:
    data = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
    jobs = WebDriverWait(data, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
    for job in jobs:
        try:
            title=job.find_element_by_class_name("desig").text
            print('title:',title)
            school=job.find_element_by_class_name("org").text
            print('school:',school)
            location=job.find_element_by_class_name("loc").text
            print("location:",location)
            salary=job.find_element_by_class_name("salary").text
            print("salary:",salary)
        except:
            pass
            print('-------')
    Button = driver.find_elements_by_class_name("grayBtn")[-1]
    time.sleep(1)
    driver.execute_script("window.scrollTo(0,document.body.scrollHeight - 1300)")
    Button.click()

As requested, here is the modified code to append data to a pandas dataframe and convert the dataframe to excel:

from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re
import pandas as pd

df = pd.DataFrame(columns = ['Title', 'School', 'Location', 'Salary'])

Category = input("Category?")
Category = re.sub(" ", "%20", Category)
Type = re.sub(" ", "-", Category.lower())

url ='https://www.naukri.com/' + Type + '-jobs?k=' + Category
driver = webdriver.Chrome(r"mypython/bin/chromedriver_linux64/chromedriver")
driver.get(url)

data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
result = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
i = 0
for res in result:
    data = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "srp_container.fl")))
    jobs = WebDriverWait(data, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, "row")))
    for job in jobs:
        try:
            title=job.find_element_by_class_name("desig").text
            print('title:',title)
            school=job.find_element_by_class_name("org").text
            print('school:',school)
            location=job.find_element_by_class_name("loc").text
            print("location:",location)
            salary=job.find_element_by_class_name("salary").text
            print("salary:",salary)
            df.loc[i] = [title, school, location, salary]
            i += 1
        except:
            pass
            print('-------')
    Button = driver.find_elements_by_class_name("grayBtn")[-1]
    time.sleep(1)
    driver.execute_script("window.scrollTo(0,document.body.scrollHeight - 1300)")
    Button.click()
df.to_excel("all_results.xlsx")