无法获取来自多个页面的所有链接不变的URL

Question

I want to get all links from 10 pages but i am unable to click the second page link. From the url https://10times.com/search?cx=partner-pub-8525015516580200%3Avtujn0s4zis&cof=FORid%3A10&ie=ISO-8859-1&q=%22Private+Equity%22&searchtype=All

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import  bs4

from selenium import webdriver
import time

url = "https://10times.com/search?cx=partner-pub-8525015516580200%3Avtujn0s4zis&cof=FORid%3A10&ie=ISO-8859-1&q=%22Private+Equity%22&searchtype=All"
driver = webdriver.Chrome("C:\\Users\Ritesh\PycharmProjects\BS\drivers\chromedriver.exe")
driver.get(url)

def getnames(driver):
    soup = bs4.BeautifulSoup(driver.page_source, 'lxml')
    sink = soup.find("div", {"class": "gsc-results gsc-webResult"})
    links = sink.find_all('a')
    for link in links:
        try:
            print(link['href'])
        except:
            print("")

while True:
    getnames(driver)
    time.sleep(5)
    nextpage = driver.find_element_by_link_text("2")
    nextpage.click()
    time.sleep(2)

Please help me in solving this issue.

Answer 1

You will need to use selenium, since you have dynamic elements in the page. The code below will get all the links from each page:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
import time

url = "https://10times.com/search?cx=partner-pub-8525015516580200%3Avtujn0s4zis&cof=FORid%3A10&ie=ISO-8859-1&q=%22Private+Equity%22&searchtype=All"
driver = webdriver.Chrome("C:\\Users\Ritesh\PycharmProjects\BS\drivers\chromedriver.exe")
driver.get(url)

WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, """//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[2]/div[11]/div""")))


pages_links = driver.find_elements_by_xpath("""//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[2]/div[11]/div/div""")

all_urls = []

for page_index in range(len(pages_links)):

    WebDriverWait(driver, 20).until(
 EC.presence_of_element_located((By.XPATH, """//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[2]/div[11]/div""")))

    pages_links = driver.find_elements_by_xpath("""//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[2]/div[11]/div/div""")

    page_link = pages_links[page_index]
    print "getting links for page: ", page_link.text

    page_link.click()

    time.sleep(1)


    #wait untill all links are loaded
    WebDriverWait(driver, 20).until(
  EC.presence_of_element_located((By.XPATH, """//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]""")))

    first_link = driver.find_element_by_xpath("""//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[1]/div[1]/div[1]/div/a""")

    results_links = driver.find_elements_by_xpath("""//*[@id="___gcse_0"]/div/div/div/div[5]/div[2]/div/div/div[2]/div/div[1]/div[1]/div/a""")

    urls = [first_link.get_attribute("data-cturl")] + [l.get_attribute("data-cturl") for l in results_links]

    all_urls = all_urls + urls


driver.quit()

You can use this code as it is or try to combined with the one you already have.

Note it does not consider the ad links, since I don't think you will need them, right?

Let me know if this helps.