Unable to fetch all the necessary links during Iteration - Selenium Python

Question

I am newbie to Selenium Python. I am trying to fetch the profile URLs which will be 10 per page. Without using while , I am able to fetch all 10 URLs but for only the first page alone. When I use while , it iterates, but fetches only 3 or 4 URLs per page.

I need to fetch all the 10 links and keep iterating through pages. I think, I must do something with StaleElementReferenceException

Kindly help me solve this problem.

Given the code below.

def test_connect_fetch_profiles(self):
    driver = self.driver
    search_data = driver.find_element_by_id("main-search-box")
    search_data.clear()
    search_data.send_keys("Selenium Python")
    search_submit = driver.find_element_by_name("search")
    search_submit.click()
    noprofile = driver.find_elements_by_xpath("//*[text() = 'Sorry, no results containing all your search terms were found.']")
    self.assertFalse(noprofile)
    while True:
        wait = WebDriverWait(driver, 150)
        try:
            profile_links = wait.until(EC.presence_of_all_elements_located((By.XPATH,"//*[contains(@href,'www.linkedin.com/profile/view?id=')][text()='LinkedIn Member'or contains(@href,'Type=NAME_SEARCH')][contains(@class,'main-headline')]")))
            for each_link in profile_links:
                page_links = each_link.get_attribute('href')
                print(page_links)
                driver.implicitly_wait(15)
                appendFile = open("C:\\Users\\jayaramb\\Documents\\profile-links.csv", 'a')
                appendFile.write(page_links + "\n")
                appendFile.close()
                driver.implicitly_wait(15)
                next = wait.until(EC.visibility_of(driver.find_element_by_partial_link_text("Next")))
                if next.is_displayed():
                    next.click()
                else:
                    print("End of Page")
                    break
        except ValueError:
            print("It seems no values to fetch")
        except NoSuchElementException:
            print("No Elements to Fetch")
        except StaleElementReferenceException:
             print("No Change in Element Location")
        else:
                break

Please let me know if there are any other effective ways to fetch the required profile URL and keep iterating through pages.

Answer 1

I created a similar setup which works alright for me. I've had some problems with selenium trying to click on the next-button but it throwing a WebDriverException instead, likely because the next-button is not in view. Hence, instead of clicking the next-button I get its href-attribute and load the new page up with driver.get() and thus avoiding an actual click making the test more stable.

def test_fetch_google_links():

    links = []

    # Setup driver
    driver = webdriver.Firefox()
    driver.implicitly_wait(10)
    driver.maximize_window()

    # Visit google
    driver.get("https://www.google.com")

    # Enter search query
    search_data = driver.find_element_by_name("q")
    search_data.send_keys("test")

    # Submit search query
    search_button = driver.find_element_by_xpath("//button[@type='submit']")
    search_button.click()

    while True:
        # Find and collect all anchors
        anchors = driver.find_elements_by_xpath("//h3//a")
        links += [a.get_attribute("href") for a in anchors]

        try:
            # Find the next page button
            next_button = driver.find_element_by_xpath("//a[@id='pnnext']")
            location = next_button.get_attribute("href")
            driver.get(location)

        except NoSuchElementException:
            break

    # Do something with the links
    for l in links:
        print l

    print "Found {} links".format(len(links))

    driver.quit()