Selenium 在 Chrome 中有效，但在使用无头 Chrome 时无效

Question

我有这段代码在我使用 Chrome 时可以正常工作，但是，当我想切换到无头 Chrome 时，我得到selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document and它不返回任何东西。 任何想法为什么？

代码：

class IOLSpider(scrapy.Spider):
    name = 'iol'
    start_urls = [
        'http://www.iolproperty.co.za/all_properties/For_Sale/Western_Cape',
        'http://www.iolproperty.co.za/all_properties/Rentals/Western_Cape',
    ]

    def __init__(self):
        #path to driver
        chrome_options = webdriver.ChromeOptions()
        chrome_options.headless = True
        self.driver = webdriver.Chrome('/path/chromedriver',chrome_options=chrome_options)

    def parse(self, response):
        . . . 

    def parse_area(self, response):

        . . .

    def parse_property(self,response):
        #the link that comes here is the link of property, like this one
    #https://www.iolproperty.co.za/view-property.jsp?PID=2000026825
        item = response.meta.get('item')
        self.driver.get(response.url)
        self.driver.current_url
        self.driver.execute_script("document.getElementById('footcont').setAttribute('hidden', true)")
        elem = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//a[text()="Show email address"]')))
        elem.click()
        contact_email=self.driver.find_element_by_xpath('//span[@id="viewagmail" and @style="display: block;"]/a').text

Answer 1

您尝试抓取的网站检测到您正在使用无头浏览器并且播放效果不佳。 这在过去对我有用，但您可能需要根据您的特定需求进行调整。

url = "https://www.iolproperty.co.za/view-property.jsp?PID=2000026825"

options = Options()
options.add_argument('--no-sandbox')
options.add_argument("--headless")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

with webdriver.Chrome(options=options) as driver:
    driver.execute_cdp_cmd('Network.setUserAgentOverride', {
        "userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})
    driver.get(url)
    driver.execute_script("document.getElementById('footcont').setAttribute('hidden', true)")

    elem = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.XPATH, '//a[text()="Show email address"]')))
    elem.click() # This can be combined with the webdriver wait line, but isn't necessary.

    contact_email = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, '//span[@id="viewagmail" and @style="display: block;"]/a[starts-with(@href, "mailto")]'))).text