Scrapy 不解析来自 Selenium 的 page_source

Question

Im trying to create a spider with Selenium that would search for merchants in https://www.trustpilot.com and then retrieve tree rating/score from the search. Since there are many merchants to find I have created a list which Seleniuem loops through and then stores the page_source in a list. Idea is that this list of page_source's should then be parsed by Scrapy and return a .json file with merchant ratings. After running the spider I see that the result is 0 pages crawled and an empty .json file. Cant seem to figure out why nothing is parsed. here is my code:-

# -*- coding: utf-8 -*-
import scrapy
from scrapy import Selector
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from shutil import which

queries = ['yeewtuden.com', '1a.lv', 'grishkoshop.com']


class SeleniumTestSpider(scrapy.Spider):
    name = 'selenium_test'
    allowed_domains = ['www.trustpilot.com']
    start_urls = ["www.trustpilot.com"]

    page_responses = []

    def __init__(self):
        super().__init__()
        chrome_options = Options()
        chrome_options.add_argument("--headless")

        chrome_path = which("chromedriver")

        driver = webdriver.Chrome(executable_path=chrome_path, options=chrome_options)
        driver.implicitly_wait(10)
        driver.get("https://www.trustpilot.com")

        # search_field = driver.find_element_by_xpath("//input[@class='searchInputField___3e9zp']")

        for query in queries:
            search_field = WebDriverWait(driver, 7).until(EC.presence_of_element_located((
                By.CLASS_NAME, 'searchInputField___3e9zp')))
            search_field = driver.find_element_by_xpath("//input[@class='searchInputField___3e9zp']")
            search_field.send_keys(query)
            search_field.send_keys(Keys.ENTER)
            self.page_responses.append(driver.page_source)
            driver.back()

        driver.close()

    def parse(self, response):
        for resp in self.page_responses:
            resp = Selector(text=resp)
            score = resp.xpath("//p[@class='header_trustscore']/text()").get()
            yield {
                'rating': score
            }

Answer 1

You can use below code which will return rating.

Create object of class and create a generator which will be used to get the rating.

testSpider = SeleniumTestSpider()
parseGenerator = testSpider.parse(testSpider.page_responses)
for i in parseGenerator:
    print(i, end=" ")