如何使用selenium和Scrapy從Flipkart等動態網站中提取數據？

Question

由於Flipkart.com在第一頁上僅顯示15到20個結果，因此滾動顯示更多結果。 Scrapy成功提取第1頁的結果，但不提取下一頁的結果。 我嘗試使用Selenium，但找不到成功。 這是我的代碼： -

from scrapy.spider import Spider

from scrapy.selector import Selector

from flipkart.items import FlipkartItem

from scrapy.spider import BaseSpider

from selenium import webdriver

class FlipkartSpider(BaseSpider):
    name = "flip1"
    allowed_domains = ["flipkart.com"]
    start_urls = [
        "http://www.flipkart.com/beauty-and-personal-care/personal-care-appliances/hair-dryers/pr?sid=t06,79s,mh8&otracker=nmenu_sub_electronics_0_Hair%20Dryers"
]

    def __init__(self):
        self.driver = webdriver.Firefox()

    def parse(self, response):
        sel = Selector(response)
        self.driver.get(response.url)
        while True:
            next = self.driver.find_element_by_xpath('//div[@id="show-more-results"]')
            try:

                sites = sel.select('//div[@class="gd-col gu12 browse-product fk-inf-scroll-item"] | //div[@class="pu-details lastUnit"]')
                for site in sites:
                    item = FlipkartItem()
                    item['title'] = site.select('div//a[@class="lu-title"]/text() | div[1]/a/text()').extract()
                    item['price'] = site.select('div//div[@class="pu-price"]/div/text() | div//div[@class="pu-final"]/span/text()').extract()
                    yield item
                next.wait_for_page_to_load("30")
            except:
                break
            self.driver.close()

，我的items.py是： -

import scrapy
class FlipkartItem(scrapy.Item):
    title=scrapy.Field()
    price=scrapy.Field()

以下輸出我只得到15項： -

[{"price": ["Rs. 599"], "title": ["\n Citron Elegant 1400 W HD001 Hair Dryer (Pink)\n "]},

{"price": ["Rs. 799"], "title": ["\n Citron Vogue 1800 W HD002 Hair Dryer (White)\n "]},

{"price": ["Rs. 645"], "title": ["\n Philips HP8100/00 Hair Dryer (Blue)\n "]},

{"price": ["Rs. 944"], "title": ["\n Philips HP8111/00 Hair Dryer\n "]},

{"price": ["Rs. 171"], "title": ["\n Nova Professional With 2 Speed NV-1290 Hair Dryer (Pink...\n "]},

{"price": ["Rs. 175"], "title": ["\n Nova NHD 2840 Hair Dryer\n "]},

{"price": ["Rs. 775"], "title": ["\n Philips HP 8112 Hair Dryer\n "]},

{"price": ["Rs. 1,925"], "title": ["\n Philips HP8643/00 Miss Fresher's Pack Hair Straightener...\n "]},

{"price": ["Rs. 144"], "title": ["\n Nova Foldable N-658 Hair Dryer (White, Pink)\n "]},

{"price": ["Rs. 1,055"], "title": ["\n Philips HP8100/46 Hair Dryer\n "]},

{"price": ["Rs. 849"], "title": ["\n Panasonic EH-ND12-P62B Hair Dryer (Pink)\n "]},

{"price": ["Rs. 760"], "title": ["\n Panasonic EH-ND11 Hair Dryer (White)\n "]},

{"price": ["Rs. 1,049"], "title": ["\n Panasonic EH-ND13-V Hair Dryer (Violet)\n "]},

{"price": ["Rs. 1,554"], "title": ["\n Philips 1600 W HP4940 Hair Dryer (White & Light Pink)\n "]},

{"price": ["Rs. 2,008"], "title": ["\n Philips Kerashine HP8216/00 Hair Dryer\n "]}]

Answer 1

你必須強制webdriver加載更多的結果。 為了能夠與其他結果進行交互，webdriver需要滾動頁面直到元素出現。

滾動代碼是：

driver.execute_script("window.scrollTo(0, location.get('y')")

要確定滾動的位置，您可以在頁面的下半部分找到一個元素（例如頁腳）並繼續滾動到它。要獲取元素的坐標，您可以使用Webelement屬性位置

driver = webdriver.Firefox()
down = driver.find_element_by_xpath("//someXpath")
location = down.location

Answer 2

您可以使用Javascript向下滾動頁面。

以下代碼將在x和y方向上向下滾動頁面10000,10000。 因為10000是大數字所以它會帶你到頁面底部。 一旦你到達底部，翻轉卡片就會激活AJAX請求以加載更多項目。

window.scrollBy(10000,10000);

我不確定如何在scrapy中做到這一點，但使用硒很容易。

這是代碼

((JavascriptExecutor) driver).executeScript("window.scrollBy(10000,10000);");

Answer 3

我以不同方式管理它..請參閱我的代碼以供進一步參考 完整的網站工作正常..

class FlipkartSpider(BaseSpider):
    name = "flip1"
    allowed_domains = ["flipkart.com"]
    start_urls = [
        "http://www.flipkart.com/tablets/pr?sid=tyy%2Chry&q=mobile&ref=b8b64676-065a-445c-a6a1-bc964d5ff938"
    ]
    '''def is_element_present(self, finder, selector, wait_time=None):
        wait_time = wait_time or self.wait_time
        end_time = time.time() + wait_time
        while time.time() < end_time:
            if finder(selector):
                return True
        return False
        def is_element_present_by_xpath(self, xpath, wait_time=None):
        return self.is_element_present(self.find_by_xpath, xpath, wait_time)
        '''
    def __init__(self):
        self.driver = webdriver.Firefox()


    def parse(self, response):
        sel = Selector(response) 
        self.driver.get(response.url)
        block="block"
        hyper="http://www.flipkart.com"
        print hyper
        #i=0
        while True:
            self.driver.execute_script("window.scrollTo(10000000,10000000)")
            self.driver.set_page_load_timeout(10000)
            try:
                show = self.driver.find_element_by_xpath('//div[@id="show-more-results"]').value_of_css_property('display')
                if show==block:
                    self.driver.find_element_by_xpath('//div[@id="show-more-results"]').click()
                no_more = self.driver.find_element_by_xpath('//*[@id="no-more-results" and @class="dont-show"]').value_of_css_property('display')
                if no_more==block:
                    break;
                time.sleep(5)
                self.driver.execute_script("window.scrollTo(10000000,10000000)")

                self.driver.set_page_load_timeout(10000)
                #if i==7:
                #   break
            except NoSuchElementException:
                print "pungi"
                break
        #down = self.driver.find_element_by_xpath('//div[@id="show-more-results"]')
        #location = down.location
        #self.((JavascriptExecutor) driver).executeScript("window.scrollBy(10000,10000);");
        #next = self.driver.find_element_by_xpath('//div[@id="show-more-results"]')
        response = TextResponse(url=response.url, body=self.driver.page_source, encoding='utf-8')
        try:
            #self.driver.set_page_load_timeout(10000)
            #driver.execute_script("window.scrollTo(0, location.get('y')")
            sites = response.xpath('//div[@class="gd-col gu12 browse-product fk-inf-scroll-item"] | //div[@class="pu-details lastUnit"] |  //div[@class="pu-visual-section"]')
            for site in sites:
                item = FlipkartItem()
                item['title'] = site.xpath('div//a[@class="lu-title"]/text() | div[1]/a/text()').extract()
                item['price'] = site.xpath('div//div[@class="pu-price"]/div/text() | div//div[@class="pu-final"]/span/text()').extract()
                item['rating'] = site.xpath('div[@class="pu-rating"]/div/@title').extract()
                item['image'] = site.xpath('a/img/@src').extract()
                data = site.xpath('a/@href').extract()
                print data
                item['link'] = data

                #print rating
                yield item
            '''for site in sites:
                item = FlipkartItem()
                item['title'] = site.xpath('div//a[@class="lu-title"]/text() | div[1]/a/text()').extract()
                item['price'] = site.xpath('div//div[@class="pu-price"]/div/text() | div//div[@class="pu-final"]/span/text()').extract()
                item['rating'] = site.xpath('div[@class="pu-rating"]/div/@title').extract()
                #print rating
                yield item'''
            #next.click()
            #self.driver.execute_script("window.scrollTo(10000000,10000000)")
        except:
            #break
            a=10
        self.driver.close()