使用硒从URL列表中抓取并抓取

Question

我是scrapy和Python的新手。 我正在尝试从使用Selenium的URL列表中抓取并抓取。 我尝试了这段代码：

class TechstarSpider(scrapy.Spider):

    name = "techstar"
    allowed_domains = ["techstar.com"]
    start_urls = [l.strip() for l in open('pages.txt').readlines()]

    def __init__(self, **kwargs):
        super(TechstarSpider, self).__init__(**kwargs)
        self.driver = webdriver.Firefox()

    def parse(self, response):
        self.driver.get(response.url)
        time.sleep(10)
        resp = TextResponse(url=self.driver.current_url, body=self.driver.page_source, encoding='utf-8')
        item = StartupweekendataItem()
        for data in resp.xpath('//*[@id="main"]/div[2]'):
            item['Title'] = data.xpath('//*[@id="main"]/div[2]/div[1]/div/div/ol/h3/text()')[0].extract(),
            item['Date'] = data.xpath('//*[@id="main"]/div[2]/div[1]/div/div/ol/h3/span/text()')[0].extract(),
            item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()')[0].extract(),
            item['Judge1FB'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-facebook font-18"]/@href')[0].extract(),
            item['Judge1Linked'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Judge2'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][2]/div[2]/h2/text()')[0].extract(),
            item['Judge2FB'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][2]/div[2]/h2/a[@class="fa fa-facebook font-18"]/@href')[0].extract(),
            item['Judge2Linked'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/a[@class="fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Coach1'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//h2//text()')[0].extract(),
            item['Coach1FB'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//a[@class="fa fa-facebook font-18"]//@href')[0].extract(),
            item['Coach1Linked'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][1]//a[@class="fa fa-linkedin font-18"]//@href')[0].extract(),
            item['Coach2'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//h2//text()')[0].extract(),
            item['Coach2FB'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//a[@class="fa fa-facebook font-18"]//@href')[0].extract(),
            item['Coach2Linked'] = data.xpath('//*[@id="event-coaches"]/following-sibling::div[@class="row"][2]//a[@class="fa fa-linkedin font-18"]//@href')[0].extract(),
            item['Organizer1'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/h5/text()')[0].extract(),
            item['Organizer1FB'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/a["fa fa-facebook font-18"]/@href')[0].extract(),
            item['Organizer1Linked'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[1]/a["fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Organizer2'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/h5/text()')[0].extract(),
            item['Organizer2FB'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/a["fa fa-facebook font-18"]/@href')[0].extract(),
            item['Organizer2Linked'] = data.xpath('//*[@id="event-organizers"]/following-sibling::div[2]/div[@class="large-15 columns"]/ul/li[2]/a["fa fa-linkedin font-18"]/@href')[0].extract(),
            item['Facilitator'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//h2//text()")[0].extract(),
            item['FacilitatorFB'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//a['fa fa-facebook font-18']//@href")[0].extract(),
            item['FacilitatorLinked'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[1]//a['fa fa-linkedin font-18']//@href")[0].extract(),
            item['Staff1'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//h2//text()")[0].extract(),
            item['Staff1FB'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//a[@class'fa fa-facebook font-18']//@href")[0].extract(),
            item['Staff1Linked'] = data.xpath("//h2[contains(string(), 'Facilitators')]/../../following-sibling::div[3]//a[@class='fa fa-linkedin font-18']//@href")[0].extract(),
            yield item

Pages.txt包含如下网址列表：

http://communities.techstars.com/spain/madrid/startup-weekend/11425  
http://communities.techstars.com/usa/phlsw/startup-weekend/3334  
http://communities.techstars.com/bolivia/santacruz/startup-weekend/10815

但是它抛出了这个错误：

2017-11-12 12:38:26 [scrapy] ERROR: Spider error processing <GET http://communities.techstars.com/bolivia/santacruz/startup-weekend/10815> (referer: None)
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/scrapy/utils/defer.py", line 102, in iter_errback
    yield next(it)
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/offsite.py", line 29, in process_spider_output
    for x in result:
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/referer.py", line 22, in <genexpr>
    return (_set_referer(r) for r in result or ())
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/urllength.py", line 37, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/usr/lib/python2.7/site-packages/scrapy/spidermiddlewares/depth.py", line 58, in <genexpr>
    return (r for r in result or () if _filter(r))
  File "/home/habenn/Projects/Aleksandr/startupweekendata/startupweekendata/spiders/techstar.py", line 53, in parse
    item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()')[0].extract(),
  File "/usr/lib/python2.7/site-packages/parsel/selector.py", line 61, in __getitem__
    o = super(SelectorList, self).__getitem__(pos)
IndexError: list index out of range

这是我尝试达到的输出是：

问题 ：抛出错误，我该如何解决？ 和代码太长，您能帮我简化代码吗？ 任何帮助将是真正有帮助的。

Answer 1

问题是使用[0]访问选择器列表的第一个元素，更详细地访问选择器的结果//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text() （请参阅错误消息第四行，从底部开始）。

这是您选择Judge1的选择器。

查看列表中第一页的源代码，我找不到具有id="event-judges"的元素，因此该元素不可用。

如果您希望能够选择缺少一个或多个属性的项目（例如Judge1 ），则建议您使用extract_first()方法代替[0].extract() 。

如果您的选择器不匹配任何元素，或者您可以使用extract_first(default='no-judge')定义另一个默认值， extract_first()将返回None 。

对于Judge1选择器，可以这样使用它：

item['Judge1'] = data.xpath('//*[@id="event-judges"]/following-sibling::div[@class="row"][1]/div[2]/h2/text()').extract_first(),