在Scrapy中添加了对页面ID的迭代，解析方法中的响应不再运行

Question

我的Spider中有一些用于调试的打印功能。 在start_request函数中，我通过在范围为[0,4]的数字上加上基本URL生成URL，该基本URL由parse_grant函数解析。 parse_grant该函数中，第一个print函数被调用，但第二个不被调用。

仍然在这里学习，所以我可能犯了一个愚蠢的错误，并且不太了解Twisted在后台发生了什么。

# -*- coding: utf-8 -*-

from scrapy.spiders import Spider, Rule
from scrapy.http import Request

from scraper_app.items import NSERCGrant

from scrapy.selector import Selector

class NSERC_Spider(Spider):

name = 'NSERCSpider'
allowed_domains = ["http://www.nserc-crsng.gc.ca"]
# Maximum page id to use.
max_id = 5

def start_requests(self):
    for i in range(self.max_id):
        if i == 0:
            continue
        yield Request("http://www.nserc-crsng.gc.ca/ase-oro/Details-Detailles_eng.asp?id=%d" % i,
                      callback=self.parse_grant)

def parse_grant(self, response):

    print("Being called")

    sel = Selector(response)
    grants = sel.xpath('.//html//body')
    items = []

    for response in grants:

        print("Responses being called")

        item = NSERCGrant()

        # Row one
        item['Competition_Year'] = response.xpath('.//tr[1]//td[2]//text()').extract()
        item['Fiscal_Year'] = response.xpath('.//tr[1]//td[4]//text()').extract()

        # Row two
        item['Project_Lead_Name'] = response.xpath('.//tr[2]//td[2]//text()').extract()
        item['Institution'] = response.xpath('.//tr[2]//td[4]//text()').extract()

        # Row three
        item['Department'] = response.xpath('.//tr[3]//td[2]//text()').extract()
        item['Province'] = response.xpath('.//tr[3]//td[4]//text()').extract()

        # Row four
        item['Award_Amount'] = response.xpath('.//tr[4]//td[2]//text()').extract()
        item['Installment'] = response.xpath('.//tr[4]//td[4]//text()').extract()

        # Row five
        item['Program'] = response.xpath('.//tr[5]//td[2]//text()').extract()
        item['Selection_Committee'] = response.xpath('.//tr[5]//td[4]//text()').extract()

        # Row six
        item['Research_Subject'] = response.xpath('.//tr[6]//td[2]//text()').extract()
        item['Area_of_Application'] = response.xpath('.//tr[6]//td[4]//text()').extract()

        # Row seven
        item['Co_Researchers'] = response.xpath(".//tr[7]//td[2]//text()").extract()
        item['Partners'] = response.xpath('.//tr[7]//td[4]//text()').extract()

        # Award Summary
        item['Award_Summary'] = response.xpath('.//p//text()').extract()

        items.append(item)

    return items

Answer 1

当我尝试

grants = sel.xpath('.//html//body')

从我的刮擦壳，这就是我得到的

In [10]: grants = sel.xpath('.//html//body')

In [11]: grants
Out[11]: []

当我将其更改为以下代码时，

In [12]: grants = sel.xpath('/html/body')

In [13]: grants
Out[13]: [<Selector xpath='/html/body' data=u'<body>\r\n<div id="cn-body-inner-1col">\r\n<'>]

Answer 2

您要查找的信息在每个页面上仅出现一次，并且body标签在每个页面上都出现，因此循环和一行

grants = sel.xpath('.//html//body')

是多余的。 另外， response.xpath('... your xpath here ...')保存了一些代码。 尝试这个

# -*- coding: utf-8 -*-

from scrapy.spiders import Spider
from scrapy.http import Request

from scraper_app.items import NSERCGrant

class NSERC_Spider(Spider):

    name = 'NSERCSpider'
    allowed_domains = ["http://www.nserc-crsng.gc.ca"]
    # Maximum page id to use.
    max_id = 5

    def start_requests(self):
        for i in range(1, self.max_id):
            yield Request("http://www.nserc-crsng.gc.ca/ase-oro/Details-Detailles_eng.asp?id=%d" % i,
                          callback=self.parse_grant)

    def parse_grant(self, response):

        print("Being called")

        item = NSERCGrant()

        # Row one
        item['Competition_Year'] = response.xpath('//tr[1]//td[2]//text()').extract()
        item['Fiscal_Year'] = response.xpath('//tr[1]//td[4]//text()').extract()

        # Row two
        item['Project_Lead_Name'] = response.xpath('//tr[2]//td[2]//text()').extract()
        item['Institution'] = response.xpath('//tr[2]//td[4]//text()').extract()

        # Row three
        item['Department'] = response.xpath('//tr[3]//td[2]//text()').extract()
        item['Province'] = response.xpath('//tr[3]//td[4]//text()').extract()

        # Row four
        item['Award_Amount'] = response.xpath('//tr[4]//td[2]//text()').extract()
        item['Installment'] = response.xpath('//tr[4]//td[4]//text()').extract()

        # Row five
        item['Program'] = response.xpath('//tr[5]//td[2]//text()').extract()
        item['Selection_Committee'] = response.xpath('//tr[5]//td[4]//text()').extract()

        # Row six
        item['Research_Subject'] = response.xpath('//tr[6]//td[2]//text()').extract()
        item['Area_of_Application'] = response.xpath('//tr[6]//td[4]//text()').extract()

        # Row seven
        item['Co_Researchers'] = response.xpath("//tr[7]//td[2]//text()").extract()
        item['Partners'] = response.xpath('//tr[7]//td[4]//text()').extract()

        # Award Summary
        item['Award_Summary'] = response.xpath('//p//text()').extract()

        yield item

我还调整了start_request例程，以删除if i = 0 。

看一下scrapy shell ，它使您可以尝试xpath并以交互方式查看结果。