使用scrapy将变量传递到Spider文件夹中的test.py

Question

我正在使用Scrapy。 以下是Spider文件夹中test.py的代码。

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from craigslist_sample.items import CraigslistSampleItem

class MySpider(BaseSpider):
    name = "craig"
    allowed_domains = ["craigslist.org"]
    start_urls = ["http://seattle.craigslist.org/npo/"]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        titles = hxs.select("//span[@class='pl']")
        items = []
        for titles in titles:
            item = CraigslistSampleItem()
            item["title"] = titles.select("a/text()").extract()
            item["link"] = titles.select("a/@href").extract()
            items.append(item)
        return items

从本质上讲，我想重复我的网址列表，并通过链接进入MySpider类start_ulrs 。 有人可以给我建议如何做吗？

Answer 1

无需“静态定义” start_urls您需要重写start_requests()方法：

from scrapy.http import Request

class MySpider(BaseSpider):
    name = "craig"
    allowed_domains = ["craigslist.org"]

    def start_requests(self)
        list_of_urls = [...]  # reading urls from a text file, for example
        for url in list_of_urls:
            yield Request(url)

    def parse(self, response):
        ...