使用scrapy将变量传递到Spider文件夹中的test.py

Question

I'm using Scrapy. The following is the code for test.py in spider folder.

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from craigslist_sample.items import CraigslistSampleItem

class MySpider(BaseSpider):
    name = "craig"
    allowed_domains = ["craigslist.org"]
    start_urls = ["http://seattle.craigslist.org/npo/"]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        titles = hxs.select("//span[@class='pl']")
        items = []
        for titles in titles:
            item = CraigslistSampleItem()
            item["title"] = titles.select("a/text()").extract()
            item["link"] = titles.select("a/@href").extract()
            items.append(item)
        return items

Essentially, I want to iterate my url list and pass url into MySpider class for start_ulrs . Could you anyone give me suggestion on how to make this?

Answer 1

Instead of having "statically defined" start_urls you need to override start_requests() method:

from scrapy.http import Request

class MySpider(BaseSpider):
    name = "craig"
    allowed_domains = ["craigslist.org"]

    def start_requests(self)
        list_of_urls = [...]  # reading urls from a text file, for example
        for url in list_of_urls:
            yield Request(url)

    def parse(self, response):
        ...