Scrapy：存储损坏的外部链接并丢弃其余链接

Question

I want Scrapy to store only the external links that are broken (a response code different from 200, 301 or 302) but I'm stuck with this and the script keeps storing every external link on the output file. This is what I'm using:

@staticmethod
def remote_file_to_array(url):

    return filter(None, urllib2.urlopen(url).read().splitlines())

@staticmethod
def sitemap_to_array(url):
    results = []
    body = urllib2.urlopen(url).read()
    sitemap = Sitemap(body)
    for item in sitemap:
        results.append(item['loc'])
    return results


def start_requests(self):


    target_domain = self.arg_target_domain
    print 'Target domain: ', target_domain


    self.rules = (

        Rule(LinkExtractor(allow_domains=[target_domain], unique=True),
             follow=True),

        Rule(LinkExtractor(unique=True),
             callback='parse_item',
             process_links='clean_links',
             follow=False),
    )
    self._compile_rules()


    start_urls = []
    if self.arg_start_urls.endswith('.xml'):
        print 'Sitemap detected!'
        start_urls = self.sitemap_to_array(self.arg_start_urls)
    elif self.arg_start_urls.endswith('.txt'):
        print 'Remote url list detected!'
        start_urls = self.remote_file_to_array(self.arg_start_urls)
    else: 
        start_urls = [self.arg_start_urls]
    print 'Start url count: ', len(start_urls)
    first_url = start_urls[0]
    print 'First url: ', first_url


    for url in start_urls:


        yield scrapy.Request(url, dont_filter=True)


def clean_links(self, links):
    for link in links:

        link.fragment = ''
        link.url = link.url.split('
        yield link


def parse_item(self, response):
    item = BrokenLinksItem()
    item['url'] = response.url
    item['status'] = response.status
    yield item

Answer 1

You need to pass the errback argument on the Request object, that works like the callback but for not accepted response statuses.

I am not sure if that can be also achieved with rules , if not you'll need to define your own behaviour

Answer 2

Your best bet would be to use a Downloader Middleware to log the desired responses.

from twisted.internet import defer
from twisted.internet.error import (ConnectError, ConnectionDone, ConnectionLost, ConnectionRefusedError,
                                    DNSLookupError, TCPTimedOutError, TimeoutError,)

class BrokenLinkMiddleware(object):

    ignore_http_status_codes = [200, 301, 302]
    exceptions_to_log = (ConnectError, ConnectionDone, ConnectionLost, ConnectionRefusedError, DNSLookupError, IOError,
                         ResponseFailed, TCPTimedOutError, TimeoutError, defer.TimeoutError)

    def process_response(self, request, response, spider):
        if response.status not in self.ignore_http_status_codes:
            # Do your logging here, response.url will have the url, 
            # response.status will have the status.
        return response

    def process_exception(self, request, exception, spider):
        if isinstance(exception, self.exceptions_to_log):
            # Do your logging here

That handles some exceptions which may not indicate a broken link (like ConnectError , TimeoutError , and TCPTimedOutError ), but you may want to log them anyway.