Scrapy 拒绝规则语法

Question

I'm a complete newby to scrapy and python, however, my project and my knowledge is making good progress thanks to the awesome people here, To finish my spider, I just need to configure some URL parts (for example, alls URL containing bottom .htm ，操作栏或诸如？ 但是我认为，我在使用正则表达式语法时遇到了困难，因此爬虫至少在页面上运行，但似乎没有过滤。 有人在这里解释我做错了什么吗？

这是蜘蛛：

import scrapy

from scrapy.loader import ItemLoader

from ..items import NorisbankItem

from scrapy.linkextractors import LinkExtractor

from scrapy.spiders import CrawlSpider, Rule

class NorisbankSpider(CrawlSpider):

    name = "nbtest"

    allowed_domains = ['norisbank.de']

    start_urls = ['https://www.norisbank.de']

    custom_settings={ 'FEED_URI': "norisbank_%(time)s.json",

                      'FEED_FORMAT': 'json',

                      }

    rules = (

        Rule(

            LinkExtractor(allow=(''),

                          deny=('\*start\.do\?*',

                                '\*WT\.mc_id*',

                                '\*.js',

                                '\*.ico',

                                '\*_frame\.htm*',

                                '\*actionbar*',

                                '\*actionframe*',

                                '\*bottom\.htm*',

                                '\*navbar_m\.html',

                                '\*top\.htm*',

                                '\*expandsection*\.*',

                                '\*\?*\?*',

                                '\*\.xml',

                                '\*kid=*',

                                '\*\/dienste\/*',

                                '\*\.do',

                                '\*\.db',

                                '\*redirect',

                                '\*.html\?pi_*',

                          ),

            ),

           callback='parse_item',

           follow=True

           ),

         )

 

    def parse(self, response):

        page = response.url.split("/")[-2]

        filename = 'nbtest-%s.html' % page

        with open(filename, 'wb') as f:

            f.write(response.body)

        self.log('Saved file %s' % filename)

        #Content Extraction

        print(response.url)

        l = ItemLoader(NorisbankItem(), response=response)

        l.add_xpath('sitename', "//meta[@property='og:site_name']/@content")

        l.add_xpath('siteurl', "//link[@rel='canonical']/@href")

        l.add_xpath('dbCategory',"//meta[@name='dbCategory']/@content")

        l.add_css('title','title::text')

        l.add_xpath('descriptions',"normalize-space(//meta[@name='description']/@content)")

        l.add_xpath('date',"//meta[@name='date']/@content")

        l.add_xpath('version',"//meta[@name='v']/@content")

        l.add_xpath('time',"//meta[@name='time']/@content")

        l.add_xpath('sitecontent','//body//p//text()')

        yield l.load_item()

        all_pages = response.xpath('//a[contains(@href, "html")]/@href').getall()

        for next_page in all_pages :

            next_page = response.urljoin(next_page)

            yield scrapy.Request(next_page, callback=self.parse)

Answer 1

基本上，我希望使用这些规则，但是语法似乎不起作用：

规则 = (

    Rule(
        LinkExtractor(allow=(''),
                      deny=('\*start\.do\?*',
                            '\*WT\.mc_id*',
                            '\*.js',
                            '\*.ico',
                            '\*_frame\.htm*',
                            '\*actionbar*',
                            '\*actionframe*',
                            '\*bottom\.htm*',
                            '\*navbar_m\.html',
                            '\*top\.htm*',
                            '\*expandsection*\.*',
                            '\*\?*\?*',
                            '\*\.xml',
                            '\*kid=*',
                            '\*\/dienste\/*',
                            '\*\.do',
                            '\*\.db',
                            '\*redirect',
                            '\*.html\?pi_*',
                      ),
        ),
       callback='parse_item',
       follow=True
        ),
     )