爬蟲未爬取第二個 start_url

Question

我正在嘗試從hindustantimes.com 中抓取電影評論和電視新聞。 當我運行此代碼時，它只會抓取第一個 start_url，但無法抓取第二個 start_url。 我認為必須重置計數器，但我不知道如何重置。 我想從 start_url 中抓取 n 個頁面。

import scrapy
#test_push
from..items import HindustantimesItem

class HindustantimesSpider(scrapy.Spider):
    name = 'Hindustantimes_review'
    page_number = 2
    count = 0

    def start_requests(self): 

        urls = ['https://www.hindustantimes.com/movie-reviews/page/?pageno={}',
        'https://www.hindustantimes.com/tv/page/?pageno={}',
        ]
        ur = []
        for url in urls:
            for i in range(1,3):
                x = url.format(i)
                yield scrapy.Request(url=x, callback=self.parse,)

    def parse(self, response):
        print("-------^^^^^^---------")
        print(response.request.url)
        items = {}


        with open('output.txt', 'a') as the_file:
            the_file.write(response.request.url)
            the_file.write( "\n")

        title_xpath = ['//*[@id="scroll-container"]/ul/li[{}]/div/div[2]/div/a/text()', '/html/body/div[1]/section/div[2]/div/div[1]/div[2]/ul/li[{}]/div/div[2]/div[1]/a/text()']
        page_review_xpath = ['//*[@id="scroll-container"]/ul/li[{}]/div/div[2]/p/text()','/html/body/div[1]/section/div[2]/div/div[1]/div[2]/ul/li[{}]/div/div[2]/div[2]/text()']
        page_link_xpath = ['//*[@id="scroll-container"]/ul/li[{}]/div/div[2]/div/a/@href', '/html/body/div[1]/section/div[2]/div/div[1]/div[2]/ul/li[{}]/div/div[2]/div[1]/a/@href']
        if HindustantimesSpider.count ==0:
            current_title_xpath = title_xpath[0]
            current_review_xpath = page_review_xpath[0]
            current_link_xpath = page_link_xpath[0]
            HindustantimesSpider.count+=1
        else:
            current_title_xpath = title_xpath[1]
            current_review_xpath = page_review_xpath[1]
            current_link_xpath = page_link_xpath[1]

        count = response.xpath(current_title_xpath.format("*")).getall()
        count = len(count)
        i = 1
        while i<=count:
            outputs = HindustantimesItem()
            outputs['page_title'] = response.xpath(current_title_xpath.format(i)).get()
            outputs['review_content'] = response.xpath(current_review_xpath.format(i)).get()
            outputs['review_link'] = response.xpath(current_link_xpath.format(i)).get()
            i+=1
            fl = 0
            if outputs['page_title'] == []:
                outputs['page_title'] = ''
                fl+=1
            if outputs['review_content'] == []:
                outputs['review_content'] = ''
                fl+=1
            if outputs['review_link'] == []:
                outputs['review_link'] = ''
                fl += 1
            else:
                if 'Review:' in outputs['page_title'].split(" ") or 'review:' in outputs['page_title'].split(" "):
                    yield outputs

        pass

Answer 1

def start_requests(self): 

        urls = ['https://www.hindustantimes.com/movie-reviews/page/?pageno={}',
        'https://www.hindustantimes.com/tv/page/?pageno={}',
        ]
        ur = []
        for url in urls:
            for i in range(1,3):
                x = url.format(i)
                yield scrapy.Request(url=x, callback=self.parse,dont_filter=True)