Scrapy等待function完成

Question

Is there a way to wait for the loop to finish all the scrapy.Request and then do something? In this case i'd like to yield the payload after the for loop since i don't want to yield the payload in each pagination (parse_stores), i want it only on the parse function after the for.

import scrapy
import json
import math


class UnitsSpider(scrapy.Spider):
    """ docstrings """
    name = 'units'

    def start_requests(self):

        urls = [{
            "url":
            "https://url.com/"
        }, {
            "url":
            "https://url2.com"
        }]

        for url in urls:
            yield scrapy.Request(
                url['url'],
                meta={
                    'playwright': True,
                }
            )

    def parse(self, response):

        url = response.url

        data = response.css('script#__NEXT_DATA__::text').get()
        json_data = json.loads(data)

        total_pages = math.ceil(
            json_data['props']['pageProps']['totalStores'] / 50)

        payload = {
            'base_url': url,
            'stores': 0,
            'stores_data': []
        }

        for page in range(0, total_pages):
            next_page = f'{url}{page + 1}'

            req = response.follow(url=next_page)

            yield scrapy.Request(url=req.url, callback=self.parse_stores, meta={
                'playwright': True}, cb_kwargs={'payload': payload})

        # here after all the requests are done i'd like to do something    

    def parse_stores(self, response, payload):
        data = response.css('script#__NEXT_DATA__::text').get()
        json_data = json.loads(data)

        stores = json_data['props']['pageProps']['cityData']['stores']

        payload['stores'] += len(stores)
        # append stores to stores_data
        payload['url'] = response.url
        yield payload

Answer 1

If I understood correct, yes you can.

The scraper will close and you want to yield the results then instead of yielding after every iteration.

in your pipelines.py you need something like this

    def close_spider(self, spider):
        yield {"data": self.data}

make sure you defined the item_pipeline in your settings to make it work.

More information: https://docs.scrapy.org/en/latest/topics/item-pipeline.html