scrapy out of callback function

Question

This is my first scrapy project I need help with returning the output of parse_details and use in the main parse

import scrapy,csv,requests
from scrapy.crawler import CrawlerProcess
from scrapy.selector import Selector
import re

class PythonEventsSpider(scrapy.Spider):
    name = 'hello'
    start_urls=['https://www.amazon.com/s?me=A3JBCFF24SVI66&marketplaceID=ATVPDKIKX0DER']
    details=[]
    def parse(self, response):
        base_url="https://www.amazon.com"
        for row in response.xpath('//div[@class="sg-col-4-of-12 sg-col-8-of-16 sg-col-16-of-24 sg-col-12-of-20 sg-col-24-of-32 sg-col sg-col-28-of-36 sg-col-20-of-28"]/div[@class="sg-col-inner"]'):
            item={}
            Name =row.xpath('div/div/div/div[@class="a-section a-spacing-none"]/h5/a/span/text()').extract_first().replace(",","")
            url=base_url+row.xpath('div/div/div/div[@class="a-section a-spacing-none"]/h5/a/@href').extract_first()
            try:
                asin=re.search('.*dp/(.*)/',url).groups()[0]
                if asin is None:
                    raise AttributeError
            except AttributeError:
                asin=re.search('dp/(.*)',url).groups()[0]
            product_url = "https://www.amazon.com/gp/offer-listing/{}/ref=dp_olp_all_mbc?ie=UTF8&condition=all".format(asin)
            print(product_url)
            yield scrapy.Request(url=product_url,callback=self.parse_details)
            #amazon=??
            #four_prices=???
            item={
            "Name":Name,
            "ASIN":asin,
            "Product URL":product_url,
            #"Amazon":amazon,
            #"Price 1":four_prices[0],
            #"price 2":four_prices[1],
            #"Price 3":four_prices[2],
            #"Price 4":four_prices[3],
            }           
            yield item
    def parse_details(self,response):
        rows=response.xpath('//div[@class="a-row a-spacing-mini olpOffer"]')
        prices=[]
        for row in rows[:4]:
            prices.append(row.xpath('div[@class="a-column a-span2 olpPriceColumn"]/span[1]/text()').extract_first().strip().replace(",","").replace("$",""))
        if "Amazon.com" ==response.xpath('//h3[@class="a-spacing-none olpSellerName"]/img/@alt').extract_first():
            amazon = True
        else:
            amazon=False
        while len(prices)<4:
            prices.append("N/a")
        return prices,amazon

My parse_details function should return 2 values(1 list of length 4 and True or False) which I want to add my item dictionary in parse , I tried replacing

yield scrapy.Request(url=product_url,callback=self.parse_details) with res=scrapy.Request(url=product_url,callback=self.parse_details) to get output of return but doesn't work it simply returns Request object

Answer 1

Try to pass your item in meta from parse to parse_details . Check this example:

def parse(self, response):
    for row in response.xpath('...'):
        # skip some logics here
        item = {
            "Name": Name,
            "ASIN": asin,
            "Product URL": product_url,
        }           
        yield scrapy.Request(product_url, self.parse_details, meta={'item': item})

def parse_details(self, response):
    item = response.meta['item']
    # your logics here
    item['prices'] = ... # your calculations here
    item['amazon'] = ... # your calculations here
    yield item