Scrapy exports everything in the same row

Question

I'm trying to scrape E-commerce with this python file:

import scrapy
from scrapy.item import Field
from scrapy.loader import ItemLoader


class RipleyscraperItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    marca = scrapy.Field()
    descripcion = scrapy.Field()
    precio_normal = scrapy.Field()
    precio_internet = scrapy.Field()
    precio_tarjeta = scrapy.Field()
    vinculo = scrapy.Field()


class RipleySpider(scrapy.Spider):
    name = 'ripley'

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36',
        'FEED_EXPORT_FIELDS': ['marca', 'descripcion', 'precio_normal', 'precio_internet', 'precio_tarjeta', 'vinculo'],
        'CLOSESPIDER_PAGECOUNT': 50
    }


    allowed_domains = ['simple.ripley.cl']
    start_urls = ['https://simple.ripley.cl/otras-categorias/instrumentos-musicales/pianos-y-teclados?source=menu&s=mdco']

    def parse(self, response):
        for products in response.xpath('.//div[@class="catalog-product-item catalog-product-item__container col-xs-6 col-sm-6 col-md-4 col-lg-4"]'):
            item = ItemLoader(RipleyscraperItem(), selector = products)

            item.add_xpath('marca', '//div[@class="catalog-product-details__logo-container"]/div/span/text()')
            item.add_xpath('descripcion', '//div[@class="catalog-product-details__name"]/text()' ) 
            item.add_xpath('precio_normal', '//ul[@class="catalog-prices__list"]/li[@class="catalog-prices__list-price catalog-prices__lowest catalog-prices__line_thru"]/text()')
            item.add_xpath('precio_internet', '//ul[@class="catalog-prices__list"]/li[@class="catalog-prices__offer-price"]/text()')
            item.add_xpath('precio_tarjeta', '//ul[@class="catalog-prices__list"]/li[@class="catalog-prices__card-price"]/text()')
            item.add_xpath('vinculo', '//div[@class="catalog-product-item catalog-product-item__container col-xs-6 col-sm-6 col-md-4 col-lg-4"]/a/@href')

            yield item.load_item()

            next_page = response.xpath('//*[@id="catalog-page"]/div/div[2]/div[4]/nav/ul/li[6]/a/@href')
            if next_page is not None:
                yield response.follow(next_page, callback=self.parse)

  

And export to CSV with this:

scrapy runspider ripley_end.py -o tablaripley.csv -t csv but my csv output is: CSV export

Isn't a project. Is a python file.

I can send more details if you need.

Thanks!!!

Answer 1

You are missing the FEEDS setting in your custom_settings . Define your custom_settings as below and then simply run the script as scrapy runspider ripley_end.py without the -o argument.

custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36',
        'FEEDS': {
            'tablaripley.csv':{
                'format': 'csv',
                'fields': ['marca', 'descripcion', 'precio_normal', 'precio_internet', 'precio_tarjeta', 'vinculo']
            }
        },
        'CLOSESPIDER_PAGECOUNT': 50
    }