XPATH Selector not able to select the block of html code

Question

I am trying to extract some data from alibaba.com. For that, I am using scrapy. While it was working for most parts, the selector does not seem to grab the code block from Company profile. Can anyone help me with this problem?

# -*- coding: utf-8 -*-
import scrapy
import csv
import os
import numpy as np

class AlibabaCrawlerSpider(scrapy.Spider):
    name = 'alibaba_crawler'
    allowed_domains = ['alibaba.com']
    start_urls = ['http://alibaba.com/']
    delimiter = '|'

    def start_requests(self):
        """Read keywords from keywords file amd construct the search URL"""

        with open(os.path.join(os.path.dirname(__file__), "../resources/keywords.csv")) as search_keywords:
            for keyword in csv.DictReader(search_keywords):
                search_text=keyword["keyword"]
                url="https://www.alibaba.com/trade/search?fsb=y&IndexArea=product_en&CatId=&SearchText={0}&viewtype=G".format(
                    search_text)
                # The meta is used to send our search text into the parser as metadata
                yield scrapy.Request(url, callback = self.parse, meta = {"search_text": search_text})


    def parse(self, response):
        """Function to process alibaba search results page"""
        search_keyword=response.meta["search_text"]
        products=response.xpath("//div[@class='item-main']")

        # Defining the XPaths

        XPATH_PRODUCT_LINK=".//div[@class='item-info']//h2/a/@href"

        # iterating over search results
        for product in products:

            raw_product_link=product.xpath(XPATH_PRODUCT_LINK).extract()

            print(raw_product_link)

            product_link="https:" + raw_product_link[0] if raw_product_link else None

            yield scrapy.Request(product_link, callback=self.parse_product)

            break

    def parse_product(self, response):

        product=response.xpath("//div[@class='content-body']")

        # Defining the XPaths

        XPATH_COMPANY_FIELD=".//div[@class='tab-body']//div[contains(@class,'ls-company')]"#//div[@class='alisite']"#td[@class='field-title']/text()"

        raw_company_field=product.xpath(XPATH_COMPANY_FIELD) #.extract()

        print(raw_company_field)

I am trying to print raw_company_field. It works upto this point. But it gives empty list when I go to levels below eg alisite and beyond. enter image description here

Answer 1

XPath doesn't check for classes that way.

A selector like //div[@class='tab-body'] will match only have tab-body as its only class. To select elements that have a class among others, you do something like this:

//div[contains(concat(' ',normalize-space(@class),' '),' tab-body ')]

or use css selectors instead:

div.tag-body