Python3，Scrape 返回 []

Question

I have a python script for the scraping site, I need to scrape a few info(name, price, links, ID) this info will be put in MongoDB. But I have a problem with my scraping function. It returns to me a blank list.

Can you help me please with this? Sorry for my English and thank you in advance

from bs4 import BeautifulSoup
import requests
import time
import pymongo
import difflib
import functools

URL = 'https://www.nickollsandperks.co.uk/New-and-Special-Offers/New-Whisky?order=relevance:asc'
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/39.0.2171.95 Safari/537.36'}


def newScrapItems():
    # global productNames,Names, links, Price, products_info_final
    content3 = requests.get(URL, headers=headers)
    soup3 = BeautifulSoup(content3.text, 'html.parser')
    itemList = []


    for products in soup3.find_all("div", {"class": 'facets-facet-browse-items'}):
        for products_info in products.find_all("div", {"class": "facets-items-collection-view-row"}):
            for products_info_final in products_info.find_all("div", {"class": "facets-item-cell-list"}):
                for generalInfo in products_info_final.find_all("div", {"class": "facets-item-cell-list-right"}):
                    for links in generalInfo.find_all("meta"):
                        for itemNamesNext in products_info.find_all("div", {"class": "item-title-description"}):
                            for prePrice in generalInfo.find_all("div", {"class": "item-button"}):
                                for Names in itemNamesNext.find_all("span"):
                                    for priceInfo in prePrice.find_all("div", {"class": "ProductViewsPrice.Price"}):
                                        for Price in priceInfo.find_all("span", {"class": "product-views-price-lead"}):
                                            productNames = {}

                                            productNames['price'] = Price.get_text()
                                            productNames['name'] = Names.get_text()
                                            productNames['link'] = links['content']
                                            productNames['ID'] = products_info_final['data-item-id']
                                            itemList.append(productNames)


    return itemList
newItems = newScrapItems()
print(newScrapItems())

Its returned:

[]

Process finished with exit code 0

I try to search for this trouble but this is didn't give me results. I really hope that some kind person will be able to help me with this problem since I have been struggling with this for a couple of days.

Answer 1

I realized it like a string in another site, but I had less nesting:

content3 = requests.get(URL, headers=headers)
    soup3 = BeautifulSoup(content3.text, 'html.parser')
    newItemList = []
    for products in soup3.find_all("li", {"class": 'product-item'}):
        for products_info in products.find_all("strong", {"class": "product-item-name"}):
            for name in products_info.find_all("a"):
                for productsPriceInfo in products.find_all("div", {"class": "price-box price-final_price"}):
                    for productsPriceInfoAdv in productsPriceInfo.find_all("span", {
                        "class": "price-wrapper price-including-tax"}):
                        for finalPrice in productsPriceInfoAdv.find_all("span", {"class": "price"}):
                            productNames = {}
                    productNames['name'] = name['title']
                    productNames['price'] = finalPrice.get_text()
                    productNames['link'] = name['href']
                    productNames['ID'] = productsPriceInfo['data-product-id']
                    # dict = {'names': name['title']}
                    newItemList.append(productNames)

    return newItemList

this returned string with format - name: "name", link:"link" etc.

Answer 2

Re-write to use a single loop of a parent element collection. Choose a parent that encompasses all the data you want as blocks; 1 block per product; then, select relationally from that block within the loop to get the items within each block.

I have assumed id is product SKU.

import requests
import numpy as np
from bs4 import BeautifulSoup as bs

newItemList = []
base = 'https://www.nickollsandperks.co.uk/'

r = requests.get('https://www.nickollsandperks.co.uk/New-and-Special-Offers/New-Whisky?order=relevance:asc')
soup = bs(r.content, 'lxml')

for listing in soup.select('.facets-items-collection-view-cell-span12'):
    price = listing.select_one('.product-views-price') #price
    if not price is None: #not every listing has a price
        price = price.text
    else:
        price = np.NaN
        
    newItemList.append(
       {'name': listing.select_one('.facets-item-cell-list-name span').text,
        'link': base + listing.select_one('.facets-item-cell-list-name')['href'],
        'price': price, 
        'id': listing.select_one('.facets-item-cell-list')['data-sku']
       }
    )