How to web scrape and access <script> wtih bs4 using python
Question
Im trying to scrape the data from " https://www.deadstock.ca/products/adidas-futurepacer-grey-one "
I want to be able to read the variant data which looks like this:
<script>
window.ShopifyAnalytics = window.ShopifyAnalytics || {};
window.ShopifyAnalytics.meta = window.ShopifyAnalytics.meta || {};
window.ShopifyAnalytics.meta.currency = 'CAD';
var meta = {"product":{"id":223724142613,"vendor":"Adidas","type":"Footwear
- QS","variants":[{"id":3063231774741,"price":26000,"name":"adidas
Futurepacer \/ Grey One - 8","public_title":"8","sku":"AQ0907-Grey One-8"},
{"id":3063231807509,"price":26000,"name":"adidas Futurepacer \/ Grey One -
8.5","public_title":"8.5","sku":"AQ0907-Grey One-8.5"},
{"id":3063231840277,"price":26000,"name":"adidas Futurepacer \/ Grey One -
9","public_title":"9","sku":"AQ0907-Grey One-9"},
{"id":3063231873045,"price":26000,"name":"adidas Futurepacer \/ Grey One -
9.5","public_title":"9.5","sku":"AQ0907-Grey One-9.5"},
{"id":3063231905813,"price":26000,"name":"adidas Futurepacer \/ Grey One -
10","public_title":"10","sku":"AQ0907-Grey One-10"},
{"id":3063231938581,"price":26000,"name":"adidas Futurepacer \/ Grey One -
10.5","public_title":"10.5","sku":"AQ0907-Grey One-10.5"},
{"id":3063231971349,"price":26000,"name":"adidas Futurepacer \/ Grey One -
11","public_title":"11","sku":"AQ0907-Grey One-11"},
{"id":3063232004117,"price":26000,"name":"adidas Futurepacer \/ Grey One -
12","public_title":"12","sku":"AQ0907-Grey One-12"},
{"id":3063232036885,"price":26000,"name":"adidas Futurepacer \/ Grey One -
13","public_title":"13","sku":"AQ0907-Grey One-13"}]},"page":
{"pageType":"product","resourceType":"product","resourceId":223724142613}};
for (var attr in meta) {
window.ShopifyAnalytics.meta[attr] = meta[attr];
}
</script>
I don't think im targeting the properly. I want the code to be able to print all the "id": numbers. Here is my code so far, im still new to bs4 but any help would be appreciated. Thanks
import bs4 as bs
import urllib.request
import lxml
link = urllib.request.urlopen ('https://www.deadstock.ca/products/adidas-futurepacer-grey-one').read()
soup = bs.BeautifulSoup(link,'lxml')
for variants in soup.find_all('script'):
print (variants)
I tried something like this ontop of @Andrej Kesely answer.
for id in data['variants']:
size = id['variants']['option1']
variantid = id['variants']['id']
print (size)
print (variantid)
Its coming back with a keyerror though, i just want it to show all the ID
1 answers
solution1
0 ACCPTED 2018-07-16 20:58:59
For this web site you need to target with id="ProductJson-product-template" . It contains JSON with all variants:
import bs4 as bs
import urllib.request
import json
link = urllib.request.urlopen('https://www.deadstock.ca/products/adidas-futurepacer-grey-one').read()
soup = bs.BeautifulSoup(link,'lxml')
variant = soup.find('script', id='ProductJson-product-template')
data = json.loads(variant.text)
print(json.dumps(data, indent=4, sort_keys=True))
Prints:
{
"available": true,
"compare_at_price": null,
"compare_at_price_max": 0,
"compare_at_price_min": 0,
"compare_at_price_varies": false,
"content": "<p>adidas digs into its vault to redesign the 1984 Micropacer, a computerized shoe that was designed to track running statistics. Today, the adidas Futurepacer takes cues from the Micropacer, and the more recent NMD collection, to deliver a futuristic lifestyle runner. Featuring a premium leather upper, complete with a modernized lace cover, the adidas Futurepacer sits on a Boost midsole for lightweight cushioning and responsiveness. NMD inspired bumpers are found at the heel and forefoot for added style and stability.</p>\n<ul>\n<li>Premium leather upper</li>\n<li>Premium leather lace cover</li>\n<li>Subtle adidas branding</li>\n<li>Boost midsole</li>\n<li>adidas midsole plugs</li>\n<li>Grey One</li>\n</ul>\n<p>Product Code: AQ0907</p>",
"created_at": "2018-03-05T14:23:57-08:00",
"description": "<p>adidas digs into its vault to redesign the 1984 Micropacer, a computerized shoe that was designed to track running statistics. Today, the adidas Futurepacer takes cues from the Micropacer, and the more recent NMD collection, to deliver a futuristic lifestyle runner. Featuring a premium leather upper, complete with a modernized lace cover, the adidas Futurepacer sits on a Boost midsole for lightweight cushioning and responsiveness. NMD inspired bumpers are found at the heel and forefoot for added style and stability.</p>\n<ul>\n<li>Premium leather upper</li>\n<li>Premium leather lace cover</li>\n<li>Subtle adidas branding</li>\n<li>Boost midsole</li>\n<li>adidas midsole plugs</li>\n<li>Grey One</li>\n</ul>\n<p>Product Code: AQ0907</p>",
"featured_image": "//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one.jpg?v=1530313621",
"handle": "adidas-futurepacer-grey-one",
"id": 223724142613,
"images": [
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one.jpg?v=1530313621",
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one_1.jpg?v=1531247436",
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one_2.jpg?v=1531247441",
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one_3.jpg?v=1531247444",
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one_4.jpg?v=1531247447",
"//cdn.shopify.com/s/files/1/0616/3517/products/aq0907_adidas_futurepacer_grey_one_5.jpg?v=1531247449"
],
"options": [
"US Size"
],
"price": 26000,
"price_max": 26000,
"price_min": 26000,
"price_varies": false,
"published_at": "2018-07-14T12:00:00-07:00",
"tags": [
"07142018",
"cf-type-footwear-qs",
"cf-us-size-10",
"cf-us-size-10-5",
"cf-us-size-11",
"cf-us-size-12",
"cf-us-size-13",
"cf-us-size-8",
"cf-us-size-8-5",
"cf-us-size-9",
"cf-us-size-9-5",
"cf-vendor-adidas",
"free_shipping",
"limit-quantity",
"plsmerch"
],
"title": "adidas Futurepacer / Grey One",
"type": "Footwear - QS",
"variants": [
{
"available": true,
"barcode": "193050061142",
"compare_at_price": null,
"featured_image": null,
"id": 3063231774741,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 3,
"name": "adidas Futurepacer / Grey One - 8",
"option1": "8",
"option2": null,
"option3": null,
"options": [
"8"
],
"price": 26000,
"public_title": "8",
"requires_shipping": true,
"sku": "AQ0907-Grey One-8",
"taxable": true,
"title": "8",
"weight": 0
},
{
"available": true,
"barcode": "193050061159",
"compare_at_price": null,
"featured_image": null,
"id": 3063231807509,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 2,
"name": "adidas Futurepacer / Grey One - 8.5",
"option1": "8.5",
"option2": null,
"option3": null,
"options": [
"8.5"
],
"price": 26000,
"public_title": "8.5",
"requires_shipping": true,
"sku": "AQ0907-Grey One-8.5",
"taxable": true,
"title": "8.5",
"weight": 0
},
{
"available": true,
"barcode": "193050061166",
"compare_at_price": null,
"featured_image": null,
"id": 3063231840277,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 6,
"name": "adidas Futurepacer / Grey One - 9",
"option1": "9",
"option2": null,
"option3": null,
"options": [
"9"
],
"price": 26000,
"public_title": "9",
"requires_shipping": true,
"sku": "AQ0907-Grey One-9",
"taxable": true,
"title": "9",
"weight": 0
},
{
"available": true,
"barcode": "193050061173",
"compare_at_price": null,
"featured_image": null,
"id": 3063231873045,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 5,
"name": "adidas Futurepacer / Grey One - 9.5",
"option1": "9.5",
"option2": null,
"option3": null,
"options": [
"9.5"
],
"price": 26000,
"public_title": "9.5",
"requires_shipping": true,
"sku": "AQ0907-Grey One-9.5",
"taxable": true,
"title": "9.5",
"weight": 0
},
{
"available": true,
"barcode": "193050061180",
"compare_at_price": null,
"featured_image": null,
"id": 3063231905813,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 6,
"name": "adidas Futurepacer / Grey One - 10",
"option1": "10",
"option2": null,
"option3": null,
"options": [
"10"
],
"price": 26000,
"public_title": "10",
"requires_shipping": true,
"sku": "AQ0907-Grey One-10",
"taxable": true,
"title": "10",
"weight": 0
},
{
"available": true,
"barcode": "193050061197",
"compare_at_price": null,
"featured_image": null,
"id": 3063231938581,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 6,
"name": "adidas Futurepacer / Grey One - 10.5",
"option1": "10.5",
"option2": null,
"option3": null,
"options": [
"10.5"
],
"price": 26000,
"public_title": "10.5",
"requires_shipping": true,
"sku": "AQ0907-Grey One-10.5",
"taxable": true,
"title": "10.5",
"weight": 0
},
{
"available": true,
"barcode": "193050061203",
"compare_at_price": null,
"featured_image": null,
"id": 3063231971349,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 1,
"name": "adidas Futurepacer / Grey One - 11",
"option1": "11",
"option2": null,
"option3": null,
"options": [
"11"
],
"price": 26000,
"public_title": "11",
"requires_shipping": true,
"sku": "AQ0907-Grey One-11",
"taxable": true,
"title": "11",
"weight": 0
},
{
"available": true,
"barcode": "193050061210",
"compare_at_price": null,
"featured_image": null,
"id": 3063232004117,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 4,
"name": "adidas Futurepacer / Grey One - 12",
"option1": "12",
"option2": null,
"option3": null,
"options": [
"12"
],
"price": 26000,
"public_title": "12",
"requires_shipping": true,
"sku": "AQ0907-Grey One-12",
"taxable": true,
"title": "12",
"weight": 0
},
{
"available": true,
"barcode": "193050061227",
"compare_at_price": null,
"featured_image": null,
"id": 3063232036885,
"inventory_management": "shopify",
"inventory_policy": "deny",
"inventory_quantity": 1,
"name": "adidas Futurepacer / Grey One - 13",
"option1": "13",
"option2": null,
"option3": null,
"options": [
"13"
],
"price": 26000,
"public_title": "13",
"requires_shipping": true,
"sku": "AQ0907-Grey One-13",
"taxable": true,
"title": "13",
"weight": 0
}
],
"vendor": "Adidas"
}
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.
Related Question
How to scrape this using bs4
How web scrape with request, Bs4 when there is a script result?
How to Scrape <Script> tag with Beautiful Soup BS4 (Python)
How to fix scrape web table output csv with python and bs4
How to scrape a span from a div using BS4 and Python
How to Scrape Shopify Product Information Using Python BS4
python bs4, how to scrape this text in html?
Python table how to scrape with bs4
how to scrape multiple pages in python with bs4
How to scrape a livetable with BS4 Python
Related Tags