Issue with findall over a string (expected string or byte-like object)

Question

Previous question I did about this topic got put on hold as off topic, did the modifications but it's still holded, don't know why. Here is what i asked:

I am currently working on an amazon scraper and I needed to get the images from a product, for example:

https://www.amazon.com/gp/product/B0711BMXVB?pf_rd_p=1581d9f4-062f-453c-b69e-0f3e00ba2652&pf_rd_r=X7FDBW1DN25C8PM5A01C

What I did was, using the xpath

//script[contains(., "ImageBlockATF")]/text()

Get a bunch of text that, inside, contains all the urls for the 'large' images

Basically this:

P.when('A').register("ImageBlockATF", function(A){
var data = {
            'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
            'colorToAsin': {'initial': {}},
            'holderRatio': 1.2,
            'holderMaxHeight': 700,
            'heroImage': {'initial': []},
            'heroVideo': {'initial': []},
            'spin360ColorData': {'initial': {}},
            'spin360ColorEnabled': {'initial': 0},
            'spin360ConfigEnabled': false,
            'spin360LazyLoadEnabled': false,
            'playVideoInImmersiveView':'false',
            'tabbedImmersiveViewTreatment':'C',
            'totalVideoCount':'0',
            'videoIngressATFSlateThumbURL':'',
            'mediaTypeCount':'0',
            'atfEnhancedHoverOverlay' : true,
            'winningAsin': 'B072596K2C',
            'weblabs' : {},
            'aibExp3Layout' : 1,
            'aibRuleName' : 'frank-powered',
            'acEnabled' : false
            };
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});

That I named imagesString .

I get this string by doing, with scrapy:

imagesString = (response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first())

When you search the previous xpath in the example URL, 2 'blocks' of text pop up. With the extract_first() you get the first one extracted, which is the long string above.

Then I had to obtain the urls so I did this:

images = re.findall('\"large\":\"(https.*?\.jpg)\"', imagesString)

Which gave me a list of all the large images urls.

The issue I'm having is that, at one point of the program running, i get this error:

    Traceback (most recent call last):
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
      File "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 52, in parse
    imagenes = re.findall('\"large\":\"(https.*?\.jpg)\"', imagenesString)
     File "C:\Users\Manuel\Anaconda3\lib\re.py", line 223, in findall
        return _compile(pattern, flags).findall(string)
    TypeError: expected string or bytes-like object

I honestly have no idea what's going on. The thing I can see is that this error never occurs at the start of the process. If I have to do this for 30 products it works fine but when I start to get more products this happens.

By using Json approach with @Maurice Mayer help

s = response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first()
m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
jsonObj = json.loads(data[:-1].replace("'", '"'))

I'm getting this error

File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
  File     "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 59, in parse
    data = m.groups()[0]
AttributeError: 'NoneType' object has no attribute 'groups'

EDIT: Added user suggestion and new error

EDIT2: Added Json tag

Answer 1

The javascript variable data is a JSON object, it might be easier to treat as such and you can iterate quickly over the object:

import json
import re


s = """P.when('A').register("ImageBlockATF", function(A){
var data = {
            'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
            'colorToAsin': {'initial': {}},
            'holderRatio': 1.2,
            'holderMaxHeight': 700,
            'heroImage': {'initial': []},
            'heroVideo': {'initial': []},
            'spin360ColorData': {'initial': {}},
            'spin360ColorEnabled': {'initial': 0},
            'spin360ConfigEnabled': false,
            'spin360LazyLoadEnabled': false,
            'playVideoInImmersiveView':'false',
            'tabbedImmersiveViewTreatment':'C',
            'totalVideoCount':'0',
            'videoIngressATFSlateThumbURL':'',
            'mediaTypeCount':'0',
            'atfEnhancedHoverOverlay' : true,
            'winningAsin': 'B072596K2C',
            'weblabs' : {},
            'aibExp3Layout' : 1,
            'aibRuleName' : 'frank-powered',
            'acEnabled' : false
            };
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});"""

m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
jsonObj = json.loads(data[:-1].replace("'", '"'))  # remove the last semicolon and replace single quotes!

for img in jsonObj['colorImages']['initial']:
    print (img['large'])

Prints:

https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg
https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg
https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg
https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg
https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg
https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg
https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg