Scrape pictures from JavaScript rendered webpage

Question

I am trying to scrape the pictures out of a webpage. It is rendered using JS and the picture links in the source code are not complete. This is the source where the pictures are:

<script language="javascript" type="text/javascript">
</script>
<div id="ImagesSection" class="ImagesSection">
<div id='HybridImageViewPrimaryImageDiv'>
<a href='/ItemImages/000450/18190933_1_lg.jpeg'  class="MagicZoom" data-options="  zoomMode:off; cssClass: dark-bg; zoomOn: click"  title='Multi-Faced Doll By Cark Bergner.' id="xxxyyyzzz"     ><img id='fullimage' src='/ItemImages/000450/18190933_1_med.jpeg'  alt='Multi-Faced Doll By Cark Bergner.' /></a>
</div>
<div style="margin-top:15px;width:300px;"> <button class="cfg-btn" onclick="MagicZoom.prev('xxxyyyzzz');return false;">Prev</button> <button class="cfg-btn" onclick="MagicZoom.next('xxxyyyzzz') ;return false;">Next</button>
</div><div style="margin-top:15px;" width="350px" >
 <a data-zoom-id="xxxyyyzzz" href="/ItemImages/000450/18190933_1_lg.jpeg"    data-image="/ItemImages/000450/18190933_1_med.jpeg"       >  <img    src="/ItemImages/000450/18190933_1_sm.jpeg"  height="60px"   />  </a>   
 <a data-zoom-id="xxxyyyzzz" href="/ItemImages/000450/18190933_2_lg.jpeg"    data-image="/ItemImages/000450/18190933_2_med.jpeg"       >  <img    src="/ItemImages/000450/18190933_2_sm.jpeg"  height="60px"   />  </a>   
 <a data-zoom-id="xxxyyyzzz" href="/ItemImages/000450/18190933_3_lg.jpeg"    data-image="/ItemImages/000450/18190933_3_med.jpeg"       >  <img    src="/ItemImages/000450/18190933_3_sm.jpeg"  height="60px"   />  </a>   
 <a data-zoom-id="xxxyyyzzz" href="/ItemImages/000450/18190933_4_lg.jpeg"    data-image="/ItemImages/000450/18190933_4_med.jpeg"       >  <img    src="/ItemImages/000450/18190933_4_sm.jpeg"  height="60px"   />  </a>   
 <a data-zoom-id="xxxyyyzzz" href="/ItemImages/000450/18190933_5_lg.jpeg"    data-image="/ItemImages/000450/18190933_5_med.jpeg"       >  <img    src="/ItemImages/000450/18190933_5_sm.jpeg"  height="60px"   />  </a>   
</div>
</div>

All I want to extract are the following images:

/ItemImages/000450/18190933_1_sm.jpeg
/ItemImages/000450/18190933_2_sm.jpeg
/ItemImages/000450/18190933_3_sm.jpeg
/ItemImages/000450/18190933_4_sm.jpeg
/ItemImages/000450/18190933_5_sm.jpeg

And this is my code:

import os
import shutil
import time
import requests
from bs4 import BeautifulSoup as bSoup
from selenium import webdriver

url = "https://auctions.morphyauctions.com/French_Fashion_Doll_with_Unusual_Body_-LOT450029.aspx"

driver = webdriver.Chrome(executable_path="/mypath/")

driver.get(url)

iterations = 0
while iterations <10:
    html = driver.execute_script("return document.documentElement.outerHTML")
    sel_soup = bSoup(html, 'html.parser')
    print (sel_soup.findAll('img'))
    images = []
    for i in sel_soup.findAll('img'):
        src = i['src']
        images.append(src)
    print(images)
    current_path = os.getcwd()
    for img in images:
        try:
            file_name = os.path.basename(img)
            img_r = requests.get(img, stream=True)
            new_path = os.path.join(current_path, 'images', file_name)
            with open(new_path, 'wb') as output_file:
                shutil.copyfilobj(img_r.raw, output_file)
            del img_r
        except:
            pass
    iterations +=1
    time.sleep(5)

The result of running this code is no images saved. Any help would be greatly appreciated.

Answer 1

the html for image is not rendered by Javascript so you don't need selenium. use beautifulsoup with re.compile to match href in the a element that start with /ItemImages/ .

note it use relative URL you need to append the domain to images URL.

base_url = 'https://auctions.morphyauctions.com'
url = base_url + "/French_Fashion_Doll_with_Unusual_Body_-LOT450029.aspx"

html = requests.get(url).text
sel_soup = BeautifulSoup(html, 'html.parser')

images = []
for a in sel_soup.findAll('a', href=re.compile(r'^/ItemImages/')):
    ahref = base_url + a['href'] # wee need to append the domain here
    images.append(ahref)
print(images)
current_path = os.getcwd()
for img in images:
    try:
        file_name = os.path.basename(img)
        img_r = requests.get(img)
        new_path = os.path.join(current_path, 'images', file_name)
        with open(new_path, 'wb') as output_file:
            output_file.write(img_r.content)
    except:
        print(ex)