在opensea中抓取页面时如何打破while循环

Question

I'm trying to scrap opnesea.io, I have code which scrapes all pages, but I need only first five page for scraping, so I have try to break loop, but it doesn't do it.

time.sleep(2)  # Allow 2 seconds for the web page to open
data = []

path = ChromeDriverManager().install()
url = 'https://opensea.io/rankings?sortBy=seven_day_volume'
driver = webdriver.Chrome(path)
driver.get(url)
start = time.time()
def scroll():

""" Get urls from Opensea.io """

  global data
  scroll_pause_time = 1 
  screen_height = driver.execute_script("return window.screen.height;") 
  i = 1

  num = 0
  while num True:
    
    driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))  
    i += 1
    time.sleep(scroll_pause_time)
    main_url = 'https://opensea.io'
    # update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
    scroll_height = driver.execute_script("return document.body.scrollHeight;")
    soup = BS(driver.page_source, 'html.parser')
    divs = soup.find_all('a', class_='sc-1pie21o-0 elyzfO sc-1xf18x6-0 sc-1twd32i-0 sc-1idymv7-0 dGptxx kKpYwv iLNufV fresnel-lessThan-xl')
    for items in divs:
        link = main_url + items['href'] 
        print(link)
        d = {'link' : link}
        print('Done!')
       
        data.append(d)
        if (screen_height) * i > scroll_height:
            el = driver.find_element_by_xpath('//*[@id="main"]/div/div[3]/button[2]').click()
            time.sleep(7)
            scroll()
            num += 1
            if num == 5:
                return
            

    
    

scroll()
print('Done ----> Opensea.io urls')

So, you can see I use recursion for my task, I know that using a while loop and recursion at same time is not good idea, but only in this way it's scraping more than one page.

Answer 1

Add a parameter to the function and create a global variable outside of the function called pages and pass it the function. Check if it's less than 5 using if statement and increment it before a recursion. Like below:

time.sleep(2)  # Allow 2 seconds for the web page to open
data = []

path = ChromeDriverManager().install()
url = 'https://opensea.io/rankings?sortBy=seven_day_volume'
driver = webdriver.Chrome(path)
driver.get(url)
start = time.time()
#create a new integer to count the number of recursions outside of the function
pages = 0
#pass it to the function
def scroll(pages):

""" Get urls from Opensea.io """

  global data
  scroll_pause_time = 1 
  screen_height = driver.execute_script("return window.screen.height;") 
  i = 1

  while num True:
    
    driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))  
    i += 1
    time.sleep(scroll_pause_time)
    main_url = 'https://opensea.io'
    # update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
    scroll_height = driver.execute_script("return document.body.scrollHeight;")
    soup = BS(driver.page_source, 'html.parser')
    divs = soup.find_all('a', class_='sc-1pie21o-0 elyzfO sc-1xf18x6-0 sc-1twd32i-0 sc-1idymv7-0 dGptxx kKpYwv iLNufV fresnel-lessThan-xl')
    for items in divs:
        link = main_url + items['href'] 
        print(link)
        d = {'link' : link}
        print('Done!')
       
        data.append(d)
        if (screen_height) * i > scroll_height and pages < 5:
            el = driver.find_element_by_xpath('//*[@id="main"]/div/div[3]/button[2]').click()
            time.sleep(7)
            #incremnt it before every recursion
            pages += 1
            scroll(pages)
            return