Scraped data to csv file beautifulsoup

Question

as the headline says im scraping data from a site with beautifulsoup scraper works fine but when i try to load the data into a csv file it only saves 1 area of data instead of the 500 the scraper gives here is my code:

#from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import csv



#launch url
url = "https://www.canlii.org/en/#search/type=decision&jId=bc,ab,sk,mb,on,qc,nb,ns,pe,nl,yk,nt,nu&startDate=1990-01-01&endDate=1992-01-14&text=non-pecuniary%20award%20&resultIndex=1"

# create a new Chrome session
driver = webdriver.Chrome('C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\selenium\webdriver\common\chromedriver.exe')
driver.implicitly_wait(30)
driver.get(url)


#Selenium hands the page source to Beautiful Soup
soup=BeautifulSoup(driver.page_source, 'lxml')

csv_file = open('test.csv', 'w')

csv_writer = csv.writer(csv_file, quoting=csv.QUOTE_ALL)
csv_writer.writerow(['Reference', 'case', 'link', 'province', 'keywords','snippets'])

#Scrape all
for scrape in soup.find_all('li', class_='result '):
    print(scrape.text)    
    
#Reference Index
    Reference = scrape.find('span', class_='reference')
    print(Reference.text)

#Case Name Index
    case = scrape.find('span', class_='name')
    print(case.text)
    
#Canlii Keywords Index
    keywords = scrape.find('div', class_='keywords')
    print(keywords.text)
    
#Province Index
    province = scrape.find('div', class_='context')
    print(province.text)
               
#snippet Index
    snippet = scrape.find('div', class_='snippet')
    print(snippet.text)
 
# Extracting URLs from the attribute href in the <a> tags.
    link = scrape.find('a', href=True)
    print(link)        
            
csv_writer.writerow([Reference.text, case.text,link.href, province.text, keywords.text, snippet.text])
csv_file.close()

Answer 1

Your csv_writer.writerow() function is outside your for loop. Try indenting it and see if it works.