Syntax Error using Selenium “Select” in Python 3

Question

I am using Selenium in Python 3 to automate my Chrome driver, log in to HathiTrust , and add new items to a collection called "INSULAE".

My log-in details are stored in my Chrome profile and my script manages to navigate to the correct results page and to select all items on that page. However, my script breaks when I am trying to use "Select" to access the collections dropdown menu and choose the "INSULAE" collection by name. I fear that the problem behind it is that I did not import "Select" correctly in the first place. I found two different versions in tutorials but do not know which one to use in Python 3 (see comment in script below).

The error notification I get is:

 File "<ipython-input-43-a99cf2afc99b>", line 57
    dropdown.select_element_by_visible_text("INSULAE") # choose collection by name
           ^
SyntaxError: invalid syntax

Here is my full script:

# Script to automatically add new items to the HathiTrust Featured Collection

# Starting SELENIUM for web automation

import selenium
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By

# I found these two ways of importing "Select" and do not know which one is correct:
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.ui import Select

from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from time import sleep # may be needed to introduce pauses when performance issues occur
import urllib # needed to merge strings to new URLs

# Opening CHROME with custom profile

options=webdriver.ChromeOptions()
options.add_argument("--user-data-dir=C:\\Users\\mbarg\\AppData\\Local\\Google\\Chrome\\User Data\\")
options.add_argument('--profile-directory=Profile 1')
driver = webdriver.Chrome(executable_path='C:\\chromedriver_win32\\chromedriver.exe', options=options)
driver.implicitly_wait(2)
driver.maximize_window()


print("Browser opened.")

# Navigate to HathiTrust and log in

driver.get("https://babel.hathitrust.org/Shibboleth.sso/Login?entityID=https://google.cirrusidentity.com/gateway&target=https://babel.hathitrust.org") #URL of guest log-in via GOOGLE
print("You have successfully logged in!")  

# Iterate through result pages

QueryURL="https://babel.hathitrust.org/cgi/ls?a=srchls;q1=insula%20insulae%20insulis%20insul%20insuln%20island%20islands%20isle%20isles%20%C3%AEle%20%C3%AEles%20isola%20isole;field1=ocr;anyall1=any;yop=between;pdate_start=1600;pdate_end=1800;facet=countryOfPubStr%3A%22Germany%22;sz=100;pn="
pages=range(3, 138)

for p in pages:
    driver.refresh()
    target=[QueryURL, str(p)]
    s=""
    targetURL="".join(target)
    print(targetURL)   
# navigate to target URL
    driver.get(targetURL)
    driver.find_element_by_xpath("//button[@id='action-select-all']").click() # select all items on page
    dropdown=select(driver.find_element_by_name("Choose collection") # navigate to dropdown
    dropdown.select_element_by_visible_text("INSULAE") # choose collection by name
    driver.find_element_by_id("line1980").click() # add selected results to collection
    print("Page no.", p, "has been added!") 
                    
# finish and close browser
print("All requested items have been added!")
driver.quit()

I have never tried to automate selections from drop-down menus before and am very grateful for your advice.

Answer 1

I figured out that my script had two main issues:

a) I forgot the second round bracket in line 56 (one line above the one indicated in the error notification). b) I initially used select_element_by_visible_text instead of select_by_visible_text .

When that was fixed, I also managed to improve calling the individual elements from the HTML source code.

This is what the updated script looks like:

# Script to automatically add new items to the HathiTrust Featured Collection

# Starting SELENIUM for web automation

import selenium
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from time import sleep # needed to introduce pauses
import urllib # needed to merge strings to new URLs

# Opening CHROME with custom profile

options=webdriver.ChromeOptions()
options.add_argument("--user-data-dir=C:\\Users\\mbarg\\AppData\\Local\\Google\\Chrome\\User Data\\")
options.add_argument('--profile-directory=Profile 1')
#driver = webdriver.Chrome(ChromeDriverManager().install())
driver = webdriver.Chrome(executable_path='C:\\chromedriver_win32\\chromedriver.exe', options=options)
driver.implicitly_wait(2)
driver.maximize_window()

print("Browser opened.")

# Navigate to the application home page and log in

driver.get("https://babel.hathitrust.org/Shibboleth.sso/Login?entityID=https://google.cirrusidentity.com/gateway&target=https://babel.hathitrust.org") #URL of guest log-in via GOOGLE
print("You have successfully logged in!")  

# Iterate through result pages

QueryURL="https://babel.hathitrust.org/cgi/ls?a=srchls;q1=insula%20insulae%20insulis%20insul%20insuln%20island%20islands%20isle%20isles%20%C3%AEle%20%C3%AEles%20isola%20isole;field1=ocr;anyall1=any;yop=between;pdate_start=1600;pdate_end=1800;facet=countryOfPubStr%3A%22Germany%22;sz=100;pn="
pages=range(3, 138)

for p in pages:
    driver.refresh()
    target=[QueryURL, str(p)]
    s=""
    targetURL="".join(target)
    print(targetURL)   
# navigate to target URL
    driver.get(targetURL)
    driver.find_element_by_xpath("//button[@id='action-select-all']").click() # select all items on page
    # <select size="1" id="collection-chooser"></select>
    dropdown=Select(driver.find_element_by_id("collection-chooser")) # navigate to dropdown
    # <option value="1751299776">INSULAE</option>
    dropdown.select_by_visible_text("INSULAE") # choose collection from list of options
    # <button class="button btn btn-primary" id="addits">Add</button>
    driver.find_element_by_id("addits").click() # add selected results to collection
    print("Page no.", p, "has been added!") 
                    
# finish and close browser
print("All requested items have been added!")
driver.quit()

Answer 2

You forgot the closing parentheses on the line before:

dropdown=select(driver.find_element_by_name("Choose collection")) # navigate to dropdown
dropdown.select_element_by_visible_text("INSULAE")