TimeoutException error on opening the browser in headless mode in Selenium

Question

I'm working with this vezeeta.com to scrape some information about drugs and I am using selenium to deal with this site. I have allowed the user to enter the name of the drug, then Selenium will type the name of the drug that the user entered in the search box for the mentioned site, and it will select the first option in the drop-down menu, then it will open a new page in the browser, and then it will scrape the information from this page. During this process, Selenium will open the browser But I want to make Selenium stop opening the browser when I try to run the following code:

from bs4 import BeautifulSoup
import requests 
import re
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# Launch the Chrome browser and navigate to the website
options = webdriver.ChromeOptions()
options.add_argument('--headless')

driver = webdriver.Chrome(options=options)

#driver = webdriver.Chrome()

driver.get("https://www.vezeeta.com/en-eg/pharmacy")

#allow user to input the name of drug  
drug_name = input('Please enter name of drug : ')

# Find the search box and input drug name 
search_box = WebDriverWait(driver , 10).until(EC.presence_of_element_located((By.ID, "search-input")))
search_box.clear()
search_box.send_keys(drug_name)
search_box.send_keys(Keys.RETURN)

# Wait for the first option in the dropdown to appear and click on it
first_option = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//li[@class='ItemCardstyle__Card-sc-1fei2io-0 lcRscm']")))
first_option.click()

#Open the option that choose from dropdown menu in new window and get content of this page
driver.switch_to.window(driver.window_handles[-1])
content = driver.page_source
soup = BeautifulSoup(content, features='html.parser')
    
#Get all information about drug
all_info = soup.find('div' , class_ = 'Productstyle__ProductSideInfo-stlavj-11 VTzhP')
content_text = all_info.text

#Get description of drug
description = soup.find('div' , {'data-testid' : 'description-product'}).text
print('Description : ', description)

I get the following error:

TimeoutException                          Traceback (most recent call last)
C:\Users\ABDELR~1\AppData\Local\Temp/ipykernel_9888/1805791653.py in <module>
     22 
     23 # Find the search box and input drug name
---> 24 search_box = WebDriverWait(driver , 10).until(EC.presence_of_element_located((By.ID, "search-input")))
     25 search_box.clear()
     26 search_box.send_keys(drug_name)

~\anaconda3\lib\site-packages\selenium\webdriver\support\wait.py in until(self, method, message)
     93             if time.monotonic() > end_time:
     94                 break
---> 95         raise TimeoutException(message, screen, stacktrace)
     96 
     97     def until_not(self, method, message: str = ""):

TimeoutException: Message: 
Stacktrace:
Backtrace:
    GetHandleVerifier [0x00878893+48451]
    (No symbol) [0x0080B8A1]
    (No symbol) [0x00715058]
    (No symbol) [0x00740467]
    (No symbol) [0x0074069B]
    (No symbol) [0x0076DD92]
    (No symbol) [0x0075A304]
    (No symbol) [0x0076C482]
    (No symbol) [0x0075A0B6]
    (No symbol) [0x00737E08]
    (No symbol) [0x00738F2D]
    GetHandleVerifier [0x00AD8E3A+2540266]
    GetHandleVerifier [0x00B18959+2801161]
    GetHandleVerifier [0x00B1295C+2776588]
    GetHandleVerifier [0x00902280+612144]
    (No symbol) [0x00814F6C]
    (No symbol) [0x008111D8]
    (No symbol) [0x008112BB]
    (No symbol) [0x00804857]
    BaseThreadInitThunk [0x772D00C9+25]
    RtlGetAppContainerNamedObjectPath [0x775D7B4E+286]
    RtlGetAppContainerNamedObjectPath [0x775D7B1E+238]

This error only appear when i use this part of code otherwise, The code is run very well.

options.add_argument('--headless')  
driver = webdriver.Chrome(options=options)

Answer 1

Here is the solutions:

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# Launch the Chrome browser and navigate to the website
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36")

driver = webdriver.Chrome(options=options)
driver.get("https://www.vezeeta.com/en-eg/pharmacy")

# allow user to input the name of drug
drug_name = input('Please enter name of drug : ')

# Find the search box and input drug name
search_box = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "search-input")))
search_box.clear()
search_box.send_keys(drug_name)
search_box.send_keys(Keys.RETURN)

# Wait for the first option in the dropdown to appear and click on it
first_option = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.XPATH, "//li[@class='ItemCardstyle__Card-sc-1fei2io-0 lcRscm']")))
first_option.click()

soup = BeautifulSoup(driver.page_source, features='html.parser')

# Get all information about drug
all_info = soup.find('div', class_='Productstyle__ProductSideInfo-stlavj-11 VTzhP')
content_text = all_info.text

# Get description of drug
description = soup.find('div', {'data-testid': 'description-product'}).text
print('Description : ', description)

output:

Please enter name of drug : pain relief
Description :  Used to treat minor aches and pains of the muscles/joints (e.g., arthritis, backache, sprains). Applied topically 2 times daily or as prescribed

If you notice the only extra thing you need to add is the user-agent and that solves the TimeoutException error.

I hope this is helpful, Cheers!