python requests: (SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)'))

Question

I have a python script that gets me some information from basketball-reference.com

It stopped working today due to this error:

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
    httplib_response = self._make_request(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
    self._validate_conn(conn)

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 978, in _validate_conn
    conn.connect()

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connection.py", line 362, in connect
    self.sock = ssl_wrap_socket(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\util\ssl_.py", line 386, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 500, in wrap_socket
    return self.sslsocket_class._create(

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 1040, in _create
    self.do_handshake()

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 1309, in do_handshake
    self._sslobj.do_handshake()

SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)


During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\adapters.py", line 439, in send

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 726, in urlopen
    retries = retries.increment(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\util\retry.py", line 446, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))

MaxRetryError: HTTPSConnectionPool(host='www.basketball-reference.com', port=443): Max retries exceeded with url: /search/search.fcgi?search=RJ+Barrett (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))


During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File "G:\fan_duel\april_version\main_2.py", line 13, in <module>
    test = [f.get_bref_id_tester(x) for x in full_names]

  File "G:\fan_duel\april_version\main_2.py", line 13, in <listcomp>
    test = [f.get_bref_id_tester(x) for x in full_names]

  File "G:\fan_duel\april_version\functions.py", line 30, in get_bref_id_tester
    search_results_page = requests.get(url, headers=headers, allow_redirects='False')

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\api.py", line 76, in get

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\api.py", line 61, in request

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\sessions.py", line 542, in request

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\sessions.py", line 655, in send

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\adapters.py", line 514, in send

SSLError: HTTPSConnectionPool(host='www.basketball-reference.com', port=443): Max retries exceeded with url: /search/search.fcgi?search=RJ+Barrett (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))


runfile('G:/fan_duel/april_version/main_2.py', wdir='G:/fan_duel/april_version')
Reloaded modules: functions
Traceback (most recent call last):

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
    httplib_response = self._make_request(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
    self._validate_conn(conn)

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 978, in _validate_conn
    conn.connect()

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connection.py", line 362, in connect
    self.sock = ssl_wrap_socket(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\util\ssl_.py", line 386, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 500, in wrap_socket
    return self.sslsocket_class._create(

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 1040, in _create
    self.do_handshake()

  File "C:\Users\pphotsauce\anaconda3\lib\ssl.py", line 1309, in do_handshake
    self._sslobj.do_handshake()

SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)


During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\adapters.py", line 439, in send

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\connectionpool.py", line 726, in urlopen
    retries = retries.increment(

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\urllib3\util\retry.py", line 446, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))

MaxRetryError: HTTPSConnectionPool(host='www.basketball-reference.com', port=443): Max retries exceeded with url: /search/search.fcgi?search=Chris+Boucher (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))


During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File "G:\fan_duel\april_version\main_2.py", line 13, in <module>
    test = [f.get_bref_id_tester(x) for x in full_names]

  File "G:\fan_duel\april_version\main_2.py", line 13, in <listcomp>
    test = [f.get_bref_id_tester(x) for x in full_names]

  File "G:\fan_duel\april_version\functions.py", line 33, in get_bref_id_tester
    search_results_page = requests.get(url, headers=headers, allow_redirects='False')

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\api.py", line 76, in get

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\api.py", line 61, in request

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\sessions.py", line 542, in request

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\sessions.py", line 655, in send

  File "C:\Users\pphotsauce\anaconda3\lib\site-packages\requests\adapters.py", line 514, in send

SSLError: HTTPSConnectionPool(host='www.basketball-reference.com', port=443): Max retries exceeded with url: /search/search.fcgi?search=Chris+Boucher (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)')))

I have a function that uses an NBA player's full name and returns a basketball reference id. If I apply this function to a pandas series or iterate through a list of names, only some of the names (different names each time) will cause the error. ('nightmare' is a dictionary with troublesome names)

nightmare = {
    
'Bol Bol': 'bolbo01',
'Nicolo Melli': 'mellini01',
'Davis Bertans': 'bertada01',
'Tomas Satoransky': 'satorto01',
'Theo Maledon': 'maledth01',
'Bogdan Bogdanovic': 'bogdabo01',

}

def get_bref_id(full_name):
    sleep(0.2)
    if full_name not in nightmare:
        try:
            first_name, last_name = full_name.split(' ')
            url = 'https://www.basketball-reference.com/search/search.fcgi?search='+first_name+'+'+last_name
            search_results_page = requests.get(url, allow_redirects = "False")
            soup = bs(search_results_page.content, 'html.parser')
            potential_links = soup.find_all('div', class_="search-item-name")
            bucket = [element for element in potential_links if '202' in element.text if first_name in element.text]
            bref_id = str(bucket[0]).replace('.', '/').split('/')[3]
        except IndexError: # b-ref search occasionally takes you straight to the page
            bref_id = search_results_page.url.split('/')[5].split('.')[0]
        finally:
            return bref_id
    else:
        return nightmare[full_name]

I don't understand anything about SSL or what could be causing this issue. If you could point me in the right direction to learn more, I would be grateful.

Answer 1

The server www.basketball-reference.com requires at least TLS 1.2. It looks like your Python is linked against a version of OpenSSL which is too old to support TLS 1.2. Use the following code to check which OpenSSL version is used. Support for TLS 1.2 was added with OpenSSL 1.0.1 ages ago, but for example MacOS shipped for a long time with the old version OpenSSL 0.9.8.

import ssl
print(ssl.OPENSSL_VERSION)

Answer 2

Try add the user agent in the headers parameter. Also, I added a little input option if the search returns more than 1 option so you can choose:

Code:

from bs4 import BeautifulSoup as bs
import requests
from time import sleep

#pip install choice
import choice

nightmare = {
    
'Bol Bol': 'bolbo01',
'Nicolo Melli': 'mellini01',
'Davis Bertans': 'bertada01',
'Tomas Satoransky': 'satorto01',
'Theo Maledon': 'maledth01',
'Bogdan Bogdanovic': 'bogdabo01',

}

def get_bref_id(full_name):
    sleep(0.2)
    if full_name not in nightmare:
        first_name, last_name = full_name.split(' ')
        url = 'https://www.basketball-reference.com/search/search.fcgi?search='+first_name+'+'+last_name
        headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'}
        search_results_page = requests.get(url, headers=headers, allow_redirects = "False")
        soup = bs(search_results_page.content, 'html.parser')
        potential_links = soup.find_all('div', class_="search-item-name")
        
        if len(potential_links) > 1:
            bucket = {}
            for element in potential_links:
                player = element.find('a').text
                playerId = element.find('a')['href'].split('/')[-1].replace('.html','')
                statsType = element.find('a')['href'].split('/')[1]
                bucket[player + ' (%s)' %statsType] = playerId
            print('Which player id did you want?')
            player_choice = choice.Menu(bucket.keys()).ask()
            bref_id = bucket[player_choice]
        
        else:
            try:
                element = soup.find_all('div', class_="search-item-name")[0]
                bref_id = element.find('a')['href'].split('/')[-1].replace('.html','')
            except IndexError: # b-ref search occasionally takes you straight to the page
                bref_id = search_results_page.url.split('/')[5].split('.')[0]
            
        return bref_id
    else:
        return nightmare[full_name]

Output:

Which player id did you want?
Make a choice:
 0: Michael Jordan (1985-2003) (players)
 1: Michael Jordan (1984-1992) (international)
 2: Michael-Hakim Jordan (2006-2009) (international)
 3: Michael Jordan (2000-2003) (executives)

Enter number or name; return for next page

? 0

jordami01