Determine status code from python Retry exception

Question

import requests
from requests.adapters import HTTPAdapter
from urllib3 import Retry

DEFAULT_RETRIES = 5
DEFAULT_BACKOFF = 0.3


def session_with_retries(max_retries: int = DEFAULT_RETRIES,
                         backoff_factor: float = DEFAULT_BACKOFF,
                         proxies: dict = None) -> requests.Session:
    new_session = requests.Session()
    retries = Retry(total=max_retries,
                    connect=max_retries,
                    read=max_retries,
                    status=max_retries,
                    allowed_methods=frozenset(['HEAD', 'GET', 'POST']),
                    status_forcelist=frozenset([500, 502, 503, 504]),
                    backoff_factor=backoff_factor,
                    )
    retry_adapter = HTTPAdapter(max_retries=retries)
    new_session.mount('http://', retry_adapter)
    new_session.mount('https://', retry_adapter)

    if proxies is not None:
        new_session.proxies.update(proxies)

    return new_session

This code is for retry logic with python requests. And is working fine too. As expected it will throw exception on 503 status code after max_retries . So collectively it will throw exception for [500, 502, 503, 504] .

How I can get the status code on which the exception was thrown from this method. As exception messages i could see something

HTTPSConnectionPool(host='some-host', port=443): Max retries exceeded with url: /services/data/v52.0/connect/communities?pageSize=100 (Caused by ResponseError('too many 503 error responses',))

I traced back the exceptions urllib3.exceptions.MaxRetryError , requests.exceptions.RetryError but could not find about its status code for exception.

Strictly NO for other libraries like tenacity

Answer 1

You can handle this by turning off raising an error on 5xx responses, getting the response, and then either raising the last exception or examining the retry history.

The connect , read , and status are over-ridden when a value for total is passed.

import requests
from requests.exceptions import HTTPError
from requests.adapters import HTTPAdapter
from urllib3 import Retry

DEFAULT_RETRIES = 5
DEFAULT_BACKOFF = 0.3

def session_with_retries(max_retries: int = DEFAULT_RETRIES,
                         backoff_factor: float = DEFAULT_BACKOFF,
                         proxies: dict = None
                         ) -> requests.Session:
    """Creates a new Session with `max_retries` retry attempts at 5xx status
    codes.
    """
    new_session = requests.Session()
    retries = Retry(
        total=max_retries,
        allowed_methods=frozenset(['HEAD', 'GET', 'POST']),
        status_forcelist=frozenset([500, 502, 503, 504]),
        backoff_factor=backoff_factor,
        # ensure a response is returned:
        raise_on_status=False,
    )

    retry_adapter = HTTPAdapter(max_retries=retries)
    new_session.mount('http://', retry_adapter)
    new_session.mount('https://', retry_adapter)

    if proxies is not None:
        new_session.proxies.update(proxies)

    return new_session

Now when you send a request, the response will be returned without raising a MaxRetryError . To get the last response code, you simply call raise_for_status .

sess = session_with_retries()
res = sess.get('https://httpbin.org/status/500')

try:
    res.raise_for_status()
except HTTPError:
    print(f'The request failed with code: {res.status_code}')

# prints:
# The request failed with code: 500

If you want to see the complete retry status code history, you can use:

sess = session_with_retries()
res = sess.get('https://httpbin.org/status/500')

try:
    res.raise_for_status()
except HTTPError:
    print('The request retries failed with codes:')
    print([h.status for h in res.raw.retries.history])

# prints:
# The request retries failed with codes:
# [500, 500, 500]