简体   繁体   中英

How to use multiple proxies with requests library, python?

I have a list of proxies which I want requests lib to use it. Because some of them don't work I want to change the proxy each time one doesn't respond.

I have this code(I just tried if it would work this way it's not final)

import requests
import itertools

HTTP = [List of HTTP proxies...]
HTTPS = [list of https proxies...]


def try_proxies(http_proxies, https_proxies):
    for proxy_http_element in http_proxies:
        http_proxy = proxy_http_element
        yield http_proxy

    for proxy_https_element in https_proxies:
        https_proxy = proxy_https_element
        yield https_proxy


proxy_result = try_proxies(HTTP, HTTPS)
print(proxy_result)


# proxies = {
#         'http': http_proxy,
#         'https': https_proxy
# }
#
# try:
#     res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=5)
#     print(res)
# except requests.exceptions.ConnectTimeout:
#     print("well at last your tried")

def main():
    try_proxies(HTTP, HTTPS)


main()

but it doesn't work like I want it to work and I know that because it's not correct. How is the best way to do this?

Edit: I lowkey improved the code but still didn't get the result I expected because it doesn't test each proxy but the last value of https_proxy

import requests

HTTP = [List of HTTP proxies...]
HTTPS = [List of HTTPS proxies...]


def try_proxies(http_proxies, https_proxies):
    for proxy_http_element in http_proxies:
        http_proxy = proxy_http_element

    for proxy_https_element in https_proxies:
        https_proxy = proxy_https_element

    proxies = {
        'http': http_proxy,
        'https': https_proxy,
    }

    try:
        res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=20)
        print("here")
        print(res.status_code)
    except requests.exceptions.ConnectTimeout:
        print("well at last your tried")


def main():
    try_proxies(HTTP, HTTPS)


main()

I solved the issue in this way:

import requests

HTTP = [List of http proxies...]
HTTPS = [List of https proxies]


def try_proxies(http_proxies, https_proxies):
    for proxy_http_element in http_proxies:
        http_proxy = proxy_http_element
        yield http_proxy

    for proxy_https_element in https_proxies:
        https_proxy = proxy_https_element
        yield https_proxy


test = [p for p in try_proxies(HTTP, HTTPS)]
for proxy in test:
    print(f"{proxy}")
    proxies = {
        'http': proxy
    }
    try:
        res = requests.get("https://httpbin.org/ip", proxies=proxies, timeout=10)
        print("here")
        print(res.status_code)
    except requests.exceptions.ConnectTimeout as CT:
        print(f"well at last your tried: {CT}")

I hope this can be helpful to someone else!

Edit: I removed the 'https': proxy because because of that I had an error when I removed that the proxies worked!

The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.

 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM