连接请求错误 - BS4 报废

Question

我在过去可以正常工作的代码上遇到错误

你能不能认为你可以让它再次工作，不知道为什么页面拒绝连接，在不同的计算机上尝试过，可能必须使用 ssl。

代码在汤上运行时失败=...所以在第二个请求中

因此，如果您知道如何避免有关服务器 ip 的这种复杂性，或者我不明白并且已经努力尝试修复它的东西，请。 请启发我

# -*- coding: utf-8 -*-
"""
Created on Tue Mar 22 18:16:10 2022

@author: christian marcos
"""
from ast import Break
import time
from distutils.log import error
from ftplib import error_temp
import itertools
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from bs4 import BeautifulSoup
mylist1= ("camion"),
mylist2= ("5")
popup_linkz= list()

session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)

headers = {
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'}
p=range(1, 2, 1)
for m,z in itertools.product(mylist1,mylist2):
    for i in p:
                    
                    
       
                 

                    def get_headers(session):
                        res = session.get("https://www.mercadopublico.cl/Home")
                        if res.status_code == 200:
                            print("Got headers")
                            # return res.text
                        else:
                            print("Failed to get headers")
                    
                    
                    
                    def search(session):
                        data = {
                            "textoBusqueda": m,
                            "idEstado": z,
                            "codigoRegion": "-1",   
                            "idTipoLicitacion": "-1",
                            "fechaInicio": None,
                            "fechaFin": None,
                            "registrosPorPagina": "100",
                            "idTipoFecha": [],
                            "idOrden": "1",
                            "compradores": [],
                            "garantias": None,
                            "rubros": [],
                            "proveedores": [],
                            "montoEstimadoTipo": [0],
                            "esPublicoMontoEstimado": None,
                            "pagina": i
                        }
                        res = session.post(
                            "https://www.mercadopublico.cl/BuscarLicitacion/Home/Buscar",
                            data=data)
                        if res.status_code == 200:
                            print("Search succeeded")
                            return res.text
                        else:
                            print("Search failed with error:", res.reason)
                    
                    
                    
                    def get_popup_link(html):
                        soup = BeautifulSoup(html, "html.parser")
                        dirty_links = [link["onclick"] for link in soup.select(".lic-block-body a")]
                        # clean onclick links
                        clean_links = [link.replace("$.Busqueda.verFicha('", "").replace("')", "") for link in dirty_links]
                        return clean_links
                    
                    
                    def get_download_html(s, links):
                        for link in links:
                            res = s.get(link)
                            if res.status_code == 200:
                                print("fetch succeeded")
                                return res.text
                            else:
                                print("fetch failed with error:", res.reason)
                    
                    def get_download_links(html):
                        soup = BeautifulSoup(html, "html.parser")
                        dirty_links = [link["onclick"] for link in soup.select(".lic-block-body a")]
                        # clean onclick links
                        clean_links = [link.replace("$.Busqueda.verFicha('", "").replace("')", "") for link in dirty_links]
                        return clean_links
                    
                 

                    def main():
                        with requests.Session() as s:
                            get_headers(s)
                            html = search(s)
                            popup_links = (get_popup_link(html))
                            popup_linkz.extend(popup_links)
                            #print(popup_links)
                            #print(popup_linkz)
                            #download_html = get_download_html(s, popup_links)
                            # print(download_html)
                            #popup_linkz.extend(popup_links for i in range(0, 1, 1))
                    main()
    a = popup_linkz
                
    print(a)
    results=[]
    results2=[] 
    
for i in popup_linkz:  # set end at 121` so it will use `120`, if you set end at `120` then it will finish on `80` # eliminate and make the url equal to i to test
    
        url=i
        print(i)
        try:
            soup = BeautifulSoup(session.get(url=i,timeout=30,verify=False,headers=headers).content, "html.parser")