多线程 python 请求

Question

I have written this code:

 import requests
 redfile = open('/root/Desktop/23/1.txt','r')
 url = input('Type url :' )
 for x in redfile:

    req = requests.get(url + '/' + x.rstrip())
    if req.status_code == 200:
        print('this page' + url + '/' + x +'is avable')
        f = open('/root/Desktop/23/2.txt','a')
        f.write('\n')
        f.write(url+'/'+x)
        f.write('\n')
        f.close()

now I need a make the requests Multithread? how can I do it?

Answer 1

Maybe this is what you search for:

import requests
import threading


def thread(urls):
    while True:
        if len(urls) < 1:
            break
        url = urls.pop()
        r = requests.get(url)
        if r.ok:
            print('this page' + url  + 'is avable')
            with open('/root/Desktop/23/2.txt','a') as f:
                f.write('\n')
                f.write(url)
                f.write('\n')

urls = []
baseurl = input('Type url :' )
threads = []
for line in open('/root/Desktop/23/1.txt'):
    urls.append(baseurl + '/' + line.rstrip())
for i in range(10):
    t = threading.Thread(target=thread, args=(urls,))
    threads.append(t)
    t.start()
for t in threads:
    t.join()
print('Done')

Using the threading module like this make your script multi threaded. This script will have 10 threads:

for i in range(10):

Answer 2

I suggest using a thread pool and the concurrent.futures module is the most flexible:

import requests
from concurrent.futures import ThreadPoolExecutor

# worker thread to process one url:
def proccess_url(url, session):
    r = session.get(url)
    if r.status_code == 200:
        print('this page' + url + ' is avable')
        with open('/root/Desktop/23/2.txt', 'a') as f:
            f.write('\n')
            f.write(url)
            f.write('\n')


url = input('Type url: ' )
#create all the urls:
with open('/root/Desktop/23/1.txt','r') as redfile:
    urls = [url + '/' + x.rstrip() for x in redfile]

with ThreadPoolExecutor(max_workers=len(urls)) as executor:
    session = requests.Session()
    futures = [executor.submit(process_url, url, session) for url in urls]
    for future in futures:
        future.result() # wait for each thread to end; worker thread implicitly returns None