Python async proxy status check

Question

Good day, im trying to master async stuff in python and have stuck in a situation which should be easily solvable, but i`m still frustrated what am i doing wrong. Suggest i have a list of proxies and i want to check if servers are up and running, and what is more important i want to do it in async way, unfortunately comparison with sync function shows the same time of execution.

So, to the code:

import requests
import os
import asyncio

from lxml import html


class GetUkProxy:
    url = "https://free-proxy-list.net/uk-proxy.html"

    def __init__(self):
        self.getProxyList(self.url)

    def getProxyList(self, url):
        self.proxies = []
        data = requests.get(url)
        response = html.fromstring(data.text)
        for item in response.xpath('//table[@id="proxylisttable"]/tbody/tr'):
            ip, port = item.getchildren()[:2]
            self.proxies.append({'IP': ip.text, 'Port': port.text, 'Status': 'unknown'})

    def printProxies(self):
        try:
            for proxy in self.proxies:
                print(proxy['IP'] + ':' + proxy['Port'], proxy['Status'])
        except:
            print("Proxy list is empty")

    def checkProxies(self):
        loop = asyncio.get_event_loop()
        data = loop.run_until_complete(self.checkProxyList())
        loop.close()


    async def checkProxyList(self):
        try:
            tasks = []
            for proxy in self.proxies[:10]:
                task = asyncio.ensure_future(self.checkProxy(proxy))
                tasks.append(task)
            await asyncio.gather(*tasks)
        except:
            print('Proxy list is empty')

    async def checkProxy(self, proxy=None):
        try:
            check = os.system('ping -n 4 -w 2 ' + proxy['IP'])
            if not check:
                proxy['Status'] = 'Dead'
            else:
                proxy['Status'] = 'Alive'
        except Exception as exp:
            print(exp)

    def syncCheck(self):
        for item in self.proxies[:10]:
            check = os.system('ping -n 4 -w 2 ' + item['IP'])

time of execution of async checkProxies is equal to the time of execution of synchronous syncCheck, so any ideas what is the root of the mistake and how to fix it are welcome.

Answer 1

os.system is a synchronous function that blocks the event loop.

You can execute system commands asynchronously using asyncio.create_subprocess_exec and keep from blocking the event loop.

async def checkProxy(self, proxy=None):
    try:
        check = await asyncio.create_subprocess_exec(
            'ping', '-c', '4', '-W', '2', proxy['IP'], stdout=asyncio.subprocess.PIPE)
        out, error = await check.communicate()

        proxy['Status'] = 'Alive'

        if error:
            proxy['Status'] = 'Dead'
    except Exception as exp:
        print(exp)

Edit: In checkProxies method we'll make a list of futures to be run together.

def checkProxies(self):
    loop = asyncio.get_event_loop()
    proxies = self.proxies[:10]
    tasks = [self.checkProxy(proxy) for proxy in proxies]


    loop.run_until_complete(
        asyncio.gather(
            *tasks
        )
    )

    loop.close()

uk = GetUkProxy()
uk.checkProxies()
print(uk.proxies)