Python set timeout on Popen.stdout.readline

Question

I'd like to be able to set timeout on subprocess stdout and return an empty string if exceeded timeout.

Here's my attempt to do so using asyncio.

However, it failed on using file.stdout.readline() in asyncio.wait_for .

Any idea how to fix this ?

import threading
import select
import subprocess
import queue
import time
import asyncio

class collector():
    @staticmethod
    async def readline(file, timeout=3):
        try:
            line = await asyncio.wait_for(file.stdout.readline(), timeout)
        except asyncio.TimeoutError:
            return ""
        else:
            return line

    @staticmethod
    async def background(command):
        f = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        while True:
            line = await collector.readline(file=f, timeout=3)



asyncio.run(collector.background("tail -f /tmp/2222"))

And here's callstack :

  File "/tmp/script.py", line 13, in readline
    line = await asyncio.wait_for(file.stdout.readline(), timeout)
  File "/usr/local/Cellar/python@3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py", line 462, in wait_for
    fut = ensure_future(fut, loop=loop)
  File "/usr/local/Cellar/python@3.9/3.9.6/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py", line 679, in ensure_future
    raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
TypeError: An asyncio.Future, a coroutine or an awaitable is required

Answer 1

The subprocess library only provides synchronous functions. These cannot be directly used by asyncio , and manually wrapping them is inefficient.

asyncio already ships with its own subprocess backend . Its process representation works similar to subprocess.Popen but allows to cooperatively wait for operations.

import asyncio.subprocess

async def background(*command):
    # create subprocess via asyncio
    proc = await asyncio.create_subprocess_exec(
        *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
    )
    while True:
        line = await readline(proc.stdout, timeout=0.01)
        print("read", len(line), "characters")

# read from an async stream vvvvvvvvvvvvvvvvvv instead of a file-like object
async def readline(stream: asyncio.StreamReader, timeout: float):
    try:
        # stream.readline is a coroutine vvvvvvvvvvvv
        return await asyncio.wait_for(stream.readline(), timeout=timeout)
    except asyncio.TimeoutError:
        return ""


asyncio.run(background("cat", "/dev/random"))