In python, how can I use timeout in subprocess *and* get the output

Question

I used the code in these 2 question

how to get the return value from a thread in python

subprocess with timeout

And got this

import subprocess, threading, os

class ThreadWithReturnValue(threading.Thread):
    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, Verbose=None):
        threading.Thread.__init__(self, group, target, name, args, kwargs, Verbose)
        self._return = None

    def run(self):
        if self._Thread__target is not None:
            self._return = self._Thread__target(*self._Thread__args, **self._Thread__kwargs)

    def join(self, timeout = None):
        threading.Thread.join(self, timeout)
        return self._return

class SubprocessWrapper(object):
    def __init__(self, cmd, timeout):
        self.cmd = cmd
        self.process = None
        self.timeout = timeout
    def run(self):
        def target(cmd):
            self.process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
            returnValue = self.process.communicate()
            return [returnValue[0], returnValue[1], self.process.returncode]

        thread = ThreadWithReturnValue(target=target, args=[self.cmd])
        thread.start()
        returnValue = thread.join(self.timeout)
        if thread.is_alive():
            print 'cmd = ',self.cmd
            self.process.kill()
            returnValue = thread.join()
            print 'rc = ', returnValue[2]
        output = returnValue[0]
        error = returnValue[1]
        rc = returnValue[2]
        return (output, rc)

os.system('date +%T.%N')
s1 = SubprocessWrapper("echo 'Process started'; sleep 2; echo 'Process finished'", timeout = 3)
s1.run()
os.system('date +%T.%N')
s2 = SubprocessWrapper("echo 'Process started'; sleep 2; echo 'Process finished'", timeout = 1)
s2.run()
os.system('date +%T.%N')

The problem is that the output is

11:20:34.963947950
11:20:36.986685289
cmd =  echo 'Process started'; sleep 2; echo 'Process finished'
rc =  -9
11:20:38.995597397

So you can see the process which was supposed to be terminated after one second actually took 2 seconds. This happens because of the join() but in the question subprocess with timeout this works fine. This means that when I integrated both codes I caused this problem, my question is how to fix it? I was thinking that I might need to call threading.Thread.__init__ method in a different way but I can't understand how.

Answer 1

This code doesn't return output in one second despite the timeout. It returns it in two seconds after the sleep:

#!/usr/bin/env python3
from subprocess import TimeoutExpired, check_output as qx
from timeit import default_timer as timer

start = timer()
try:
    qx("echo 'Process started'; sleep 2; echo 'Process finished'",
       shell=True, universal_newlines=True, timeout=1)
except TimeoutExpired as e:
    print("Got %r in %.2f seconds" % (e.output, timer() - start))
else:
    assert 0 # should not get here

Output

Got 'Process started\n' in 2.00 seconds

Alarm-based solution from "Stop reading process output in Python without hang?" question works:

#!/usr/bin/env python
import signal
from subprocess import Popen, PIPE
from timeit import default_timer as timer

class Alarm(Exception):
    pass

def alarm_handler(signum, frame):
    raise Alarm

start = timer()
# start process
process = Popen("echo 'Process started'; sleep 2; echo 'Process finished'",
                shell=True, stdout=PIPE, bufsize=1, universal_newlines=True)

# set signal handler
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(1) # produce SIGALRM in a second
buf = []
try:
    for line in iter(process.stdout.readline, ""):
        buf.append(line) # assume it is atomic
except Alarm:
    process.kill()
else:
    signal.alarm(0) # reset alarm
finally:
    output = ''.join(buf)
print("Got %r in %.2f seconds" % (output, timer() - start))
process.stdout.close()
process.wait()

Output

Got 'Process started\n' in 1.00 seconds