Using Twisted to run and return output from shell commands (find)

Question

Im trying to use Twisted as a backend to allow users to search for files using shell commands.

The issue Im having is that when a request is recieved, subprocess blocks new requests to twisted and queues these requests up. This prevents me from cancelling a previous request in order to run a new one.

Ive tried a few solutions to this problem that I found else where including spawnprocess and attempting to use a deferred. However I was unable to find a way to return the output. Any help would be greatly appreciated!

global child_pid
if child_pid is None:
    pass
else:
    for i in child_pid:
        try:
            i.kill()
        except Exception as e:
             pass

try:
    find_cmd = ("find '/var/log/' -type f -name '*.log' -newermt '2018-
        01-01 00:00:00' ! -newermt '2018-02-06 23:59:59'  -exec cat {} \+  2>&1")

    # run search command
    proc = subprocess.Popen(find_cmd, stdout=subprocess.PIPE,
        stderr=subprocess.PIPE, shell=True, preexec_fn=os.setsid)
    # record pid of process so new requests can kill the previous 
    # subprocess
    child_pid.append(proc)

    # check for output until process is done
    output = ''
    while True:
        line = proc.stdout.readline()
        output += '\n' + line
        if line == '' or proc.poll() is not None:
            break
    return output
except Exception as e:
    return 'Error', False

EDIT:

Thanks! I feel like im getting closer. Its not blocking anymore. Im still a bit stuck though , I dont know why its taking me so long to figure this out... Ive read so much and its not sinking in im pretty frustrated which is probable not helping.

Here is what I have so far which may give a better idea as to what im trying to accomplish:

from __future__ import print_function
from os import environ
from twisted.internet.protocol import ProcessProtocol
from twisted.internet import reactor

class Collector(ProcessProtocol):
    def __init__(self):
        self.output = ""
        self.error = False
    def connectionMade(self):
        self.received = []

    def childDataReceived(self, fd, data):
        self.received.append(data)

    def processExited(self, reason):
        if 'code 0' in reason:
            print('0')
            self.error = False
        else:
            print('1')
            self.error = True
        reactor.stop()


def process_query(commands):
    if len(commands) > 0:
        return commands, 'success', True
    return commands, 'fail', False

def run_shell_command(commands):
    proto = Collector()
    find = reactor.spawnProcess(
        proto,
        commands[0],
        commands,
        env=environ,
    )
    reactor.callLater(.5, find.signalProcess, b"KILL")
    return proto.output, True

def format_output(output, commands):
    output_list = []
    try:
        output_list = output.split('\n')
        output_list.append(commands)
        return ouptut_list, True
    except:
        return '', False


def run_command(commands):
    response = {}
    message = ''
    result = True

    # format and validate parameters
    formatted_commands, message, result = process_query(commands)
    # where result is pass or failure of the command and message is reason
    if not result:
        response['message'] = message
        response['result'] = result
        return response
    shell_output, result = run_shell_command(formatted_commands)
    # where result is pass or failure of the command
    if not result:
        response['message'] = 'Error'
        return response
    formatted_output, result = format_output(shell_output, formatted_commands)
    response['data'] = formatted_output
    response['result'] = result
    if not result:
        response['message'] = 'Error'
        return response
    if len(formatted_output) < 1:
        response['message'] = 'No output found'
        response['result'] = False
        return response
    return response


cmd = [b"find", b"/var/log",
        b"-type", b"f",
        b"-name", b"*.log",
        b"-newermt", b"2018-01-01 00:00:00",
        b"!", b"-newermt", b"2018-02-06 23:59:59",
        b"-exec", b"cat", b"{}", b"+"]

x = run_command(cmd)
print(x)
reactor.run()

Ive been reading about deferreds and they seem like they might be able to help me... Possibly something like:

 def run_command(commands):
    response = {}
    message = ''
    result = True

    formatted_commands, message, result = process_query(commands)
    if not result:
        response['message'] = message
        response['result'] = result
        return response

    shell_output = utils.getProcessOutput(formatted_commands)
    shell_output.addCallback(format_output, formatted_commands)

    response['data'] = shell_output
    response['result'] = True
    return response

This doesnt work, however it feels close. Im completely lost. Thanks for your help so far though!

Answer 1

from __future__ import print_function
from os import environ
from twisted.internet.protocol import ProcessProtocol
from twisted.internet import reactor

class Collector(ProcessProtocol):
    def connectionMade(self):
        self.received = []

    def childDataReceived(self, fd, data):
        self.received.append(data)

    def processExited(self, reason):
        print(reason)
        reactor.stop()

proto = Collector()
find = reactor.spawnProcess(
    proto,
    b"find",
    [
        b"find", b"/var/log",
        b"-type", b"f",
        b"-name", b"*.log",
        b"-newermt", b"2018-01-01 00:00:00",
        b"!", b"-newermt", b"2018-02-06 23:59:59",
        b"-exec", b"cat", b"{}", b"+",
    ],
    env=environ,
)
reactor.callLater(.5, find.signalProcess, b"KILL")
reactor.run()