在python中用子進程重定向stdout很慢

Question

當我在Python中使用子進程重定向標准輸出時，吞吐量會非常緩慢。 我做錯了嗎？

基本上，我通過管道將外部程序的標准輸出放入隊列中。 然后在另一個功能中，我在控制台中打印它。

這是帶有hexdump的示例代碼，用於生成隨機輸出：

from subprocess import Popen, PIPE
from queue import Queue
import sys
from threading import Thread, Event
import threading

class Buffer(Queue):

    def __init__(self, *args, **kwargs):
        Queue.__init__(self, *args, **kwargs)

    def write(self, line):
        self.put_nowait(line)
        self.join()

    def read(self):
        element = self.get_nowait()
        self.task_done()
        return element

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.returncode is None:
        for line in hexdump.stdout.readlines(8192):
            buffer.write(line)
            if stopped.is_set():
                hexdump.terminate()
                hexdump.wait()
                print('process terminated.')
                break

def read_output(buffer, stopped):
    while not stopped.is_set():
        while not buffer.empty():
            output = buffer.read()
            print('********* output: {}'.format(output))
            sys.stdout.flush()
    print('stopped')
    sys.stdout.flush()


buffer = Buffer()
stopped = Event()


generate_random_output = Thread(target=write_output, args=(buffer, stopped))
generate_random_output.name = 'generate_random_output'
generate_random_output.start()

process_output = Thread(target=read_output, args=(buffer, stopped))
process_output.name = 'process_output'
process_output.start()

try:
    while True:
        continue
except KeyboardInterrupt:
    stopped.set()
    generate_random_output.join()
    process_output.join()
    print('finished generating')
    print('finished processing')

我將不勝感激任何幫助。

Answer 1

無需將輸出重定向到Queue，而是直接處理它：

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.poll() is None:
        while not stopped.is_set():
            for line in iter(hexdump.stdout.readline, b''):
                print('********* output: %s' % line.decode(), end='')
                sys.stdout.flush()

        hexdump.terminate()
        hexdump.wait()
        print('process terminated.')
        break