如何使用 pyaudio 播放音频并同时在 pyqtgraph 中显示来自同一音频文件的输入？

Question

I want to build a real time spectrum and waveform display with pyqtgraph. The input to pyqtgraph should come from an audio file that should also be playing at the same time through pyaudio. Seperately each part works, but I want to link them together.

I know that I need two processes running in parallel, but I don't know how I can implement that. From what I saw by watching the print messages, the play and animation functions are the two I would want to run in separate processes.

import tkinter as tk
from tkinter import filedialog, messagebox
from pydub import AudioSegment
from pydub.playback import play
import pyaudio
import struct
import numpy as np
from scipy.fftpack import fft
from pyqtgraph.Qt import *
import pyqtgraph as pg
import sys
import subprocess
import wave
import tempfile
import os
from threading import *
from multiprocessing import Process
from cv2 import plot
import asyncio

class Plot2D(object):

    def __init__(self):
        root = tk.Tk()
        root.withdraw()

        #audio konstanten
        self.CHUNK = 1024 * 2
        self.FORMAT = pyaudio.paInt16
        self.CHANNELS = 1
        self.RATE = 44100
        self.pause = False

        messagebox.showinfo("Info", "Wählen Sie eine Audiodatei")
        self.file = filedialog.askopenfilename()

        self.path = os.getcwd()
        subprocess.call(['sox', self.file, '-b', '16', '-c', '1', '-r', '44100', self.path + 'aud.wav'])

        self.wf = wave.open(self.path + 'aud.wav', 'rb')
        self.p = pyaudio.PyAudio()
        #self.stream = self.p.open(
        #    format = self.p.get_format_from_width(self.wf.getsampwidth()),
        #    channels = self.wf.getnchannels(),
        #    rate = self.wf.getframerate(),
        #    output = True
        #)
        #audiospur
        self.stream = self.p.open(
            format=self.FORMAT,
            channels=self.CHANNELS,
            rate=self.RATE,
            #input_device_index=chosen_device_index,
        #    input=True,
            output=True,
            frames_per_buffer=self.CHUNK
        )
        print("Audio init")

        self.traces = dict()
        self.phase = 0
        self.t = np.arange(0, 3.0, 0.01)
        pg.setConfigOptions(antialias=True)
        self.app = QtGui.QApplication(sys.argv)        
        self.win = pg.GraphicsWindow(title="Waveform & Spectrum")
        self.win.resize(1000, 600)
        self.win.setWindowTitle('Waveform & Spectrum')
        self.win.setGeometry(5, 115, 1910, 1070)

        self.waveform = self.win.addPlot(title='Waveform', row=1, col=1)
        self.spectrum = self.win.addPlot(title='Spectrum', row=2, col=1)

        #richtiges Aufnahmegerät festlegen
        #chosen_device_index = -1
        #for x in range(0,self.p.get_device_count()):
        #    info = self.p.get_device_info_by_index(x)
            #print self.p.get_device_info_by_index(x)
        #    if info["name"] == "pulse":
        #        chosen_device_index = info["index"]
        #        print( "Chosen index: ", chosen_device_index)



        #self.stream = self.p.open(
        #    channels = Audio().wf.getnchannels(),
        #    rate = self.RATE,
        #    output = True
        #)

        #plot variabeln
        self.x = np.arange(0, self.CHUNK)
        self.xf = np.linspace(0, self.RATE, self.CHUNK)
        print("Plot2D init")

    def start(self):
        if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
            QtGui.QApplication.instance().exec_()
        print("Plot2D start")

    def set_plotdata(self, name, data_x, data_y):
        if name in self.traces:
            self.traces[name].setData(data_x, data_y)
        else:
            if name == 'waveform':
                self.traces[name] = self.waveform.plot(pen='c', width=3)
                self.waveform.setYRange(-500, 500, padding=0)
                self.waveform.setXRange(20, self.CHUNK, padding=0.005)
            if name == 'spectrum':
                self.traces[name] = self.spectrum.plot(pen='m', width=3)
                self.spectrum.setLogMode(x=True, y=True)
                self.spectrum.setYRange(-4, 0, padding=0)
                self.spectrum.setXRange(4, np.log10(self.RATE), padding=0.005)
        print("Plot2D set_plotdata")

    def play(self):
        print("playing")
        data = self.wf.readframes(self.CHUNK)
        while data != '':
            self.stream.write(data)
            data = self.wf.readframes(self.CHUNK)
        print("Plot2D play")

    def update(self):
        print("updating")
        wf_data = self.stream.read(self.CHUNK)
        wf_data = struct.unpack(str(2 * self.CHUNK) + 'B', wf_data)
        wf_data = np.array(wf_data, dtype='b')[::2] + 128
        self.set_plotdata(name='waveform', data_x=self.x, data_y=wf_data)

        sp_data = fft(np.array(wf_data, dtype='int8') - 128)
        sp_data = np.abs(sp_data[0:int(self.CHUNK)]) * 2 / (128 * self.CHUNK)
        self.set_plotdata(name='spectrum', data_x=self.xf, data_y=sp_data)
        print("Plot2D update")

    def animation(self):
        timer = QtCore.QTimer()
        timer.timeout.connect(self.update)
        timer.start(30)
        print("Plot2D animation")
        self.start()

    #def close(self):
    #    self.stream.close()
    #    self.p.terminate()
    #    print("Audio close")

if __name__=='__main__':
    Plot2D().animation()
    #p1 = Process(target=Plot2D().animation())
    #p1.start()

Answer 1

Have you tried calling the play -method from within the update -method of your animation logic? The current chunk should then update your plot and be written to the audio stream in the same call.

I implemented a similiar real-time audio display using matplotlib that way.

Maybe something like:

def play(self, data):
        print("playing")
        self.stream.write(data)

def update(self):
        print("updating")
        wf_data = self.stream.read(self.CHUNK)

        # write to audio stream
        self.play(wf_data)

        # then update plot
        wf_data = struct.unpack(str(2 * self.CHUNK) + 'B', wf_data)
        wf_data = np.array(wf_data, dtype='b')[::2] + 128
        self.set_plotdata(name='waveform', data_x=self.x, data_y=wf_data)

        sp_data = fft(np.array(wf_data, dtype='int8') - 128)
        sp_data = np.abs(sp_data[0:int(self.CHUNK)]) * 2 / (128 * self.CHUNK)
        self.set_plotdata(name='spectrum', data_x=self.xf, data_y=sp_data)
        print("Plot2D update")

[hope that this is somehow helpful, this is my first answer here]