捕獲緩沖以播放實時音頻流

Question

我正在以RTP數據包的形式通過網絡獲取實時音頻流，我必須編寫代碼以捕獲，緩沖並播放音頻流。

問題

現在，為了解決此問題，我編寫了兩個線程，一個用於捕獲音頻，另一個用於播放音頻。 現在，當我同時啟動兩個線程時，捕獲線程的運行速度比播放線程 :(

緩沖區要求

• RTP音頻數據包。
• 8kHz，16位線性采樣（Linear PCM）。
• 每個RTP數據包中將發送4幀20ms音頻。
• 在AudioStart = 24（20毫秒幀數）到達之前，請勿播放。
• 播放時...如果緩沖區中的20ms幀數達到0 ...則停止播放，直到對AudioStart幀進行緩沖，然后重新開始。

• 播放時...如果緩沖區中的20ms幀數超過AudioBufferHigh = 50，則刪除24幀（以最簡單的方式-從緩沖區中刪除或僅丟棄接下來的6條RTP消息）。

  到目前為止我做了什么..

碼

BufferManager.java

public abstract class BufferManager {
    protected static final Integer ONE = new Integer(1);
    protected static final Integer TWO = new Integer(2);
    protected static final Integer THREE = new Integer(3);
    protected static final Integer BUFFER_SIZE = 5334;//5.334KB
    protected static volatile Map<Integer, ByteArrayOutputStream> bufferPool = new ConcurrentHashMap<>(3, 0.9f, 2);
    protected static volatile Integer captureBufferKey = ONE;
    protected static volatile Integer playingBufferKey = ONE;
    protected static Boolean running; 
    protected static volatile Integer noOfFrames = 0;

    public BufferManager() {
        //captureBufferKey = ONE;
        //playingBufferKey = ONE;
        //noOfFrames = new Integer(0);
    }

    protected void switchCaptureBufferKey() {
        if(ONE.intValue() == captureBufferKey.intValue()) 
            captureBufferKey = TWO;
        else if(TWO.intValue() == captureBufferKey.intValue())
            captureBufferKey = THREE;
        else 
            captureBufferKey = ONE;
        //printBufferState("SWITCHCAPTURE");
    }//End of switchWritingBufferKey() Method.

    protected void switchPlayingBufferKey() {
        if(ONE.intValue() == playingBufferKey.intValue()) 
            playingBufferKey = TWO;
        else if(TWO.intValue() == playingBufferKey.intValue())
            playingBufferKey = THREE;
        else 
            playingBufferKey = ONE;
    }//End of switchWritingBufferKey() Method.

    protected static AudioFormat getFormat() {
        float sampleRate = 8000;
        int sampleSizeInBits = 16;
        int channels = 1;
        boolean signed = true;
        boolean bigEndian = true;
        return new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian);
    }

    protected int getByfferSize() {
        return bufferPool.get(ONE).size() 
                + bufferPool.get(TWO).size() 
                + bufferPool.get(THREE).size();
    }

    protected static void printBufferState(String flag) {
        int a = bufferPool.get(ONE).size();
        int b = bufferPool.get(TWO).size();
        int c = bufferPool.get(THREE).size();
        System.out.println(flag + " == TOTAL : [" + (a + b +c) + "bytes] ");
//      int a,b,c;
//      System.out.println(flag + "1 : [" + (a = bufferPool.get(ONE).size()) + "bytes], 2 : [" + (b = bufferPool.get(TWO).size())
//              + "bytes] 3 : [" + (c = bufferPool.get(THREE).size()) + "bytes], TOTAL : [" + (a + b +c) + "bytes] ");
    }
}//End of BufferManager Class.

AudioCapture.java

public class AudioCapture extends BufferManager implements Runnable {
    private static final Integer RTP_HEADER_SIZE = 12;
    private InetAddress ipAddress; 
    private DatagramSocket serverSocket;
    long lStartTime = 0;

    public AudioCapture(Integer port) throws UnknownHostException, SocketException {
        super();
        running = Boolean.TRUE;
        bufferPool.put(ONE, new ByteArrayOutputStream(BUFFER_SIZE));
        bufferPool.put(TWO, new ByteArrayOutputStream(BUFFER_SIZE));
        bufferPool.put(THREE, new ByteArrayOutputStream(BUFFER_SIZE));
        this.ipAddress = InetAddress.getByName("0.0.0.0");
        serverSocket = new DatagramSocket(port, ipAddress);
    }

    @Override
    public void run() {
        System.out.println();
        byte[] receiveData = new byte[1300];
        DatagramPacket receivePacket = null;
        lStartTime = System.currentTimeMillis();
        receivePacket = new DatagramPacket(receiveData, receiveData.length);
        byte[] packet = new byte[receivePacket.getLength() - RTP_HEADER_SIZE];
        ByteArrayOutputStream buff = bufferPool.get(captureBufferKey);
        while (running) {
            if(noOfFrames <= 50) {
                try {
                    serverSocket.receive(receivePacket);
                    packet = Arrays.copyOfRange(receivePacket.getData(), RTP_HEADER_SIZE, receivePacket.getLength());
                    if((buff.size() + packet.length) > BUFFER_SIZE) {
                        switchCaptureBufferKey();
                        buff = bufferPool.get(captureBufferKey);
                    }
                    buff.write(packet);
                    noOfFrames += 4;
                } catch (SocketException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } // End of try-catch block.
            } else {
                //System.out.println("Packet Ignored, Buffer reached to its maximum limit ");
            }//End of if-else block.
        } // End of while loop. 
    }//End of run() Method.
}

AudioPlayer.java

public class AudioPlayer extends BufferManager implements Runnable {
    long lStartTime = 0;

    public AudioPlayer() {
        super();
    }

    @Override
    public void run() {
        AudioFormat format = getFormat();
        DataLine.Info info = new DataLine.Info(SourceDataLine.class, format);
        SourceDataLine line = null;
        try {
            line = (SourceDataLine) AudioSystem.getLine(info);
            line.open(format);
            line.start();
        } catch (LineUnavailableException e1) {
            e1.printStackTrace();
        }

        while (running) {
            if (noOfFrames >= 24) {
                ByteArrayOutputStream out = null;
                try {
                    out = bufferPool.get(playingBufferKey);
                    InputStream input = new ByteArrayInputStream(out.toByteArray());
                    byte buffer[] = new byte[640];
                    int count;
                    while ((count = input.read(buffer, 0, buffer.length)) != -1) {
                        if (count > 0) {
                            InputStream in = new ByteArrayInputStream(buffer);
                            AudioInputStream ais = new AudioInputStream(in, format, buffer.length / format.getFrameSize());

                            byte buff[] = new byte[640];
                            int c = 0;
                            if((c = ais.read(buff)) != -1)
                                line.write(buff, 0, buff.length);
                        }
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
                /*byte buffer[] = new byte[1280];
                try {
                    int count;
                    while ((count = ais.read(buffer, 0, buffer.length)) != -1) {
                        if (count > 0) {
                            line.write(buffer, 0, count);
                        }
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }*/
                out.reset();
                noOfFrames -= 4;
                try {
                    if (getByfferSize() >= 10240) {
                        Thread.sleep(15);
                    } else if (getByfferSize() >= 5120) {
                        Thread.sleep(25);
                    } else if (getByfferSize() >= 0) {
                        Thread.sleep(30);
                    } 
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } else {
                // System.out.println("Number of frames :- " + noOfFrames);
            }
        }
    }// End of run() method.
}// End of AudioPlayer Class class.

任何幫助或指向有用鏈接的指針都將不勝感激...

Answer 1

這個答案說明了流媒體技術的一些挑戰。

簡而言之，您的客戶需要處理兩個問題：

1）客戶端和服務器上的時鍾（晶體）不完全同步。 服務器可能比客戶端快/慢幾分之一赫茲。 客戶端通過檢查rtp數據包的傳輸速率，持續匹配推斷服務器的時鍾速率。 然后，客戶端通過采樣率轉換來調整播放率。 因此，與其以48k播放，不如以48000.0001 Hz播放。

2）必須處理丟包，亂序到達等情況。 如果丟失了數據包，則仍需要在緩沖區流中保留這些數據包的占位符，否則音頻將跳過並發出嘶啞的聲音，變得不對齊。 最簡單的方法是用靜音替換那些丟失的數據包，但是應該調整相鄰數據包的數量，以避免急劇的包絡變化突然變為0。

您的設計似乎有些不合常規。 我已經成功地使用了環形緩沖區。 您還必須處理極端情況。

我總是說流媒體不是一件容易的事。