捕获缓冲以播放实时音频流

Question

我正在以RTP数据包的形式通过网络获取实时音频流，我必须编写代码以捕获，缓冲并播放音频流。

问题

现在，为了解决此问题，我编写了两个线程，一个用于捕获音频，另一个用于播放音频。 现在，当我同时启动两个线程时，捕获线程的运行速度比播放线程 :(

缓冲区要求

• RTP音频数据包。
• 8kHz，16位线性采样（Linear PCM）。
• 每个RTP数据包中将发送4帧20ms音频。
• 在AudioStart = 24（20毫秒帧数）到达之前，请勿播放。
• 播放时...如果缓冲区中的20ms帧数达到0 ...则停止播放，直到对AudioStart帧进行缓冲，然后重新开始。

• 播放时...如果缓冲区中的20ms帧数超过AudioBufferHigh = 50，则删除24帧（以最简单的方式-从缓冲区中删除或仅丢弃接下来的6条RTP消息）。

  到目前为止我做了什么..

码

BufferManager.java

public abstract class BufferManager {
    protected static final Integer ONE = new Integer(1);
    protected static final Integer TWO = new Integer(2);
    protected static final Integer THREE = new Integer(3);
    protected static final Integer BUFFER_SIZE = 5334;//5.334KB
    protected static volatile Map<Integer, ByteArrayOutputStream> bufferPool = new ConcurrentHashMap<>(3, 0.9f, 2);
    protected static volatile Integer captureBufferKey = ONE;
    protected static volatile Integer playingBufferKey = ONE;
    protected static Boolean running; 
    protected static volatile Integer noOfFrames = 0;

    public BufferManager() {
        //captureBufferKey = ONE;
        //playingBufferKey = ONE;
        //noOfFrames = new Integer(0);
    }

    protected void switchCaptureBufferKey() {
        if(ONE.intValue() == captureBufferKey.intValue()) 
            captureBufferKey = TWO;
        else if(TWO.intValue() == captureBufferKey.intValue())
            captureBufferKey = THREE;
        else 
            captureBufferKey = ONE;
        //printBufferState("SWITCHCAPTURE");
    }//End of switchWritingBufferKey() Method.

    protected void switchPlayingBufferKey() {
        if(ONE.intValue() == playingBufferKey.intValue()) 
            playingBufferKey = TWO;
        else if(TWO.intValue() == playingBufferKey.intValue())
            playingBufferKey = THREE;
        else 
            playingBufferKey = ONE;
    }//End of switchWritingBufferKey() Method.

    protected static AudioFormat getFormat() {
        float sampleRate = 8000;
        int sampleSizeInBits = 16;
        int channels = 1;
        boolean signed = true;
        boolean bigEndian = true;
        return new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian);
    }

    protected int getByfferSize() {
        return bufferPool.get(ONE).size() 
                + bufferPool.get(TWO).size() 
                + bufferPool.get(THREE).size();
    }

    protected static void printBufferState(String flag) {
        int a = bufferPool.get(ONE).size();
        int b = bufferPool.get(TWO).size();
        int c = bufferPool.get(THREE).size();
        System.out.println(flag + " == TOTAL : [" + (a + b +c) + "bytes] ");
//      int a,b,c;
//      System.out.println(flag + "1 : [" + (a = bufferPool.get(ONE).size()) + "bytes], 2 : [" + (b = bufferPool.get(TWO).size())
//              + "bytes] 3 : [" + (c = bufferPool.get(THREE).size()) + "bytes], TOTAL : [" + (a + b +c) + "bytes] ");
    }
}//End of BufferManager Class.

AudioCapture.java

public class AudioCapture extends BufferManager implements Runnable {
    private static final Integer RTP_HEADER_SIZE = 12;
    private InetAddress ipAddress; 
    private DatagramSocket serverSocket;
    long lStartTime = 0;

    public AudioCapture(Integer port) throws UnknownHostException, SocketException {
        super();
        running = Boolean.TRUE;
        bufferPool.put(ONE, new ByteArrayOutputStream(BUFFER_SIZE));
        bufferPool.put(TWO, new ByteArrayOutputStream(BUFFER_SIZE));
        bufferPool.put(THREE, new ByteArrayOutputStream(BUFFER_SIZE));
        this.ipAddress = InetAddress.getByName("0.0.0.0");
        serverSocket = new DatagramSocket(port, ipAddress);
    }

    @Override
    public void run() {
        System.out.println();
        byte[] receiveData = new byte[1300];
        DatagramPacket receivePacket = null;
        lStartTime = System.currentTimeMillis();
        receivePacket = new DatagramPacket(receiveData, receiveData.length);
        byte[] packet = new byte[receivePacket.getLength() - RTP_HEADER_SIZE];
        ByteArrayOutputStream buff = bufferPool.get(captureBufferKey);
        while (running) {
            if(noOfFrames <= 50) {
                try {
                    serverSocket.receive(receivePacket);
                    packet = Arrays.copyOfRange(receivePacket.getData(), RTP_HEADER_SIZE, receivePacket.getLength());
                    if((buff.size() + packet.length) > BUFFER_SIZE) {
                        switchCaptureBufferKey();
                        buff = bufferPool.get(captureBufferKey);
                    }
                    buff.write(packet);
                    noOfFrames += 4;
                } catch (SocketException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } // End of try-catch block.
            } else {
                //System.out.println("Packet Ignored, Buffer reached to its maximum limit ");
            }//End of if-else block.
        } // End of while loop. 
    }//End of run() Method.
}

AudioPlayer.java

public class AudioPlayer extends BufferManager implements Runnable {
    long lStartTime = 0;

    public AudioPlayer() {
        super();
    }

    @Override
    public void run() {
        AudioFormat format = getFormat();
        DataLine.Info info = new DataLine.Info(SourceDataLine.class, format);
        SourceDataLine line = null;
        try {
            line = (SourceDataLine) AudioSystem.getLine(info);
            line.open(format);
            line.start();
        } catch (LineUnavailableException e1) {
            e1.printStackTrace();
        }

        while (running) {
            if (noOfFrames >= 24) {
                ByteArrayOutputStream out = null;
                try {
                    out = bufferPool.get(playingBufferKey);
                    InputStream input = new ByteArrayInputStream(out.toByteArray());
                    byte buffer[] = new byte[640];
                    int count;
                    while ((count = input.read(buffer, 0, buffer.length)) != -1) {
                        if (count > 0) {
                            InputStream in = new ByteArrayInputStream(buffer);
                            AudioInputStream ais = new AudioInputStream(in, format, buffer.length / format.getFrameSize());

                            byte buff[] = new byte[640];
                            int c = 0;
                            if((c = ais.read(buff)) != -1)
                                line.write(buff, 0, buff.length);
                        }
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
                /*byte buffer[] = new byte[1280];
                try {
                    int count;
                    while ((count = ais.read(buffer, 0, buffer.length)) != -1) {
                        if (count > 0) {
                            line.write(buffer, 0, count);
                        }
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }*/
                out.reset();
                noOfFrames -= 4;
                try {
                    if (getByfferSize() >= 10240) {
                        Thread.sleep(15);
                    } else if (getByfferSize() >= 5120) {
                        Thread.sleep(25);
                    } else if (getByfferSize() >= 0) {
                        Thread.sleep(30);
                    } 
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            } else {
                // System.out.println("Number of frames :- " + noOfFrames);
            }
        }
    }// End of run() method.
}// End of AudioPlayer Class class.

任何帮助或指向有用链接的指针都将不胜感激...

Answer 1

这个答案说明了流媒体技术的一些挑战。

简而言之，您的客户需要处理两个问题：

1）客户端和服务器上的时钟（晶体）不完全同步。 服务器可能比客户端快/慢几分之一赫兹。 客户端通过检查rtp数据包的传输速率，持续匹配推断服务器的时钟速率。 然后，客户端通过采样率转换来调整播放率。 因此，与其以48k播放，不如以48000.0001 Hz播放。

2）必须处理丢包，乱序到达等情况。 如果丢失了数据包，则仍需要在缓冲区流中保留这些数据包的占位符，否则音频将跳过并发出嘶哑的声音，变得不对齐。 最简单的方法是用静音替换那些丢失的数据包，但是应该调整相邻数据包的数量，以避免急剧的包络变化突然变为0。

您的设计似乎有些不合常规。 我已经成功地使用了环形缓冲区。 您还必须处理极端情况。

我总是说流媒体不是一件容易的事。