音频 - 来自 Java 的流音频是断断续续的

Question

My main objective is to create live streaming of encrypted voice chat from mic. The encrypted audio is then transmitted over the network from one client to another. The problem is that the audio is always getting stuttering and choppy while running the program (streaming).

1. I tried different types of hardware (PC, laptop, Raspberry Pi).
2. Different OSes as well.
3. Only sampling un-encrypted audio to eliminated any issue causes by the encryption algorithm.
4. Changing audio sample rate.

Unfortunately everything failed.

To make it simple, I only included the code needed to transmit the audio over the network without the encryption.

MAIN CLASS - both sender and receiver

package com.emaraic.securevoice;

import com.emaraic.securevoice.utils.AES;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;

import javax.sound.sampled.*;

public class SecureVoice 
{

    public static void main(String[] args) 
    {

        Receiver rx = new Receiver();
        rx.start();

        Transmitter tx = new Transmitter();
        tx.start();

    }

    public static AudioFormat getAudioFormat() 
    { //you may change these parameters to fit you mic
        float sampleRate = 8000.0f;  //8000,11025,16000,22050,44100
        int sampleSizeInBits = 16;    //8,16
        int channels = 1;             //1,2
        boolean signed = true;        //true,false
        boolean bigEndian = false;    //true,false
        return new AudioFormat(sampleRate, sampleSizeInBits, channels, signed, bigEndian);
    }

    public static final String ANSI_BOLD = "\033[0;1m"; //not working in NetBeans
    public static final String ANSI_RESET = "\033[0m";
    public static final String ANSI_BLACK = "\033[30m";
    public static final String ANSI_RED = "\033[31m";
    public static final String ANSI_GREEN = "\033[32;4m";
    public static final String ANSI_YELLOW = "\033[33m";
    public static final String ANSI_BLUE = "\033[34m";
    public static final String ANSI_PURPLE = "\033[35m";
    public static final String ANSI_CYAN = "\033[36m";
    public static final String ANSI_WHITE = "\033[37m";
}

SENDER

package com.emaraic.securevoice;

import com.emaraic.securevoice.utils.AES;
import java.io.*;
import java.io.File;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.text.SimpleDateFormat;
import java.util.Date;
import javax.sound.sampled.AudioFileFormat;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.Mixer;
import javax.sound.sampled.Port;
import javax.sound.sampled.TargetDataLine;

public class Transmitter extends Thread
{    
    // these parameters must be copied and used in the Receiver class of the other client
    private static final String TX_IP = "10.101.114.179"; //ip to send to 
    private static final int TX_PORT = 1034;

    @Override
    public void run() 
    {
        SecureVoice color = new SecureVoice();
        Mixer.Info minfo[] = AudioSystem.getMixerInfo();
        System.out.println(color.ANSI_BLUE + "Detecting sound card drivers...");
        for (Mixer.Info minfo1 : minfo) 
        {
            System.out.println("   " + minfo1);
        }

        if (AudioSystem.isLineSupported(Port.Info.MICROPHONE)) 
        {
            try 
            {
                DataLine.Info dataLineInfo = new DataLine.Info(TargetDataLine.class, SecureVoice.getAudioFormat());
                final TargetDataLine line = (TargetDataLine) AudioSystem.getLine(dataLineInfo); //recording from mic
                line.open(SecureVoice.getAudioFormat());
                line.start(); //start recording
                System.out.println(color.ANSI_GREEN + "Recording...");
                byte tempBuffer[] = new byte[line.getBufferSize()];
                System.out.println(color.ANSI_BLUE + "Buffer size = " + tempBuffer.length + " bytes");
                //AudioCapture audio = new AudioCapture(line); //capture the audio into .wav file
                //audio.start();
                while (true) //AES encryption
                {
                    int read = line.read(tempBuffer, 0, tempBuffer.length);
                    byte[] encrypt = AES.encrypt(tempBuffer, 0, read);
//                    sendToUDP(encrypt);
                    sendToUDP(tempBuffer);
                }


            }

            catch (Exception e) 
            {
                System.out.println(e.getMessage());
                System.exit(0);
            }
        }


    }




    public static void sendToUDP(byte soundpacket[]) 
    {
        try 
        {
//            EncryptedAudio encrypt = new EncryptedAudio(soundpacket);
//            encrypt.start();
            DatagramSocket sock = new DatagramSocket();
            sock.send(new DatagramPacket(soundpacket, soundpacket.length, InetAddress.getByName(TX_IP), TX_PORT));
            sock.close();
        } 
        catch (Exception e) 
        {
            System.out.println(e.getMessage());
        }
    }
}

RECEIVER

package com.emaraic.securevoice;


import com.emaraic.securevoice.utils.AES;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioSystem;
import javax.sound.sampled.DataLine;
import javax.sound.sampled.SourceDataLine;

public class Receiver extends Thread {
    // these parameters must by used in the Transmitter class of the other client
    private static final String RX_IP = "localhost"; 
    private static final int RX_PORT = 1034;


    @Override
    public void run() {
        byte b[] = null;
        while (true) {

                b = rxFromUDP();
                speak(b);

        }
    }


    public static byte[] rxFromUDP() {
        try {
            DatagramSocket sock = new DatagramSocket(RX_PORT);
             byte soundpacket[] = new byte[8192];
            DatagramPacket datagram = new DatagramPacket(soundpacket, soundpacket.length, InetAddress.getByName(RX_IP), RX_PORT);
            sock.receive(datagram);
            sock.close();

//            return AES.decrypt(datagram.getData(),0,soundpacket.length); // soundpacket ;
            return soundpacket; //  if you want to hear encrypted form 
        } catch (Exception e) {
            System.out.println(e.getMessage());
            return null;
        }

    }

    public static void speak(byte soundbytes[]) {

        try {
            DataLine.Info dataLineInfo = new DataLine.Info(SourceDataLine.class, SecureVoice.getAudioFormat());
            try (SourceDataLine sourceDataLine = (SourceDataLine) AudioSystem.getLine(dataLineInfo)) {
                sourceDataLine.open(SecureVoice.getAudioFormat());
                sourceDataLine.start();
                sourceDataLine.write(soundbytes, 0, soundbytes.length);
                sourceDataLine.drain();
            }
        } catch (Exception e) {
            System.out.println(e.getMessage());
        }

    }



}

EXTRA LINK http://emaraic.com/blog/secure-voice-chat

IDE Used - Netbeans 11.1

Java JDK version - Java 13 (Windows) - OpenJDK11 (Linux)

Answer 1

Two problems. Network streamed data will have jitter in arrival time. Starting and stopping audio play will cause delay gaps and jitter due to OS and hardware driver overhead time. There is also the smaller problem of audio sample rate clock rate synchronization between the record and play systems. All of those can impact a continuous stream of audio samples at a fixed rate.

To avoid the audio start-up latency problem, don't stop your audio play or record system between network packets, always have audio data ready to play continuously at the current sample rate. To help cover network jitter, buffer some amount of audio data before starting playback, so there is always some audio ready to play even if the next network packet is sightly delayed.

You may have to gather some statistics on the audio startup and network latency and latency variation to determine a suitable amount to buffer. The alternative is an audio dropout concealment algorithm, which is far more complicated to implement.