TCP中的TCP Webcam Streaming

Question

Thanks for reading and sorry for my bad English in advance.

I'm working on Webcam Streaming with OpenCV. My final goal is making a Skype-like application. so I'm trying basic 1:1 TCP model first.

About the TCP 1:1 model,
After connection, client send its real-time Webcam frames and server receive and display it in on its jpanel.

I did receiving a picture and displaying it on jpanel so far.
I'm trying to receive successive frames.
At first, the problem was Server side socket seems like waiting until inputs from client are finished, ie, it never stops because real-time frames are continuously sent.
So I sent every frame size before sending the frame to escape from unstoppable waiting.
But it doesn't work well. Client keeps sending frames, but server doesn't receive it well.
For example, if client send around 25k byte sized frames, the server only receive 1 to 3 bytes per one read even the buffer size is 512.

ClientThread.java

package client;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;

import javax.imageio.ImageIO;

import video.VideoCap;

public class ClientThread extends Thread
{

    String   serverIp;
    int      serverPort;
    Socket   socket;
    VideoCap videoCap;

    public ClientThread(Socket socket, String serverIp, int serverPort, VideoCap videoCap)
    {
        this.socket = socket;
        this.serverIp = serverIp;
        this.serverPort = serverPort;
        this.videoCap = videoCap;
    }

    public void run()
    {
        while (ClientUI.calling)
        {
            try
            {
                InputStream in = socket.getInputStream();
                DataInputStream dis = new DataInputStream(in);
                OutputStream out = socket.getOutputStream();
                DataOutputStream dos = new DataOutputStream(out);

                // receive
                int bufSize = dis.readInt();
                while (ClientUI.calling)
                {
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    ImageIO.write(videoCap.getOneFrame(), "jpg", baos);
                    InputStream inputImage = new ByteArrayInputStream(baos.toByteArray());

                    // frame size
                    dos.writeInt(baos.size());
                    out(inputImage, baos, bufSize);
                    Thread.sleep(5000);
                }

            }
            catch (IOException | InterruptedException e)
            {
                e.printStackTrace();
            }
        }
    }

    void out(InputStream in, OutputStream out, int bufSize)
    {
        long size = 0;
        try
        {
            byte[] buf = new byte[bufSize];
            int n;
            while ((n = in.read(buf)) > 0)
            {
                out.write(buf, 0, n);
                size += n;
                System.out.println("size: " + size);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }

        finally
        {
            System.out.println(getClass().getName() + " :: out >>> sent size: " + size);
        }
    }
}

ServerThread.java

package server;

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;

import javax.imageio.ImageIO;
import javax.swing.JPanel;

public class ServerThread extends Thread
{
    ServerSocket serverSocket;
    Socket       socket;
    JPanel       panel;
    byte[]       buf;

    public ServerThread(ServerSocket serverSocket, JPanel panel, int bufSize)
    {
        this.serverSocket = serverSocket;
        this.panel = panel;
        buf = new byte[bufSize];
    }

    public void run()
    {
        try
        {
            System.out.println("waiting for client");
            socket = serverSocket.accept();
            System.out.println("client accepted");
            InputStream in = socket.getInputStream();
            DataInputStream dis = new DataInputStream(in);
            OutputStream out = socket.getOutputStream();
            DataOutputStream dos = new DataOutputStream(out);
            dos.writeInt(buf.length);

            while (ServerUI.calling)
            {
                int frameSize = dis.readInt();
                ByteArrayOutputStream outImage = new ByteArrayOutputStream();
                long size = 0;
                int n;

                while (frameSize >= size)
                {
                    n = dis.read(buf);
                    if (n == -1)
                        break;
                    outImage.write(buf, 0, n);
                    size += n;
                    System.out.println(n);

                }

                InputStream inputImage = new ByteArrayInputStream(outImage.toByteArray());
                BufferedImage bufferedImage = ImageIO.read(inputImage);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
    }

}

Answer 1

I changed DataOutput/InputStream to ObjectOutput/InputStream. I'm not sure why it did not go well, but I guess this is because of serializing problem. but byte is not necessary to be serialized so I don't know exactly.

I'll provide with anyhow working codes. Because of AudioServer I divided into two Thread so the previous codes and the codes below are pretty different.

VideoServerThread.java

import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.net.ServerSocket;
import java.net.Socket;
import javax.imageio.ImageIO;
import javax.swing.JPanel;

public class VideoServerThread extends Thread
{
    private ServerSocket serverSocket;
    int                  videoServerPort;
    private Socket       socket;
    private JPanel       panel;
    private boolean      calling;

    public VideoServerThread(ServerSocket serverSocket, int videoServerPort, JPanel panel, boolean calling)
    {
        this.serverSocket = serverSocket;
        this.videoServerPort = videoServerPort;
        this.panel = panel;
        this.calling = calling;
    }

    @Override
    public void run()
    {
        System.out.println("Video Server opened!");
        try
        {
            serverSocket = new ServerSocket(videoServerPort);
            socket = serverSocket.accept();
            InputStream in = socket.getInputStream();
            ObjectInputStream ois = new ObjectInputStream(in);
            BufferedImage bufferedImage;
            InputStream inputImage;
            Frame f;
            while (calling)
            {
                f = (Frame) ois.readObject();
                inputImage = new ByteArrayInputStream(f.bytes);
                bufferedImage = ImageIO.read(inputImage);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, panel.getWidth(), panel.getHeight(), null);
                panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
                bufferedImage.flush();
                inputImage.close();
                f = null;
            }

        }

        catch (IOException e)
        {
            e.printStackTrace();
        }
        catch (ClassNotFoundException e)
        {
            e.printStackTrace();
        }
    }

    class Frame implements Serializable
    {
        public byte[] bytes;
        public Frame(byte[] bytes)
        { 
            this.bytes = bytes;
        }

        public int size()
        {
            return bytes.length;
        }
    }
}

VideoClientThread.java

import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.Socket;
import javax.imageio.ImageIO;
import common.Frame;
import video.VideoCap;

public class VideoClientThread extends Thread
{
    private final String formatType = "jpg";
    private VideoCap     videoCap;
    private Socket       socket;
    private String       ip;
    private int          port;
    private boolean      calling;

    public VideoClientThread(VideoCap videoCap, Socket socket, String ip, int port, boolean calling)
    {
        this.videoCap = videoCap;
        this.socket = socket;
        this.ip = ip;
        this.port = port;
        this.calling = calling;
    }

    public void run()
    {
        try
        {
            socket = new Socket(ip, port);
            socket.setSoTimeout(5000);
            ObjectOutputStream oos = new ObjectOutputStream(socket.getOutputStream());
            Frame f;
            BufferedImage bufferedImage;
            while (calling)
            {
                ByteArrayOutputStream fbaos = new ByteArrayOutputStream();
                bufferedImage = videoCap.getOneFrame();
                ImageIO.write(bufferedImage, formatType, fbaos);
                f = new Frame(fbaos.toByteArray());
                oos.writeObject(f);
                oos.flush();
                bufferedImage.flush();
                // Thread.sleep(33);
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        // catch (InterruptedException e)
        // {
        // e.printStackTrace();
        // }
    }
}

Answer 2

TCP transport layer is reliable and packet lost never occur, therefore may packets have delay, you can using DatagramSocket instead of ServerSocket and Socket to transport via UDP protocol.

sample server and client via DatagramSocket :

import java.io.IOException; 
import java.net.DatagramPacket; 
import java.net.DatagramSocket; 
public class smallserver { 

public static void main(String[] args) throws IOException { 

    string host = "127.0.0.1";
    int port = 5252;
    byte buf[] = new byte[2]; 
    byte send[] = { 13, 18 };

    //server
    DatagramSocket serverSocket  = new DatagramSocket(port);  
    DatagramPacket receivePacket = new DatagramPacket(buf, 2); 
    serverSocket.receive(receivePacket); 

    // client
    DatagramSocket clientSocket  = new DatagramSocket(host, port); 
    DatagramPacket sendPacket = new DatagramPacket(send, 2, clientSocket.getAddress(), clientSocket.getPort()); 
    clientSocket.send(sendPacket); 
 } 

}