Java Socket InputStream read missing bytes

Question

I got a very strange behaviour by reading bytes from the input stream of a socket.

In my project clients does requests to a service. For each request a new connection will be established.

At first the bytes are send that tells the service what kind of request will follow.

Then the request itself is send.

The service receives the bytes and proceeds the request. This does work for at least 95% of all request. For the remaining 5% there is a strange behaviour that i can not figure out.

The bytes are not all the bytes that got sended. But the most strange matter on this topic is that the missing bytes are not at the start or at the end of the stream. They are spread through the entire stream.

Sadly i can not provide the full code here cause it is work related. But i can provide the test code that shows the issue itself.

To figure out what is going on i wrote 2 classes. One derives from java.net.Socket and the other one from java.net.ServerSocket .

Here the code:

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.util.ArrayList;
import java.util.List;



public class DebugSocket extends Socket
{
    private class InputStreamWrapper extends InputStream
    {
        private int
            availables,
            closes,
            marksupporteds,
            resets;

        private List<Integer>
            marks   = new ArrayList<Integer>(),
            reads   = new ArrayList<Integer>();

        private List<Long>
            skips   = new ArrayList<Long>();


        @Override
        public int available() throws IOException
        {
            availables++;
            return DebugSocket.this.origininput.available();
        }

        @Override
        public void close() throws IOException
        {
            closes++;
            DebugSocket.this.origininput.close();
        }

        @Override
        public synchronized void mark(int readlimit)
        {
            marks.add(readlimit);
            DebugSocket.this.origininput.mark(readlimit);
        }

        @Override
        public boolean markSupported()
        {
            marksupporteds++;
            return DebugSocket.this.origininput.markSupported();
        }

        @Override
        public synchronized void reset() throws IOException
        {
            resets++;
            DebugSocket.this.origininput.reset();
        }

        @Override
        public int read() throws IOException
        {
            int read = DebugSocket.this.origininput.read();

            reads.add(read);

            if ( read != -1 )
            {
                DebugSocket.this.inputdebugbuffer.write(read);
            }

            return read;
        }

        @Override
        public int read(byte[] b) throws IOException
        {
            int read = DebugSocket.this.origininput.read(b);

            DebugSocket.this.inputdebugbuffer.write(b, 0, read);

            return read;
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException
        {
            int read = DebugSocket.this.origininput.read(b, off, len);

            DebugSocket.this.inputdebugbuffer.write(b, off, read);

            return read;
        }

        @Override
        public long skip(long n) throws IOException
        {
            long skipped = DebugSocket.this.origininput.skip(n);

            skips.add(skipped);

            return skipped;
        }
    }

    private class OutputStreamWrapper extends OutputStream
    {
        private int
            flushes,
            closes;


        @Override
        public void close() throws IOException
        {
            closes++;
            DebugSocket.this.originoutput.close();
        }

        @Override
        public void flush() throws IOException
        {
            flushes++;
            DebugSocket.this.originoutput.flush();
        }

        @Override
        public void write(int b) throws IOException
        {
            DebugSocket.this.outputdebugbuffer.write(b);
            DebugSocket.this.originoutput.write(b);
            DebugSocket.this.originoutput.flush();
        }

        @Override
        public void write(byte[] b) throws IOException
        {
            DebugSocket.this.outputdebugbuffer.write(b);
            DebugSocket.this.originoutput.write(b);
            DebugSocket.this.originoutput.flush();
        }

        @Override
        public void write(byte[] b, int off, int len) throws IOException
        {
            DebugSocket.this.outputdebugbuffer.write(b, off, len);
            DebugSocket.this.originoutput.write(b, off, len);
            DebugSocket.this.originoutput.flush();
        }
    }


    private static final Object
        staticsynch = new Object();

    private static long
        idcounter   = 0;


    private final long
        id;

    private final ByteArrayOutputStream
        inputdebugbuffer,
        outputdebugbuffer;

    private final InputStream
        inputwrapper;

    private final OutputStream
        outputwrapper;

    private InputStream
        origininput;

    private OutputStream
        originoutput;


    public InputStream getInputStream() throws IOException
    {
        if ( origininput == null )
        {
            synchronized ( inputdebugbuffer )
            {
                if ( origininput == null )
                {
                    origininput = super.getInputStream();
                }
            }
        }

        return inputwrapper;
    }

    public OutputStream getOutputStream() throws IOException
    {
        if ( originoutput == null )
        {
            synchronized ( outputdebugbuffer )
            {
                if ( originoutput == null )
                {
                    originoutput    = super.getOutputStream();
                }
            }
        }

        return outputwrapper;
    }


    public DebugSocket()
    {
        id                  = getNextId();
        inputwrapper        = new InputStreamWrapper();
        outputwrapper       = new OutputStreamWrapper();
        inputdebugbuffer    = new ByteArrayOutputStream();
        outputdebugbuffer   = new ByteArrayOutputStream();
    }


    private static long getNextId()
    {
        synchronized ( staticsynch )
        {
            return ++idcounter;
        }
    }
}

import java.io.IOException;
import java.net.ServerSocket;


public class DebugServerSocket extends ServerSocket
{
    public DebugServerSocket() throws IOException
    {
        super();
    }

    public DebugSocket accept() throws IOException
    {
        DebugSocket s = new DebugSocket();

        implAccept(s);

        return s;
    }
}

The class DebugSocket takes notification of each interaction with the InputStream as well as OutputStream

Now when the issue occurs i always can see that bytes are missing.

Here an example:

The client send 1758 bytes. I got the 23 top bytes from the member outputdebugbuffer in the DebugSocket .

Bytes: 0,0,0,0,0,0,0,2,0,0,6,-46,31,-117,8,0,0,0,0,0,0,0,-83

The server received 227 Bytes. For debug issues i always do read the input stream till i get a -1, so that all bytes got proceeded. Now the 16 leading bytes on serverside that i got from the member inputdebugbuffer in the DebugSocket .

Bytes: 0,0,0,6,-46,31,-117,8,0,0,0,0,0,0,0,-83

As shown there are 7 bytes missing. the first 8 bytes are a long value this one i changed to a byte value for debugging. So i figured that the first byte is always correct.

If it were a failure in the code no request would be proceeded but as i said before this happens only to 5% of all connections at best.

Got anyone an idea whats going on here?

I also used the DataInputStream and DataOutputStream to send the data. I always flush after each write operation as you can see in the OutputStreamWrapper of the DebugSocket .

Do i miss something here?

If some other code is required i will try to post it.

PS The service is multi threaded and processes 100 request parallel. Also the clients are multi threaded and do 20 requests parallel. As said each request uses its one connection and closes this one right after the request got proceeded.

I hope someone got an idea on this matter.

Edit:

There is no main method to show that does anything like asked in the comments but here the the code blocks of the client and the server that are used.

Client: (Run parallel in 20 threads)

    public void sendRequest(long _requesttype, byte[] _bytes)
    {
        Socket              socket  = null;
        DataInputStream     input   = null;
        DataOutputStream    output  = null;
        InputStream         sinput  = null;
        OutputStream        soutput = null;

        try
        {
            socket  = new DebugSocket();

            socket.connect(serveraddress);

            sinput  = socket.getInputStream();
            soutput = socket.getOutputStream();

            input   = new DataInputStream(sinput);
            output  = new DataOutputStream(soutput);

            output.writeLong(_requesttype);

            output.flush();
            soutput.flush();

            output.write(_bytes);

            output.flush();
            soutput.flush();

            // wait for notification byte that service had received all data.
            input.readByte();
        }
        catch (IOException ex)
        {
            LogHelper.log(ex);
        }
        catch (Error err)
        {
            throw err;
        }
        finally
        {
            output.flush();
            soutput.flush();

            input.close();
            output.close();

            finishSocket(socket);
        }
    }

Server: (Run in a thread for each request. Up to 100 threads)

    public void proceedRequest(DebugSocket _socket)
    {
        DataInputStream     input   = null;
        DataOutputStream    output  = null;
        InputStream         sinput  = null;
        OutputStream        soutput = null;

        try
        {
            sinput  = _socket.getInputStream();
            soutput = _socket.getOutputStream();

            input   = new DataInputStream(sinput);
            output  = new DataOutputStream(soutput);

            RequestHelper.proceed(input.readLong(), input, output);

            // send notification byte to the client.
            output.writeByte(1);

            output.flush();
            soutput.flush();
        }
        catch (IOException ex)
        {
            LogHelper.log(ex);
        }
        catch (Error err)
        {
            throw err;
        }
        finally
        {
            output.flush();
            soutput.flush();

            input.close();
            output.close();
        }
    }

In the server code the readLong() already fails cause of the missing bytes.

Answer 1

My guess is there is a bug somewhere else in the code. I copied the DebugSocket class from the question and created a MCVE (see below). It works fine, I was unable to reproduce the "server cannot read long-value" problem. Try modifying the code below to include more of the your own code until you can reproduce the problem, that should give you an idea where to look for the underlying cause.

import java.io.*;
import java.net.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;

public class TestDebugSocket implements Runnable, Closeable {

    public static void main(String[] args) {

        TestDebugSocket m = new TestDebugSocket();
        try {
            m.run();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            m.close();
        }
    }

    final int clients = 20;
    final boolean useDebugSocket = true;
    final byte[] someBytes = new byte[1758];
    final ThreadPoolExecutor tp = (ThreadPoolExecutor) Executors.newCachedThreadPool();
    final AtomicLong clientId = new AtomicLong();
    final ConcurrentLinkedQueue<Closeable> closeables = new ConcurrentLinkedQueue<Closeable>();
    final long maxWait = 5_000L;
    final CountDownLatch serversReady = new CountDownLatch(clients);
    final CountDownLatch clientsDone = new CountDownLatch(clients);

    ServerSocket ss;
    int port;

    @Override public void run()  {

        try {
            ss = useDebugSocket ? new DebugServerSocket() : new ServerSocket();
            ss.bind(null);
            port = ss.getLocalPort();
            tp.execute(new SocketAccept());
            for (int i = 0; i < clients; i++) {
                ClientSideSocket css = new ClientSideSocket();
                closeables.add(css);
                tp.execute(css);
            }
            if (!clientsDone.await(maxWait, TimeUnit.MILLISECONDS)) {
                System.out.println("CLIENTS DID NOT FINISH");
            } else {
                System.out.println("Finished");
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            close();
        }
    } 

    @Override public void close() {

        try { if (ss != null) ss.close(); } catch (Exception ignored) {}
        Closeable c = null;
        while ((c = closeables.poll()) != null) {
            try { c.close(); } catch (Exception ignored) {}
        }
        tp.shutdownNow();
    }

    class DebugServerSocket extends ServerSocket {

        public DebugServerSocket() throws IOException {
            super();
        }

        @Override public DebugSocket accept() throws IOException {

            DebugSocket s = new DebugSocket();
            implAccept(s);
            return s;
        }
    }

    class SocketAccept implements Runnable {

        @Override public void run() {
            try {
                for (int i = 0; i < clients; i++) {
                    SeverSideSocket sss = new SeverSideSocket(ss.accept());
                    closeables.add(sss);
                    tp.execute(sss);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    class SeverSideSocket implements Runnable, Closeable {

        Socket s;
        public SeverSideSocket(Socket s) {
            this.s = s;
        }
        @Override public void run() {

            Long l = -1L;
            byte[] received = new byte[someBytes.length];
            try {
                DataInputStream in = new DataInputStream(s.getInputStream());
                DataOutputStream out = new DataOutputStream(s.getOutputStream());
                serversReady.countDown();
                if (!serversReady.await(maxWait, TimeUnit.MILLISECONDS)) {
                    System.out.println("CLIENTS DID NOT CONNECT ON TIME TO SERVER");
                }
                l = in.readLong();
                in.readFully(received);
                out.writeByte(1);
                out.flush();
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                // write to console at end to prevent synchronized socket I/O
                System.out.println("received long: " + l);
                close();
            }
        }

        @Override public void close() {
            TestDebugSocket.close(s);
            s = null;
        }
    }

    class ClientSideSocket implements Runnable, Closeable {

        Socket s;

        @SuppressWarnings("resource")
        @Override public void run() {

            Long l = -1L;
            Byte b = -1;
            try {
                s = useDebugSocket ? new DebugSocket() : new Socket();
                s.connect(new InetSocketAddress(port));
                DataInputStream in = new DataInputStream(s.getInputStream());
                DataOutputStream out = new DataOutputStream(s.getOutputStream());
                l = clientId.incrementAndGet();
                out.writeLong(l);
                out.write(someBytes);
                out.flush();
                b = in.readByte();
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                System.out.println("long send: " + l + ", result: " + b);
                close();
                clientsDone.countDown();
            }
        }

        @Override public void close() {
            TestDebugSocket.close(s);
            s = null;
        }
    }

    static void close(Socket s) {

        try { if (s != null) s.close(); } catch (Exception ignored) {}
    }
}

Answer 2

Ok im done with all possible ways to locate the cause. From my experience with socket programming and parallel processing i can say that there is no bug in the code itself. Sniffers as well tell me that. Something on my machine is messing with the transmission.

I deactivated all i could think of (firewall/antivir/malware scanner) but no effect.

Got someone an idea what else could mess with tcp packages?

Edit:

Ok i got it. AVG 2014 is messing. Jetzt deactivating the components did not work. In Options->Settings there is a menu point were you can deactivate the AVG-Protection.

Got someone knowledge on this topic?