Does async/await affects tcp server performance?

Question

I am creating a Tcp server in C# 5.0 and I am using the await keyword when calling tcpListener.AcceptTcpClientAsync and networkStream.ReadAsync

However when I check the CPU usage of my server with Process Explorer I have the following results:

Tcp Sync version: 10% CPU usage

Tcp Async Version: 30% CPU usage Half of the usage is kernel usage.

Moreover, I measured how many time I received data by adding a counter inside the while look of the network stream, and the async version loops 120,000 times while the sync version loops 2,500,000 times.

In term of message received/second the async version is 15% slower than the sync version when receiving messages from 3 different clients.

Why does the Async Version use a lot more CPU than the Sync version?

Is this because of the async/await keyword ?

Is this normal that an Async Tcp server is slower than its sync counterpart?

EDIT: Here is an example of the async tcp server code

public class AsyncTcpListener : ITcpListener
{ 
    private readonly ServerEndpoint _serverEndPoint;  // Custom class to store IpAddress and Port

    public bool IsRunning { get; private set; }

    private readonly List<AsyncTcpClientConnection> _tcpClientConnections = new List<AsyncTcpClientConnection>(); 

    private TcpListener _tcpListener;

    public AsyncTcpMetricListener()
    {
        _serverEndPoint = GetServerEndpoint();  
    }

    public async void Start()
    {
        IsRunning = true;

        RunTcpListener();
    }

    private void MessageArrived(byte[] buffer)
    { 
        // Deserialize
    }

    private void RunTcpListener(){
       _tcpListener = null;
        try
        {
            _tcpListener = new TcpListener(_serverEndPoint.IpAddress, _serverEndPoint.Port);
            _tcpListener.Start();
            while (true)
            {
                var tcpClient = await _tcpListener.AcceptTcpClientAsync().ConfigureAwait(false);
                var asyncTcpClientConnection = new AsyncTcpClientConnection(tcpClient,  MessageArrived);
                _tcpClientConnections.Add(asyncTcpClientConnection);
            }
        } 
        finally
        {
            if (_tcpListener != null)
                _tcpListener.Stop();

            IsRunning = false;
        }
    }

    public void Stop()
    {
        IsRunning = false; 
        _tcpListener.Stop();
        _tcpClientConnections.ForEach(c => c.Close());
    }
}

For each new client we create a new AsyncTcpConnection

public class AsyncTcpClientConnection
{ 
    private readonly Action<byte[]> _messageArrived;
    private readonly TcpClient _tcpClient; 

    public AsyncTcpClientConnection(TcpClient tcpClient, Action<byte[]> messageArrived)
    {
        _messageArrived = messageArrived;
        _tcpClient = tcpClient; 
        ReceiveDataFromClientAsync(_tcpClient); 
    }

    private async void ReceiveDataFromClientAsync(TcpClient tcpClient)
    {
        var readBuffer = new byte[2048];
        // PacketProtocol class comes from http://blog.stephencleary.com/2009/04/sample-code-length-prefix-message.html
        var packetProtocol = new PacketProtocol(2048);  
        packetProtocol.MessageArrived += _messageArrived;

        try
        {
            using (tcpClient)
            using (var networkStream = tcpClient.GetStream())
            {
                int readSize;
                while ((readSize = await networkStream.ReadAsync(readBuffer, 0, readBuffer.Length).ConfigureAwait(false)) != 0)
                {
                    packetProtocol.DataReceived(readBuffer, readSize); 
                }
            }
        } 
        catch (Exception ex)
        {
            // log
        } 
    } 

    public void Close()
    {
        _tcpClient.Close();
    }
}

EDIT2: Synchronous server

 public class TcpListener : ITcpListener
{  
    private readonly ObserverEndpoint _serverEndPoint; 
    private readonly List<TcpClientConnection> _tcpClientConnections = new List<TcpClientConnection>();

    private Thread _listeningThread;
    private TcpListener _tcpListener;
    public bool IsRunning { get; private set; }

    public TcpMetricListener()
    {
        _serverEndPoint = GetServerEndpoint();   

    }


    public void Start()
    {
        IsRunning = true;
        _listeningThread = BackgroundThread.Start(RunTcpListener);  
    }

    public void Stop()
    {
        IsRunning = false;

        _tcpListener.Stop();
        _listeningThread.Join();
        _tcpClientConnections.ForEach(c => c.Close());
    }

    private void MessageArrived(byte[] buffer)
    {
        // Deserialize
    }

    private void RunTcpListener()
    {
        _tcpListener = null;
        try
        {
            _tcpListener = new TcpListener(_serverEndPoint.IpAddress, _serverEndPoint.Port);
            _tcpListener.Start();
            while (true)
            {
                var tcpClient = _tcpListener.AcceptTcpClient();
                _tcpClientConnections.Add(new TcpClientConnection(tcpClient, MessageArrived));
            }
        } 
        finally
        {
            if (_tcpListener != null)
                _tcpListener.Stop();

            IsRunning = false;
        }
    }
}

And the connection

public class TcpClientConnection
{ 
    private readonly Action<byte[]> _messageArrived;
    private readonly TcpClient _tcpClient;
    private readonly Task _task; 
    public TcpClientConnection(TcpClient tcpClient,   Action<byte[]> messageArrived)
    {
        _messageArrived = messageArrived;
        _tcpClient = tcpClient; 
        _task = Task.Factory.StartNew(() => ReceiveDataFromClient(_tcpClient), TaskCreationOptions.LongRunning);

    }

    private void ReceiveDataFromClient(TcpClient tcpClient)
    {
        var readBuffer = new byte[2048];
        var packetProtocol = new PacketProtocol(2048);
        packetProtocol.MessageArrived += _messageArrived;


            using (tcpClient)
            using (var networkStream = tcpClient.GetStream())
            {
                int readSize;
                while ((readSize = networkStream.Read(readBuffer, 0, readBuffer.Length)) != 0)
                {
                    packetProtocol.DataReceived(readBuffer, readSize); 
                }
            } 
    }


    public void Close()
    {
        _tcpClient.Close();
        _task.Wait();
    }
}

Answer 1

I have also issues with an async and these are my findings: https://stackoverflow.com/a/22222578/307976

Also, I have an asynchronous TCP server/client using async example here that scales well.

Answer 2

Try the following implementation of ReceiveInt32Async and ReceiveDataAsync for receiving your length-prefixed messages directly, instead of using tcpClient.GetStream and networkStream.ReadAsync :

public static class SocketsExt
{
    static public async Task<Int32> ReceiveInt32Async(
        this TcpClient tcpClient)
    {
        var data = new byte[sizeof(Int32)];
        await tcpClient.ReceiveDataAsync(data).ConfigureAwait(false);
        return BitConverter.ToInt32(data, 0);
    }

    static public Task ReceiveDataAsync(
        this TcpClient tcpClient,
        byte[] buffer)
    {
        return Task.Factory.FromAsync(
            (asyncCallback, state) =>
                tcpClient.Client.BeginReceive(buffer, 0, buffer.Length, 
                    SocketFlags.None, asyncCallback, state),
            (asyncResult) =>
                tcpClient.Client.EndReceive(asyncResult), 
            null);
    }
}

See if this gives any improvements. On a side note, I also suggest making ReceiveDataFromClientAsync an async Task method and storing the Task it returns inside AsyncTcpClientConnection (for status and error tracking).