I have a serious issue with my .NET server application. In production, the application will reach 100% CPU usage and get stuck there until I restart the server. It seems completely random. Sometimes it will happen 10 minutes after I start the server. Sometimes a week after I start the server. There are no logs that indicate what causes it either. But I am guessing I wrote the TCP client/server wrong and there is some edge case that can cause this. I believe this issue didn't start happening until I added this TCP client/server. I say client and server because this class does both and I actually have two different server applications that use it to communicate to each other and they both experience this issue randomly (not at the same time). Also side note user clients from all over the world use this same TCP client/server class: Bridge
to connect to the server as well.
Is there anything I can do to try and figure out what's causing this? It's a .NET console app running on a Linux VM on Google Cloud Platform.
If you are knowledgable with .NET TCP classes then perhaps you can find an issue with this code?
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace SBCommon
{
public class Bridge<T> where T : struct, IConvertible
{
public Dictionary<int, BridgeHandler> bridgeHandlers;
public ClientHandler clientHandler;
TcpListener server = null;
public Bridge(int port, Dictionary<int, BridgeHandler> bridgeHandlers)
{
server = new TcpListener(IPAddress.Any, port);
server.Start();
this.bridgeHandlers = bridgeHandlers;
Logging.Info($"bridge listener on address {IPAddress.Any} port {port}");
}
public void StartListener()
{
try
{
Logging.Info($"Starting to listen for TCP connections");
while (true)
{
Logging.Debug("TCP client ready");
TcpClient client = server.AcceptTcpClient();
Logging.Debug("TCP client connected");
Thread t = new Thread(new ParameterizedThreadStart(HandleConnection));
t.Start(client);
}
}
catch (SocketException e)
{
Logging.Error($"SocketException: {e}");
server.Stop();
}
}
public void HandleConnection(object obj)
{
TcpClient client = (TcpClient)obj;
client.ReceiveTimeout = 10000; // adding this to see if it fixes the server crashes
var stream = client.GetStream();
try
{
if (stream.CanRead)
{
byte[] messageLengthBytes = new byte[4];
int v = stream.Read(messageLengthBytes, 0, 4);
if (v != 4)
{
// this is happening and then the server runs at over 100% so adding lots of logging to figure out what's happening
StringBuilder sb = new StringBuilder($"could not read incoming message. Read {v} bytes");
try
{
sb.Append($"\nfrom {(IPEndPoint)client.Client.RemoteEndPoint}");
}
catch (Exception e)
{
sb.Append($"\ncould not get client's IP address because {e}");
}
sb.Append($"\nclient.Available: {client.Available}");
sb.Append($"\nclient.SendBufferSize: {client.SendBufferSize}");
Logging.Error(sb.ToString());
stream.Close();
client.Close();
stream.Dispose();
client.Dispose();
return;
}
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
v = stream.Read(recievedData, readPos, size);
readPos += v;
}
Bits incoming = new Bits(recievedData);
incoming.InitReadableBuffer();
int packetType = incoming.ReadInt();
Bits outgoing;
if (packetType == int.MaxValue)
{
Logging.Info($"recieved client handler message");
outgoing = clientHandler(incoming, client);
}
else
{
if (bridgeHandlers.ContainsKey(packetType))
{
Logging.Info($"recieved {(T)(object)packetType}");
outgoing = bridgeHandlers[packetType](incoming);
}
else
{
Logging.Error($"recieved unhandled packetType {packetType}!!!!!");
return;
}
}
if (outgoing != null)
{
#region send response
byte[] sendData = new byte[outgoing.Length() + 4];
// first write the length of the message
BitConverter.GetBytes(outgoing.Length()).CopyTo(sendData, 0);
// then write the message
outgoing.ToArray().CopyTo(sendData, 4);
stream.Write(sendData, 0, sendData.Length);
outgoing.Dispose();
#endregion
}
else
{
byte[] sendData = new byte[4];
BitConverter.GetBytes(0).CopyTo(sendData, 0);
stream.Write(sendData, 0, sendData.Length);
}
}
else
{
Logging.Info("Sorry. You cannot read from this NetworkStream.");
}
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
stream.Close();
client.Close();
stream.Dispose();
client.Dispose();
}
}
public static void SendTCPmessageFireAndForget(IPEndPoint iPEndPoint, Bits bits)
{
Task.Run(() =>
{
SendTCPmessage(iPEndPoint, bits, out _);
bits.Dispose();
});
}
public static async Task<Bits> SendTCPmessageAsync(IPEndPoint iPEndPoint, Bits bits)
{
TcpClient client = new TcpClient();
client.Connect(iPEndPoint);
NetworkStream stream = client.GetStream();
stream.WriteTimeout = 5000;
stream.ReadTimeout = 5000;
// Send the message
byte[] bytes = new byte[bits.Length() + 4];
BitConverter.GetBytes(bits.Length()).CopyTo(bytes, 0); // write length of message
bits.ToArray().CopyTo(bytes, 4);
await stream.WriteAsync(bytes, 0, bytes.Length);
// Read the response
byte[] messageLengthBytes = new byte[4];
int v = await stream.ReadAsync(messageLengthBytes, 0, 4);
if (v != 4) throw new Exception("could not read incoming message");
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
if (messageLength > 0)
{
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
v = await stream.ReadAsync(recievedData, readPos, size);
readPos += v;
}
stream.Close();
client.Close();
bits = new Bits(recievedData);
}
else bits = null;
return bits;
}
public static void SendTCPmessage(IPEndPoint iPEndPoint, Bits bits, out Bits responseBits)
{
try
{
TcpClient client = new TcpClient();
client.Connect(iPEndPoint);
NetworkStream stream = client.GetStream();
stream.WriteTimeout = 50000;
stream.ReadTimeout = 50000;
// Send the message
byte[] bytes = new byte[bits.Length() + 4];
BitConverter.GetBytes(bits.Length()).CopyTo(bytes, 0); // write length of message
bits.ToArray().CopyTo(bytes, 4);
stream.Write(bytes, 0, bytes.Length);
// Read the response
byte[] messageLengthBytes = new byte[4];
if (stream.Read(messageLengthBytes, 0, 4) != 4) throw new Exception("could not read incoming message");
int messageLength = BitConverter.ToInt32(messageLengthBytes, 0);
if (messageLength > 0)
{
int readPos = 0;
byte[] recievedData = new byte[messageLength];
while (readPos < messageLength)
{
int size = Math.Min(messageLength - readPos, client.ReceiveBufferSize);
int v = stream.Read(recievedData, readPos, size);
readPos += v;
}
stream.Close();
client.Close();
responseBits = new Bits(recievedData);
}
else responseBits = null;
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
responseBits = null;
}
}
}
public delegate Bits BridgeHandler(Bits incoming);
public delegate Bits ClientHandler(Bits incoming, TcpClient client);
}
Notes:
Bits
is a class I use for serialization. StartListener
that I start a thread for every incoming connectionwhile (true)
and AcceptTcpClient
to accept tcp connections. But maybe I shouldn't be doing it that way?int
from every packet to determine what kind of packet it is.There is a lot wrong with your existing code, so it's hard to know what exactly is causing the issue.
async
and only use those, do not do sync-over-async.stream.Read
is actually returning the whole value in one call.using
in many places.bits.ToArray
is and how efficient it is.CancellationToken
to be able to cancel the operations.Your code should look something like this:
public class Bridge<T> : IDisposable where T : struct, IConvertible
{
public Dictionary<int, BridgeHandler> bridgeHandlers;
public ClientHandler clientHandler;
TcpListener server;
public Bridge(int port, Dictionary<int, BridgeHandler> bridgeHandlers)
{
server = new TcpListener(IPAddress.Any, port);
this.bridgeHandlers = bridgeHandlers;
Logging.Info($"bridge listener on address {IPAddress.Any} port {port}");
}
public async Task StartListener()
{
try
{
Logging.Info($"Starting to listen for TCP connections");
server.Start();
while (true)
{
Logging.Debug("TCP client ready");
TcpClient client = await server.AcceptTcpClientAsync();
Logging.Debug("TCP client connected");
Task.Run(async () => await HandleConnection(client));
}
}
catch (SocketException e)
{
Logging.Error($"SocketException: {e}");
}
finally
{
if (listener.Active)
server.Stop();
}
}
public async Task HandleConnection(TcpClient client)
{
using client;
client.ReceiveTimeout = 10000; // adding this to see if it fixes the server crashes
using var stream = client.GetStream();
try
{
var incoming = await ReadMessageAsync(stream);
incoming.InitReadableBuffer();
int packetType = incoming.ReadInt();
Bits outgoing;
if (packetType == int.MaxValue)
{
Logging.Info($"recieved client handler message");
outgoing = clientHandler(incoming, client);
}
else
{
if (bridgeHandlers.TryGetValue(packetType, handler))
{
Logging.Info($"recieved {(T)(object)packetType}");
outgoing = handler(incoming);
}
else
{
Logging.Error($"recieved unhandled packetType {packetType}!!!!!");
return;
}
}
using (outgoing);
await SendMessageAsync(stream, outgoing);
}
catch (Exception e)
{
Logging.Error($"Exception: {e}");
}
}
public static void SendTCPmessageFireAndForget(IPEndPoint iPEndPoint, Bits bits)
{
Task.Run(async () =>
{
using (bits)
await SendTCPmessageAsync(iPEndPoint, bits);
});
}
public static async Task<Bits> SendTCPmessageAsync(IPEndPoint iPEndPoint, Bits bits)
{
using TcpClient client = new TcpClient();
await client.ConnectAsync(iPEndPoint);
using NetworkStream stream = client.GetStream();
stream.WriteTimeout = 5000;
stream.ReadTimeout = 5000;
await SendMessageAsync(stream, bits);
return await ReadMessageAsync(stream);
}
}
private async Task SendMessageAsync(Stream stream, Bits message)
{
var lengthArray = message == null ? new byte[4] : BitConverter.GetBytes(bits.Length());
await stream.WriteAsync(lengthArray, 0, lengthArray.Length); // write length of message
if (message == null)
return;
var bytes = bits.ToArray();
await stream.WriteAsync(bytes, 0, bytes.Length);
}
private async Task<Bits> ReadMessageAsync(Stream stream)
{
var lengthArray = new byte[4];
await FillBuffer(stream, lengthArray);
int messageLength = BitConverter.ToInt32(lengthArray, 0);
if (messageLength == 0)
return null;
byte[] receivedData = new byte[messageLength];
await FillBuffer(stream, receivedData);
bits = new Bits(receivedData);
return bits;
}
private async Task FillBuffer(Stream stream, byte[] buffer)
{
int totalRead = 0;
int bytesRead = 0;
while (totalRead < buffer.Length)
{
var bytesRead = await stream.ReadAsync(lengthArray, totalRead, buffer.Length - totalRead);
totalRead += bytesRead;
if(bytesRead <= 0)
throw new Exception("Unexpected end of stream");
}
}
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.