有没有最有效的方法来在C＃中读取/写入最多10GB的二进制文件？

Question

We have a project where we need to migrate up to 10GB of binary files. The steps are 1) read file by message size 2) do some processing 3) either write the original message or processed message back to a new binary file.

for a 10GB file, after processing it becomes 14GB. Currently it takes nearly 2 hours.

I am wondering if I can do some IO trick to trim that time down.

using (FileStream fsInput =new FileStream(inputfilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
        {
            using (FileStream fsOutput = File.Create(outputfilename))
            {
                long total = fsInput.Length;
                long progress = 0;

                unsafe
                {
                    int hdrSize = sizeof(FullMessageHeader);
                    byte[] headerBuffer = new byte[hdrSize];

                    while (fsInput.Position < fsInput.Length)
                    {                         
                        progress += fsInput.Read(headerBuffer, 0, hdrSize);
                        int msgSize = 0;
                        fixed (byte* hdr = headerBuffer)
                        {
                            msgSize = *(int*)(hdr + MessageHeaderOffsets.Size);
                        }

                        byte[] msg = new byte[msgSize];
                        Buffer.BlockCopy(headerBuffer, 0, msg, 0, headerBuffer.Length);
                        fsInput.Position -= hdrSize;
                        progress += fsInput.Read(msg, 0, msgSize);

                        fixed (byte* ptr = msg)
                        {
                            byte[] ba = ProcessMessage(ptr);
                            if (ba.Length == 0)
                            {
                                fsOutput.Write(msg, 0, msg.Length);
                            }
                            else
                            {
                                fsOutput.Write(ba, 0, ba.Length);
                            }
                        }

                    }
                }
            }
        }

Answer 1

so finally as suggested by Georg, and some clean up of the above code, I was able to reduce the time from 2 hours to 10 minutes which is not the most optimum but acceptable.

        Dictionary<int, byte[]> byteDictionary = new Dictionary<int, byte[]>();
        using (FileStream fsInput =new FileStream(inputfilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
        {
            using (FileStream fsOutput = File.Create(outputfilename))
            {
                long total = fsInput.Length;
                long progress = 0;

                unsafe
                {
                    int hdrSize = sizeof(FullMessageHeader);
                    byte[] headerBuffer = new byte[hdrSize];

                    while (fsInput.Position < fsInput.Length)
                    {                         
                        progress += fsInput.Read(headerBuffer, 0, hdrSize);
                        int msgSize = 0;
                        fixed (byte* hdr = headerBuffer)
                        {
                            msgSize = *(int*)(hdr + MessageHeaderOffsets.Size);
                        }

                        byte[] msg = byteDictionary.ContainsKey(msgSize)
                            ? byteDictionary[msgSize]
                            : new byte[msgSize];
                        if (!byteDictionary.ContainsKey(msgSize))
                        {
                            byteDictionary[msgSize] = msg;
                        }
                        //byte[] msg = new byte[msgSize];
                        //Buffer.BlockCopy(headerBuffer, 0, msg, 0, headerBuffer.Length);
                        fsInput.Position -= hdrSize;
                        progress += fsInput.Read(msg, 0, msgSize);

                        fixed (byte* ptr = msg)
                        {
                            //fsOutput.Write(msg,0,msg.Length);
                            byte[] ba = ProcessMessage(ptr);
                            if (ba.Length == 0)
                            {
                                fsOutput.Write(msg, 0, msg.Length);
                            }
                            else
                            {
                                fsOutput.Write(ba, 0, ba.Length);
                            }
                        }

                    }
                }
            }
        }