如何将大型二进制文件读入结构中以提高性能？

Question

I have some large binary files that I read into structs, and then add those structs to a list so I can loop through them at a later point. Everything works fine, except the reading into the structs goes slower than expected. Here's the relevant code:

//128 bytes total
[StructLayout(LayoutKind.Sequential, Pack = 1, CharSet = CharSet.Ansi)]
public struct RAMPrec2
{
    public double now;                 // Days in double precision (see note 1)
    public int VehLatitude;            // Milliarcseconds (see note 2)
    public int VehLongitude;           // Milliarcseconds (see note 2)
    public short VehLatUncertainty;    // Meters as 16-bit int
    public short VehLonUncertainty;    // Meters as 16-bit int
    public short Reserved1;      // Meters as 16-bit int (see note 3)
    public short Reserved2;      // Meters as 16-bit int
    public float VehAltitude;          // Meters as a float
    public float VehFirstRet;          // Meters as a float
    public float VehDepth;             // Meters as a float
    public float VehSpeed;             // Knots as a float
    public float VehHeading;           // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehCourse;            // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehRoll;              // degrees as a float (positive is counterclockwise roll)
    public float VehPitch;             // degrees as a float (negative is downward pitch)
    public float VehVOS;               // Meters/sec
    public int DisplayNorthing;        // Centimeters as 32-bit integer
    public int DisplayingEasting;      // Centimeters as 32-bit integer
    public int OriginalLatitude;       // Milliarcseconds as 32-bit integer
    public int OriginalLongitude;      // Milliarcseconds as 32-bit integer
    public int DeltaNorthing;          // Centimeters as 32-bit integer
    public int DeltaEasting;           // Centimeters as 32-bit integer
    public short FixFlags;             // 16 bit flags. (note 5)
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] // 8 sets of 32-bit values,sensor specific (note 3)
    public int[] SData;
    public ushort DataAvailFlags;      // 16 bit mask field (note 4)
    public ushort QA_Flag;             // 16 bit mask field (note 6)
    public short EventFlag;            // 2-byte reserved field
    public float Reserved3;            // 4-byte reserved field
}

public Boolean ReadRampFileType2(List<string> rampPaths) //rampPaths is just a list of filepaths for each binary file to be read
{
    for (int i = 0; i < rampPaths.Count; i++)
    {
        try
        {
            using (var stream = new FileStream(rampPaths[i], FileMode.Open, FileAccess.Read, FileShare.None)) //open up a stream on the specified file
            {
                stream.Position = 8192; //skip header

                while (stream.Position < (stream.Length)) //while not at end of file
                {
                    RAMPrec2 ramp = ReadRecFromStream<RAMPrec2>(stream, Marshal.SizeOf(typeof(RAMPrec2))); //read in each record to the ramp struct
                    AddRecDataToListsType2(ramp, vehicles[i]);
                }
            }
        }
        catch (Exception e) //something went wrong
        {
            return false;
        }
    }
    return true;
}

private T ReadRecFromStream<T>(Stream stream, int size) where T : struct
{
    byte[] buffer = new byte[Marshal.SizeOf(typeof(T))];
    stream.Read(buffer, 0, size);
    GCHandle handle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
    try
    {
        return (T)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(T));
    }
    finally
    {
        handle.Free();
    }
}

//I have a vehicle class with a bunch of lists. This adds the data from the structs I read in to these lists
public void AddRecDataToListsType2(RAMPrec2 ramp, VehicleModel vehicle)
{
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

I loop through the list of binary files I have and open a file stream on each one, skip to the position where the first 'struct' appears in the binary file, and then read that into a struct, which then has its fields added to a bunch of lists, then I repeat until the end of the file. This is slow, and I feel like there has to be a better way of doing things.

Answer 1

This is an ideal candidate for memory mapping.

Instead of reading the structs in memory, you can create a mapped view on your data file and access your structs as if they were in memory.

While I haven't used memory mapping in C#, in unsafe mode you could possibly use a RAMPrec2* directly on your mapped view and use that pointer to loop through your records/structs, just like in C++. See here for an example.

You could even eliminate the creation of the seperate lists, because all fields can be accessed directly throught the mapped view.

Answer 2

Try this...

// this would be how to call...

foreach (var item in ReadFile<RAMPrec2>("testFile.bin", 0).Select((v, i) => new { Value = v, Index = i }))
{
    var vehicle = vehicles[item.Index];
    var ramp = item.Value;
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

This is where the magic happens.

static IEnumerable<T> ReadFile<T>(string fileName, int offset) where T : struct
{
    using (var reader = File.OpenRead(fileName))
    {
        var sizeOf = Marshal.SizeOf(typeof(T));
        var ptr = Marshal.AllocHGlobal(sizeOf);
        try
        {
            reader.Position = offset;
            var fileLength = reader.Length + reader.Position;
            var buffer = new byte[sizeOf];
            for (var p = reader.Position; p < fileLength; p += sizeOf)
            {
                reader.Read(buffer, 0, sizeOf);
                Marshal.Copy(buffer, 0, ptr, sizeOf);
                var ret = Marshal.PtrToStructure(ptr, typeof(T));
                var str = (T)ret;
                yield return str;
            }
        }
        finally
        {
            Marshal.FreeHGlobal(ptr);
        }
    }
}