如何将大型二进制文件读入结构中以提高性能？

Question

我有一些大的二进制文件，我将它们读入结构中，然后将这些结构添加到列表中，以便稍后在它们中进行遍历。 一切工作正常，但读入结构的速度比预期的慢。 以下是相关代码：

//128 bytes total
[StructLayout(LayoutKind.Sequential, Pack = 1, CharSet = CharSet.Ansi)]
public struct RAMPrec2
{
    public double now;                 // Days in double precision (see note 1)
    public int VehLatitude;            // Milliarcseconds (see note 2)
    public int VehLongitude;           // Milliarcseconds (see note 2)
    public short VehLatUncertainty;    // Meters as 16-bit int
    public short VehLonUncertainty;    // Meters as 16-bit int
    public short Reserved1;      // Meters as 16-bit int (see note 3)
    public short Reserved2;      // Meters as 16-bit int
    public float VehAltitude;          // Meters as a float
    public float VehFirstRet;          // Meters as a float
    public float VehDepth;             // Meters as a float
    public float VehSpeed;             // Knots as a float
    public float VehHeading;           // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehCourse;            // degrees as a float (e.g. 0.0 - 359.9999)
    public float VehRoll;              // degrees as a float (positive is counterclockwise roll)
    public float VehPitch;             // degrees as a float (negative is downward pitch)
    public float VehVOS;               // Meters/sec
    public int DisplayNorthing;        // Centimeters as 32-bit integer
    public int DisplayingEasting;      // Centimeters as 32-bit integer
    public int OriginalLatitude;       // Milliarcseconds as 32-bit integer
    public int OriginalLongitude;      // Milliarcseconds as 32-bit integer
    public int DeltaNorthing;          // Centimeters as 32-bit integer
    public int DeltaEasting;           // Centimeters as 32-bit integer
    public short FixFlags;             // 16 bit flags. (note 5)
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)] // 8 sets of 32-bit values,sensor specific (note 3)
    public int[] SData;
    public ushort DataAvailFlags;      // 16 bit mask field (note 4)
    public ushort QA_Flag;             // 16 bit mask field (note 6)
    public short EventFlag;            // 2-byte reserved field
    public float Reserved3;            // 4-byte reserved field
}

public Boolean ReadRampFileType2(List<string> rampPaths) //rampPaths is just a list of filepaths for each binary file to be read
{
    for (int i = 0; i < rampPaths.Count; i++)
    {
        try
        {
            using (var stream = new FileStream(rampPaths[i], FileMode.Open, FileAccess.Read, FileShare.None)) //open up a stream on the specified file
            {
                stream.Position = 8192; //skip header

                while (stream.Position < (stream.Length)) //while not at end of file
                {
                    RAMPrec2 ramp = ReadRecFromStream<RAMPrec2>(stream, Marshal.SizeOf(typeof(RAMPrec2))); //read in each record to the ramp struct
                    AddRecDataToListsType2(ramp, vehicles[i]);
                }
            }
        }
        catch (Exception e) //something went wrong
        {
            return false;
        }
    }
    return true;
}

private T ReadRecFromStream<T>(Stream stream, int size) where T : struct
{
    byte[] buffer = new byte[Marshal.SizeOf(typeof(T))];
    stream.Read(buffer, 0, size);
    GCHandle handle = GCHandle.Alloc(buffer, GCHandleType.Pinned);
    try
    {
        return (T)Marshal.PtrToStructure(handle.AddrOfPinnedObject(), typeof(T));
    }
    finally
    {
        handle.Free();
    }
}

//I have a vehicle class with a bunch of lists. This adds the data from the structs I read in to these lists
public void AddRecDataToListsType2(RAMPrec2 ramp, VehicleModel vehicle)
{
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

我遍历我拥有的二进制文件列表，并在每个文件上打开一个文件流，跳至二进制文件中第一个“结构”出现的位置，然后将其读入结构，然后将其字段添加到一堆列表，然后重复直到文件末尾。 这很慢，我觉得必须有一种更好的做事方法。

Answer 1

这是内存映射的理想选择。

无需读取内存中的结构，您可以在数据文件上创建映射视图，并像在内存中一样访问结构。

虽然我没有在C＃中使用内存映射，但是在不安全模式下，您可以直接在映射视图上使用RAMPrec2* ，并使用该指针来遍历记录/结构，就像在C ++中一样。 请参阅此处的示例。

您甚至可以省去单独列表的创建，因为可以通过映射视图直接访问所有字段。

Answer 2

尝试这个...

// this would be how to call...

foreach (var item in ReadFile<RAMPrec2>("testFile.bin", 0).Select((v, i) => new { Value = v, Index = i }))
{
    var vehicle = vehicles[item.Index];
    var ramp = item.Value;
    vehicle.NowTime.Add(ramp.now);
    vehicle.VehLat.Add(ramp.VehLatitude / MILLIARCTODEG);
    vehicle.VehLong.Add(ramp.VehLongitude / MILLIARCTODEG);
    vehicle.VehHead.Add(ramp.VehHeading);
    vehicle.VehSpeed.Add(ramp.VehSpeed);
    vehicle.VehTWD.Add(ramp.VehAltitude + ramp.VehDepth);
    vehicle.VehAlt.Add(ramp.VehAltitude);
    vehicle.VehDep.Add(ramp.VehDepth);
    vehicle.VehRoll.Add(ramp.VehRoll);
    vehicle.VehPit.Add(ramp.VehPitch);
}

这就是魔术发生的地方。

static IEnumerable<T> ReadFile<T>(string fileName, int offset) where T : struct
{
    using (var reader = File.OpenRead(fileName))
    {
        var sizeOf = Marshal.SizeOf(typeof(T));
        var ptr = Marshal.AllocHGlobal(sizeOf);
        try
        {
            reader.Position = offset;
            var fileLength = reader.Length + reader.Position;
            var buffer = new byte[sizeOf];
            for (var p = reader.Position; p < fileLength; p += sizeOf)
            {
                reader.Read(buffer, 0, sizeOf);
                Marshal.Copy(buffer, 0, ptr, sizeOf);
                var ret = Marshal.PtrToStructure(ptr, typeof(T));
                var str = (T)ret;
                yield return str;
            }
        }
        finally
        {
            Marshal.FreeHGlobal(ptr);
        }
    }
}