從由 C++ 程序寫出的 C# 中的文件讀取二進制對象

Question

我正在嘗試從包含填充結構的非常大的文件中讀取對象，這些填充結構由 C++ 進程寫入其中。 我正在使用 memory map 大文件的示例並嘗試將數據反序列化為 object 但我現在可以看到它不會以這種方式工作。

如何從文件中提取所有對象以在 C# 中使用？ 我可能很遙遠，但我已經提供了代碼。 這些對象有一個 8 字節的毫秒成員，后跟 21 個 16 位整數，需要 6 字節的填充才能與 8 字節的邊界對齊。

[Serializable]
unsafe public struct DataStruct
{
    public UInt64 milliseconds;
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 21)]
    public fixed Int16 data[21];
    [MarshalAs(UnmanagedType.ByValArray, SizeConst = 3)]
    public fixed Int16 padding[3];

};

[Serializable]
public class DataArray
{
    public DataStruct[] samples;
}

public static class Helper
{
    public static Int16[] GetData(this DataStruct data)
    {
        unsafe
        {
            Int16[] output = new Int16[21];
            for (int index = 0; index < 21; ++index)
                output[index] = data.data[index];
            return output;
        }
    }
}

class FileThreadSupport
{
    struct DataFileInfo
    {
        public string path;
        public UInt64 start;
        public UInt64 stop;
        public UInt64 elements;
    };

    // Create our epoch timestamp
    private static readonly DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);

    // Output TCP client
    private Support.AsyncTcpClient output;

    // Directory which contains our data
    private string replay_directory;

    // Files to be read from
    private DataFileInfo[] file_infos;

    // Current timestamp of when the process was started
    UInt64 process_start = 0;

    // Object from current file
    DataArray current_file_data;

    // Offset into current files
    UInt64 current_file_index = 0;

    // Offset into current files
    UInt64 current_file_offset = 0;

    // Run flag
    bool run = true;

    public FileThreadSupport(ref Support.AsyncTcpClient output, ref Engine.A.Information info, ref Support.Configuration configuration)
    {
        // Set our output directory
        replay_directory = configuration.getString("replay_directory");
        if (replay_directory.Length == 0)
        {
            Console.WriteLine("Configuration does not provide a replay directory");
            return;
        }

        // Check the directory for playable files
        if(!loadDataDirectory(replay_directory))
        {
            Console.WriteLine("Replay directory {} did not have any valid files", replay_directory);
        }

        // Set the output TCP client
        this.output = output;
    }

    private bool loadDataDirectory(string directory)
    {
        string[] files = Directory.GetFiles(directory, "*.*", SearchOption.TopDirectoryOnly);
        file_infos = new DataFileInfo[files.Length];
        int index = 0;
        foreach (string file in files)
        {
            string[] parts = file.Split('\\');
            string name = parts.Last();
            parts = name.Split('.');
            if (parts.Length != 2)
                continue;
            UInt64 start, stop = 0;
            if (!UInt64.TryParse(parts[0], out start) || !UInt64.TryParse(parts[1], out stop))
                continue;

            long size = new System.IO.FileInfo(file).Length;

            // Add to our file info array
            file_infos[index] = new DataFileInfo
            {
                path = file,
                start = start,
                stop = stop,
                elements = (ulong)(new System.IO.FileInfo(file).Length / 56 
                /*System.Runtime.InteropServices.Marshal.SizeOf(typeof(DataStruct))*/)
            };
            ++index;
        }

        // Sort the array
        Array.Sort(file_infos, delegate (DataFileInfo x, DataFileInfo y) { return x.start.CompareTo(y.start); });

        // Return whether or not there were files found
        return (files.Length > 0);
    }

    public void start()
    {
        process_start = (ulong)DateTime.Now.ToUniversalTime().Subtract(epoch).TotalMilliseconds;
        UInt64 num_samples = 0;

        while(run)
        {
            // Get our samples and add it to the sample
            DataStruct[] result = getData(100);
            Engine.A.A message = new Engine.A.A();
            for (int i = 0; i < result.Length; ++i)
            {
                Engine.A.Data sample = new Engine.A.Data();
                sample.Time = process_start + num_samples * 4;
                Int16[] signal_data = Helper.GetData(result[i]);
                for(int e = 0; e < signal_data.Length; ++e)
                    sample.Value[e] = signal_data[e];
                message.Signal.Add(sample);
                ++num_samples;
            }

            // Send out the websocket
            this.output.SendAsync(message.ToByteArray());

            // Sleep 100 milliseconds
            Thread.Sleep(100);
        }
    }

    public void stop()
    {
        run = false;
    }

    private DataStruct[] getData(UInt64 milliseconds)
    {
        if (file_infos.Length == 0)
            return new DataStruct[0];

        if (current_file_data == null)
        {
            current_file_data = ReadObjectFromMMF(file_infos[current_file_index].path) as DataArray;
            if(current_file_data.samples.Length == 0)
                return new DataStruct[0];
        }

        UInt64 elements_to_read = (UInt64) milliseconds / 4;
        DataStruct[] result = new DataStruct[elements_to_read];
        Array.Copy(current_file_data.samples, (int)current_file_offset, result, 0, (int) Math.Min(elements_to_read, file_infos[current_file_index].elements - current_file_offset));
        while((UInt64) result.Length != elements_to_read)
        {
            current_file_index = (current_file_index + 1) % (ulong) file_infos.Length;
            current_file_data = ReadObjectFromMMF(file_infos[current_file_index].path) as DataArray;
            if (current_file_data.samples.Length == 0)
                return new DataStruct[0];
            current_file_offset = 0;
            Array.Copy(current_file_data.samples, (int)current_file_offset, result, result.Length, (int)Math.Min(elements_to_read, file_infos[current_file_index].elements - current_file_offset));
        }
        return result;
    }

    private object ByteArrayToObject(byte[] buffer)
    {
        BinaryFormatter binaryFormatter = new BinaryFormatter(); // Create new BinaryFormatter
        MemoryStream memoryStream = new MemoryStream(buffer);    // Convert buffer to memorystream
        return binaryFormatter.Deserialize(memoryStream);        // Deserialize stream to an object
    }

    private object ReadObjectFromMMF(string file)
    {
        // Get a handle to an existing memory mapped file
        using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(file, FileMode.Open))
        {
            // Create a view accessor from which to read the data
            using (MemoryMappedViewAccessor mmfReader = mmf.CreateViewAccessor())
            {
                // Create a data buffer and read entire MMF view into buffer
                byte[] buffer = new byte[mmfReader.Capacity];
                mmfReader.ReadArray<byte>(0, buffer, 0, buffer.Length);

                // Convert the buffer to a .NET object
                return ByteArrayToObject(buffer);
            }
        }
    }

Answer 1

一方面，您根本沒有很好地使用 memory 映射文件，您只是在緩沖區中按順序讀取它，這既不必要地低效，而且比您簡單地打開文件正常讀取要慢得多。 memory 映射文件的賣點是由操作系統的虛擬 memory 分頁支持的重復隨機訪問和隨機更新。

而且您絕對不需要閱讀 memory 中的整個文件，因為您的數據結構如此強大。 您確切知道要為一條記錄讀取多少字節： Marshal.SizeOf<DataStruct>() 。

然后你需要擺脫所有的序列化噪音。 同樣，您的數據是強類型的，只需閱讀它。 擺脫那些fixed的 arrays 並使用常規 arrays，您已經在指導編組器如何使用MarshalAs屬性讀取它們（好）。 這也擺脫了只是出於某種未知原因復制數組的助手 function。

您的讀取循環非常簡單：讀取一個條目的正確字節數，使用Marshal.PtrToStructure將其轉換為可讀結構並將其添加到列表中以在最后返回。 如果您可以使用 .Net Core 和Unsafe.As或Unsafe.Cast ，則可以加分。

編輯：不要使用object返回，您確切知道要返回的內容，將其寫下來。