[英]How to create fast and efficient filestream writes on large sparse files
I have an application that writes large files in multiple segments. 我有一个应用程序,可以在多个段中写入大文件。 I use FileStream.Seek to position each wirte.
我使用FileStream.Seek来定位每个wirte。 It appears that when I call FileStream.Write at a deep position in a sparse file the write triggers a "backfill" operation (writeing 0s) on all preceding bytes which is slow.
看来,当我在稀疏文件中的深位置调用FileStream.Write时,write会在所有前面的字节上触发“回填”操作(写入0),这很慢。
Is there a more efficient way of handling this situation? 有没有更有效的方法来处理这种情况?
The below code demonstrates the problem. 以下代码演示了该问题。 The initial write takes about 370 MS on my machine.
初始写入在我的机器上大约需要370 MS。
public void WriteToStream()
{
DateTime dt;
using (FileStream fs = File.Create("C:\\testfile.file"))
{
fs.SetLength(1024 * 1024 * 100);
fs.Seek(-1, SeekOrigin.End);
dt = DateTime.Now;
fs.WriteByte(255);
}
Console.WriteLine(@"WRITE MS: " + DateTime.Now.Subtract(dt).TotalMilliseconds.ToString());
}
NTFS does support Sparse Files , however there is no way to do it in .net without p/invoking some native methods. NTFS确实支持稀疏文件 ,但是在没有p /调用某些本机方法的情况下,无法在.net中执行此操作。
It is not very hard to mark a file as sparse, just know once a file is marked as a sparse file it can never be converted back in to a non sparse file except by coping the entire file in to a new non sparse file. 将文件标记为稀疏文件并不是很难,只要知道一旦文件被标记为稀疏文件,它就永远不能转换回非稀疏文件,除非将整个文件复制到新的非稀疏文件中。
Example useage 用例示例
class Program
{
[DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
int dwIoControlCode,
IntPtr InBuffer,
int nInBufferSize,
IntPtr OutBuffer,
int nOutBufferSize,
ref int pBytesReturned,
[In] ref NativeOverlapped lpOverlapped
);
static void MarkAsSparseFile(SafeFileHandle fileHandle)
{
int bytesReturned = 0;
NativeOverlapped lpOverlapped = new NativeOverlapped();
bool result =
DeviceIoControl(
fileHandle,
590020, //FSCTL_SET_SPARSE,
IntPtr.Zero,
0,
IntPtr.Zero,
0,
ref bytesReturned,
ref lpOverlapped);
if(result == false)
throw new Win32Exception();
}
static void Main()
{
//Use stopwatch when benchmarking, not DateTime
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
using (FileStream fs = File.Create(@"e:\Test\test.dat"))
{
MarkAsSparseFile(fs.SafeFileHandle);
fs.SetLength(1024 * 1024 * 100);
fs.Seek(-1, SeekOrigin.End);
fs.WriteByte(255);
}
stopwatch.Stop();
//Returns 2 for sparse files and 1127 for non sparse
Console.WriteLine(@"WRITE MS: " + stopwatch.ElapsedMilliseconds);
}
}
Once a file has been marked as sparse it now behaves like you excepted it to behave in the comments too. 一旦文件被标记为稀疏文件,它现在的行为就像你排除了它在评论中的行为一样。 You don't need to write a byte to mark a file to a set size.
您不需要编写一个字节来将文件标记为设置大小。
static void Main()
{
string filename = @"e:\Test\test.dat";
using (FileStream fs = new FileStream(filename, FileMode.Create))
{
MarkAsSparseFile(fs.SafeFileHandle);
fs.SetLength(1024 * 1024 * 25);
}
}
Here is some code to use sparse files: 以下是使用稀疏文件的一些代码:
using System;
using System.ComponentModel;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using Microsoft.Win32.SafeHandles;
public static class SparseFiles
{
private const int FILE_SUPPORTS_SPARSE_FILES = 64;
private const int FSCTL_SET_SPARSE = 0x000900c4;
private const int FSCTL_SET_ZERO_DATA = 0x000980c8;
public static void MakeSparse(this FileStream fileStream)
{
var bytesReturned = 0;
var lpOverlapped = new NativeOverlapped();
var result = DeviceIoControl(
fileStream.SafeFileHandle,
FSCTL_SET_SPARSE,
IntPtr.Zero,
0,
IntPtr.Zero,
0,
ref bytesReturned,
ref lpOverlapped);
if (!result)
{
throw new Win32Exception();
}
}
public static void SetSparseRange(this FileStream fileStream, long fileOffset, long length)
{
var fzd = new FILE_ZERO_DATA_INFORMATION();
fzd.FileOffset = fileOffset;
fzd.BeyondFinalZero = fileOffset + length;
var lpOverlapped = new NativeOverlapped();
var dwTemp = 0;
var result = DeviceIoControl(
fileStream.SafeFileHandle,
FSCTL_SET_ZERO_DATA,
ref fzd,
Marshal.SizeOf(typeof(FILE_ZERO_DATA_INFORMATION)),
IntPtr.Zero,
0,
ref dwTemp,
ref lpOverlapped);
if (!result)
{
throw new Win32Exception();
}
}
public static bool SupportedOnVolume(string filename)
{
var targetVolume = Path.GetPathRoot(filename);
var fileSystemName = new StringBuilder(300);
var volumeName = new StringBuilder(300);
uint lpFileSystemFlags;
uint lpVolumeSerialNumber;
uint lpMaxComponentLength;
var result = GetVolumeInformationW(
targetVolume,
volumeName,
(uint)volumeName.Capacity,
out lpVolumeSerialNumber,
out lpMaxComponentLength,
out lpFileSystemFlags,
fileSystemName,
(uint)fileSystemName.Capacity);
if (!result)
{
throw new Win32Exception();
}
return (lpFileSystemFlags & FILE_SUPPORTS_SPARSE_FILES) == FILE_SUPPORTS_SPARSE_FILES;
}
[DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
int dwIoControlCode,
IntPtr InBuffer,
int nInBufferSize,
IntPtr OutBuffer,
int nOutBufferSize,
ref int pBytesReturned,
[In] ref NativeOverlapped lpOverlapped);
[DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool DeviceIoControl(
SafeFileHandle hDevice,
int dwIoControlCode,
ref FILE_ZERO_DATA_INFORMATION InBuffer,
int nInBufferSize,
IntPtr OutBuffer,
int nOutBufferSize,
ref int pBytesReturned,
[In] ref NativeOverlapped lpOverlapped);
[DllImport("kernel32.dll", EntryPoint = "GetVolumeInformationW")]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool GetVolumeInformationW(
[In] [MarshalAs(UnmanagedType.LPWStr)] string lpRootPathName,
[Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpVolumeNameBuffer,
uint nVolumeNameSize,
out uint lpVolumeSerialNumber,
out uint lpMaximumComponentLength,
out uint lpFileSystemFlags,
[Out] [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpFileSystemNameBuffer,
uint nFileSystemNameSize);
[StructLayout(LayoutKind.Sequential)]
private struct FILE_ZERO_DATA_INFORMATION
{
public long FileOffset;
public long BeyondFinalZero;
}
}
And sample code to test the above class. 以及用于测试上述类的示例代码。
class Program
{
static void Main(string[] args)
{
using (var fileStream = new FileStream("test", FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
fileStream.SetLength(1024 * 1024 * 128);
fileStream.MakeSparse();
fileStream.SetSparseRange(0, fileStream.Length);
}
}
}
Hope this helps 希望这可以帮助
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.