[英]Increase the buffer size returned by SQL VDI (Virtual Device Interface) SQL Backup to Azure Blob
We have created a project based on this Code Project and upgraded to Visual Studio 2017. But we modified the C# DotNet part to backup to an Azure Block Blob.我们基于此代码项目创建了一个项目并升级到 Visual Studio 2017。但我们修改了 C# DotNet 部分以备份到 Azure 块 Blob。 It needs to be a Block Blob so as to add encryption later.它需要是一个 Block Blob,以便稍后添加加密。 The CPP DotNet code passes buffers to the C# portion in 64KB buffers. CPP DotNet 代码将缓冲区传递到 64KB 缓冲区中的 C# 部分。 The C# portion combines multiple 64KB blocks into one large 50MB block before uploading to Azure. C# 部分在上传到 Azure 之前将多个 64KB 块组合成一个 50MB 大块。 Combining the buffers is required due to the approx 50K limit on the number of blocks in an Azure Block Blob.由于 Azure 块 Blob 中的块数限制约为 50K,因此需要合并缓冲区。 Uploading as 64KB blocks would make us exceed that limit very quickly for a large database.上传为 64KB 块将使我们很快超过大型数据库的限制。
It all works like a charm.这一切都像一个魅力。 However it is painfully slow: about 3x longer to backup an 80GB database than the equivalent BACKUP TO URL
in Management Studio.然而,它非常缓慢:备份 80GB 数据库的时间比 Management Studio 中的等效BACKUP TO URL
长约 3 倍。
This is the command that is being passed down from the C# to the CPP VDI subsystem:这是从 C# 向下传递到 CPP VDI 子系统的命令:
BACKUP DATABASE [bench1] TO VIRTUAL_DEVICE='<deviceGUID>' WITH FORMAT, COMPRESSION
The performance drag seems to be in the combining of the 64KB blocks into 50MB blocks.性能拖累似乎在于将 64KB 块合并为 50MB 块。 My question: is there a way to force the VDI subsystem to return buffers than 64KB, or is that "built into the sauce"?我的问题:有没有办法强制 VDI 子系统返回超过 64KB 的缓冲区,还是“内置在酱汁中”?
Here is the very stripped down VDIDotNet.CPP code.这是非常精简的 VDIDotNet.CPP 代码。 For brevity most error handling has been removed.为简洁起见,大多数错误处理已被删除。
#include "vdi.h"
#include "vdierror.h"
#include "vdiguid.h"
using namespace System;
using namespace System::Data;
using namespace System::Data::Odbc;
using namespace System::Globalization;
using namespace System::IO;
using namespace System::Runtime::InteropServices;
using namespace System::Threading;
using namespace System::Reflection;
namespace VdiDotNet {
public ref class VdiEngine
{
private: Void SqlServerConnection_InfoMessage(Object^ sender, OdbcInfoMessageEventArgs^ e)
{
VdiDotNet::InfoMessageEventArgs^ i = gcnew VdiDotNet::InfoMessageEventArgs(e->Message);
InfoMessageReceived(this, i);
}
private: Void ThreadFunc(Object^ data)
{
try
{
String^ connString = "Driver={SQL Server Native Client 11.0};Server=(local);Trusted_Connection=Yes;";
//Create and configure an ODBC connection to the local SQL Server
OdbcConnection^ SqlServerConnection = gcnew OdbcConnection(connString);
SqlServerConnection->InfoMessage += gcnew OdbcInfoMessageEventHandler(this, &VdiDotNet::VdiEngine::SqlServerConnection_InfoMessage);
//Create and configure the command to be issued to SQL Server
OdbcCommand^ SqlServerCommand = gcnew OdbcCommand(data->ToString(), SqlServerConnection);
SqlServerCommand->CommandType = CommandType::Text;
SqlServerCommand->CommandTimeout = 0;
//Notify the user of the command issued
CommandIssued(this, gcnew CommandIssuedEventArgs(data->ToString()));
//Open the connection
SqlServerConnection->Open();
//Execute the command
SqlServerCommand->ExecuteNonQuery();
}
catch (Exception ^ex)
{
LogException(ex);
throw gcnew ApplicationException(ex->Message);
}
}
private: static Void ExecuteDataTransfer (IClientVirtualDevice* vd, Stream^ s)
{
VDC_Command * cmd;
DWORD completionCode;
DWORD bytesTransferred;
HRESULT hr;
while (SUCCEEDED(hr = vd->GetCommand(INFINITE, &cmd)))
{
array<System::Byte>^ arr = gcnew array<System::Byte>(cmd->size);
bytesTransferred = 0;
switch (cmd->commandCode)
{
case VDC_Read:
// ... stuff ...
case VDC_Write:
//Copy the data from the cmd object to a CLR array
Marshal::Copy((IntPtr)cmd->buffer, arr, 0, cmd->size);
//Write the data to the stream
s->Write(arr, 0, cmd->size);
//Set the number of bytes transferred
bytesTransferred = cmd->size;
//Set the completion code
completionCode = ERROR_SUCCESS;
break;
case VDC_Flush:
//Flush the stream
s->Flush();
//Set the completion code
completionCode = ERROR_SUCCESS;
break;
case VDC_ClearError:
//Set the completion code
completionCode = ERROR_SUCCESS;
break;
default:
//Set the completion code
completionCode = ERROR_NOT_SUPPORTED;
break;
}
//Complete the command
hr = vd->CompleteCommand(cmd, completionCode, bytesTransferred, 0);
}
}
public: Void ExecuteCommand(System::String^ command, Stream^ commandStream)
{
try
{
//Initialize COM
HRESULT hr = CoInitializeEx(NULL, COINIT_MULTITHREADED);
//Get an interface to the virtual device set
IClientVirtualDeviceSet2* vds = NULL;
hr = CoCreateInstance(CLSID_MSSQL_ClientVirtualDeviceSet, NULL, CLSCTX_INPROC_SERVER, IID_IClientVirtualDeviceSet2, (void**)&vds);
//Configure the device configuration
VDConfig config = { 0 };
vds->GetConfiguration(INFINITE, &config);
config.deviceCount = 1; //The number of virtual devices to create
//Create a name for the device using a GUID
String^ DeviceName = System::Guid::NewGuid().ToString()->ToUpper(gcnew CultureInfo("en-US"));
WCHAR wVdsName[37] = { 0 };
Marshal::Copy(DeviceName->ToCharArray(), 0, (IntPtr)wVdsName, DeviceName->Length);
//Create the virtual device set
hr = vds->CreateEx(NULL, wVdsName, &config);
//Format the command
command = String::Format(gcnew CultureInfo("en-US"), command, DeviceName);
//Create and execute a new thread to execute the command
Thread^ OdbcThread = gcnew Thread(gcnew ParameterizedThreadStart(this, &VdiDotNet::VdiEngine::ThreadFunc));
OdbcThread->Start(command);
//Configure the virtual device set
hr = vds->GetConfiguration(INFINITE, &config);
//Open the one device on the device set
IClientVirtualDevice* vd = NULL;
hr = vds->OpenDevice(wVdsName, &vd);
//Execute the data transfer
ExecuteDataTransfer(vd, commandStream);
//Wait for the thread that issued the backup / restore command to SQL Server to complete.
OdbcThread->Join();
}
catch (Exception ^ex)
{
LogException(ex);
throw gcnew ApplicationException(ex->Message);
}
}
};
}
And here is the very stripped down VDI.H这是非常精简的 VDI.H
#include "rpc.h"
#include "rpcndr.h"
#include "windows.h"
#include "ole2.h"
#pragma pack(8)
struct VDConfig
{
unsigned long deviceCount;
unsigned long features;
unsigned long prefixZoneSize;
unsigned long alignment;
unsigned long softFileMarkBlockSize;
unsigned long EOMWarningSize;
unsigned long serverTimeOut;
unsigned long blockSize;
unsigned long maxIODepth;
unsigned long maxTransferSize;
unsigned long bufferAreaSize;
} ;
enum VDCommands
{ VDC_Read = 1,
VDC_Write = ( VDC_Read + 1 ) ,
VDC_ClearError = ( VDC_Write + 1 ) ,
VDC_Rewind = ( VDC_ClearError + 1 ) ,
VDC_WriteMark = ( VDC_Rewind + 1 ) ,
VDC_SkipMarks = ( VDC_WriteMark + 1 ) ,
VDC_SkipBlocks = ( VDC_SkipMarks + 1 ) ,
VDC_Load = ( VDC_SkipBlocks + 1 ) ,
VDC_GetPosition = ( VDC_Load + 1 ) ,
VDC_SetPosition = ( VDC_GetPosition + 1 ) ,
VDC_Discard = ( VDC_SetPosition + 1 ) ,
VDC_Flush = ( VDC_Discard + 1 ) ,
VDC_Snapshot = ( VDC_Flush + 1 ) ,
VDC_MountSnapshot = ( VDC_Snapshot + 1 ) ,
VDC_PrepareToFreeze = ( VDC_MountSnapshot + 1 ) ,
VDC_FileInfoBegin = ( VDC_PrepareToFreeze + 1 ) ,
VDC_FileInfoEnd = ( VDC_FileInfoBegin + 1 )
} ;
struct VDC_Command
{
DWORD commandCode;
DWORD size;
DWORDLONG position;
BYTE *buffer;
} ;
Thanks for any suggestions.感谢您的任何建议。
I finally figured it out.我终于弄明白了。 Posting the answer just in case anyone else runs into a similar problem发布答案以防其他人遇到类似问题
I used MAXTRANSFERSIZE=4194304
(the max) on the BACKUP command, for example:我在 BACKUP 命令中使用了MAXTRANSFERSIZE=4194304
(最大值),例如:
BACKUP DATABASE [bench1] TO VIRTUAL_DEVICE='<deviceGUID>' WITH FORMAT, COMPRESSION,
MAXTRANSFERSIZE=4194304
After that, SQLVDI passed back buffers in 4MB per block.之后,SQLVDI 以每块 4MB 的形式传递回缓冲区。
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.