Increase the buffer size returned by SQL VDI (Virtual Device Interface) SQL Backup to Azure Blob

Question

We have created a project based on this Code Project and upgraded to Visual Studio 2017. But we modified the C# DotNet part to backup to an Azure Block Blob. It needs to be a Block Blob so as to add encryption later. The CPP DotNet code passes buffers to the C# portion in 64KB buffers. The C# portion combines multiple 64KB blocks into one large 50MB block before uploading to Azure. Combining the buffers is required due to the approx 50K limit on the number of blocks in an Azure Block Blob. Uploading as 64KB blocks would make us exceed that limit very quickly for a large database.

It all works like a charm. However it is painfully slow: about 3x longer to backup an 80GB database than the equivalent BACKUP TO URL in Management Studio.

This is the command that is being passed down from the C# to the CPP VDI subsystem:

BACKUP DATABASE [bench1] TO VIRTUAL_DEVICE='<deviceGUID>' WITH FORMAT, COMPRESSION

The performance drag seems to be in the combining of the 64KB blocks into 50MB blocks. My question: is there a way to force the VDI subsystem to return buffers than 64KB, or is that "built into the sauce"?

Here is the very stripped down VDIDotNet.CPP code. For brevity most error handling has been removed.

#include "vdi.h"
#include "vdierror.h"
#include "vdiguid.h"

using namespace System;
using namespace System::Data;
using namespace System::Data::Odbc;
using namespace System::Globalization;
using namespace System::IO;
using namespace System::Runtime::InteropServices;
using namespace System::Threading;
using namespace System::Reflection;

namespace VdiDotNet {

    public ref class VdiEngine
    {
        private: Void SqlServerConnection_InfoMessage(Object^ sender, OdbcInfoMessageEventArgs^ e)
                {
                    VdiDotNet::InfoMessageEventArgs^  i = gcnew VdiDotNet::InfoMessageEventArgs(e->Message);
                    InfoMessageReceived(this, i);
                }
        private: Void ThreadFunc(Object^ data)
        {
            try
            {
                String^ connString = "Driver={SQL Server Native Client 11.0};Server=(local);Trusted_Connection=Yes;";

                //Create and configure an ODBC connection to the local SQL Server
                OdbcConnection^ SqlServerConnection = gcnew OdbcConnection(connString);
                SqlServerConnection->InfoMessage += gcnew OdbcInfoMessageEventHandler(this, &VdiDotNet::VdiEngine::SqlServerConnection_InfoMessage);

                //Create and configure the command to be issued to SQL Server
                OdbcCommand^ SqlServerCommand = gcnew OdbcCommand(data->ToString(), SqlServerConnection);
                SqlServerCommand->CommandType = CommandType::Text;
                SqlServerCommand->CommandTimeout = 0;

                //Notify the user of the command issued
                CommandIssued(this, gcnew CommandIssuedEventArgs(data->ToString()));

                //Open the connection
                SqlServerConnection->Open();

                //Execute the command
                SqlServerCommand->ExecuteNonQuery();
            }
            catch (Exception ^ex)
            {
                LogException(ex);
                throw gcnew ApplicationException(ex->Message);
            }
        }

        private: static Void ExecuteDataTransfer (IClientVirtualDevice* vd, Stream^ s)
        {
            VDC_Command *   cmd;
            DWORD           completionCode;
            DWORD           bytesTransferred;
            HRESULT         hr;

            while (SUCCEEDED(hr = vd->GetCommand(INFINITE, &cmd)))
            {
                array<System::Byte>^ arr = gcnew array<System::Byte>(cmd->size);
                bytesTransferred = 0;
                switch (cmd->commandCode)
                {
                    case VDC_Read:
                        // ... stuff ...
                    case VDC_Write:
                        //Copy the data from the cmd object to a CLR array
                        Marshal::Copy((IntPtr)cmd->buffer, arr, 0, cmd->size);

                        //Write the data to the stream
                        s->Write(arr, 0, cmd->size);

                        //Set the number of bytes transferred
                        bytesTransferred = cmd->size;

                        //Set the completion code
                        completionCode = ERROR_SUCCESS;
                        break;

                    case VDC_Flush:
                        //Flush the stream
                        s->Flush();

                        //Set the completion code
                        completionCode = ERROR_SUCCESS;
                        break;

                    case VDC_ClearError:
                        //Set the completion code
                        completionCode = ERROR_SUCCESS;
                        break;

                    default:
                        //Set the completion code
                        completionCode = ERROR_NOT_SUPPORTED;
                        break;
                }

                //Complete the command
                hr = vd->CompleteCommand(cmd, completionCode, bytesTransferred, 0);
            }
        }

        public: Void ExecuteCommand(System::String^ command, Stream^ commandStream)
        {
            try
            {
                //Initialize COM
                HRESULT hr = CoInitializeEx(NULL, COINIT_MULTITHREADED);
                //Get an interface to the virtual device set
                IClientVirtualDeviceSet2* vds = NULL;
                hr = CoCreateInstance(CLSID_MSSQL_ClientVirtualDeviceSet, NULL, CLSCTX_INPROC_SERVER, IID_IClientVirtualDeviceSet2, (void**)&vds);

                //Configure the device configuration
                VDConfig config = { 0 };
                vds->GetConfiguration(INFINITE, &config);
                config.deviceCount = 1; //The number of virtual devices to create

                //Create a name for the device using a GUID
                String^ DeviceName = System::Guid::NewGuid().ToString()->ToUpper(gcnew CultureInfo("en-US"));
                WCHAR wVdsName[37] = { 0 };
                Marshal::Copy(DeviceName->ToCharArray(), 0, (IntPtr)wVdsName, DeviceName->Length);

                //Create the virtual device set
                hr = vds->CreateEx(NULL, wVdsName, &config);

                //Format the command
                command = String::Format(gcnew CultureInfo("en-US"), command, DeviceName);

                //Create and execute a new thread to execute the command
                Thread^ OdbcThread = gcnew Thread(gcnew ParameterizedThreadStart(this, &VdiDotNet::VdiEngine::ThreadFunc));
                OdbcThread->Start(command);

                //Configure the virtual device set
                hr = vds->GetConfiguration(INFINITE, &config);

                //Open the one device on the device set
                IClientVirtualDevice* vd = NULL;
                hr = vds->OpenDevice(wVdsName, &vd);

                //Execute the data transfer
                ExecuteDataTransfer(vd, commandStream);

                //Wait for the thread that issued the backup / restore command to SQL Server to complete.
                OdbcThread->Join();
            }
            catch (Exception ^ex)
            {
                LogException(ex);
                throw gcnew ApplicationException(ex->Message);

            }
        }
    };
}

And here is the very stripped down VDI.H

#include "rpc.h"
#include "rpcndr.h"

#include "windows.h"
#include "ole2.h"

#pragma pack(8)
struct VDConfig
    {
    unsigned long deviceCount;
    unsigned long features;
    unsigned long prefixZoneSize;
    unsigned long alignment;
    unsigned long softFileMarkBlockSize;
    unsigned long EOMWarningSize;
    unsigned long serverTimeOut;
    unsigned long blockSize;
    unsigned long maxIODepth;
    unsigned long maxTransferSize;
    unsigned long bufferAreaSize;
    } ;

enum VDCommands
    {   VDC_Read    = 1,
    VDC_Write   = ( VDC_Read + 1 ) ,
    VDC_ClearError  = ( VDC_Write + 1 ) ,
    VDC_Rewind  = ( VDC_ClearError + 1 ) ,
    VDC_WriteMark   = ( VDC_Rewind + 1 ) ,
    VDC_SkipMarks   = ( VDC_WriteMark + 1 ) ,
    VDC_SkipBlocks  = ( VDC_SkipMarks + 1 ) ,
    VDC_Load    = ( VDC_SkipBlocks + 1 ) ,
    VDC_GetPosition = ( VDC_Load + 1 ) ,
    VDC_SetPosition = ( VDC_GetPosition + 1 ) ,
    VDC_Discard = ( VDC_SetPosition + 1 ) ,
    VDC_Flush   = ( VDC_Discard + 1 ) ,
    VDC_Snapshot    = ( VDC_Flush + 1 ) ,
    VDC_MountSnapshot   = ( VDC_Snapshot + 1 ) ,
    VDC_PrepareToFreeze = ( VDC_MountSnapshot + 1 ) ,
    VDC_FileInfoBegin   = ( VDC_PrepareToFreeze + 1 ) ,
    VDC_FileInfoEnd = ( VDC_FileInfoBegin + 1 ) 
    } ;

struct VDC_Command
    {
    DWORD commandCode;
    DWORD size;
    DWORDLONG position;
    BYTE *buffer;
    } ;

Thanks for any suggestions.

Answer 1

I finally figured it out. Posting the answer just in case anyone else runs into a similar problem

I used MAXTRANSFERSIZE=4194304 (the max) on the BACKUP command, for example:

BACKUP DATABASE [bench1] TO VIRTUAL_DEVICE='<deviceGUID>' WITH FORMAT, COMPRESSION, 
    MAXTRANSFERSIZE=4194304

After that, SQLVDI passed back buffers in 4MB per block.