Why does CryptDecrpyt() from MS CryptoAPI not decrypt the first 16 bytes correctly on remote system?

Question

I was trying to come up with how to use MS CryptoAPI to create a session key and transfer to a remote so they can encrypt data being sent over a private network (so not worried about man in the middle, just packet sniffer). Also need to support Windows XP. I think I figured out most of it and built a class to do the things I need. However, when I encrypt 2604 bytes of plaintext to 2608 of ciphertext and then decrypt it on that same machine ("Machine A") which imported the session key it works fine, however on the other machine (the one that actually created the session key, "Machine B") decrypting that data doesn't properly decrypt the first 16 bytes (first block).

What I do is have Machine A send a public key over to Machine B, Machine B creates a random session key and then exports it out and sends it over to Machine A, Machine A imports that key.

Machine A                                   Machine B
------------------------------------------  -------------------------------------------
Initialize();                               Initialize();
                                            CreateRandomSessionKey();
ExportPublicKey();
                                            ExportSessionKey();
ImportSessionKey();
CalcSizeForInPlaceEncryption(4604);
[create 4608 byte buffer with plaintext]
EncryptData(); *In Place*
DecryptData(); *New Buffer*                 
                                            DecryptData(); *In Place*
[Decrypted data matches.]                   [Decrypted data first 16 bytes invalid.]
                                            [Input of encrypted data matchines Machine A]

Here's the class I used to do that:

Header File:

#include <wincrypt.h>

class CMSCryptoAPI
{
  protected:
    HCRYPTPROV m_hCryptProv=NULL;        // crypto provider
    HCRYPTKEY m_hSessionKey=NULL;        // the symmetric key (session key)
    DWORD m_dwSessionKeyBlockSize=0;     // block size for encryption
  
    HCRYPTKEY CreateExchangeKey();
    bool ExportKey(HCRYPTKEY hkey, HCRYPTKEY hexpkey, DWORD blobtype, BYTE** pblob, DWORD* blobsize);
    DWORD GetSessionKeyBlockSize();

  public:
    virtual ~CMSCryptoAPI();

    // initialize before using class object
    bool Initialize();
    // uninitialize the initialization
    bool Uninitialize();

    bool DestroySessionKey();
    bool CreateRandomSessionKey();
    bool ExportSessionKey(const BYTE* publickeyblob, DWORD publickeyblobsize, BYTE** pblob, DWORD* blobsize);
    bool ImportSessionKey(const BYTE* pkeyblob, DWORD keyblobsize);
    bool ExportPublicKey(BYTE** pblob, DWORD* blobsize);
    bool EncryptData(BYTE* ppaintext, DWORD plaintextsize, BYTE** pciphertext, DWORD* ciphertextsize);
    bool DecryptData(BYTE* pciphertext, DWORD ciphertextsize, BYTE** ppaintext, DWORD *plaintextsize);
    bool SetIV(BYTE* pbiv=NULL);

    DWORD CalcSizeForInPlaceEncryption(DWORD plaintextsize);
};

Source File:

// Link with the Advapi32.lib file.
#pragma comment (lib, "advapi32")

#define KEYLENGTHSHIFT 16
#define KEYLENGTH      4096

//-------------------------------------------------------------------------
// Purpose: Destructor
//
// Input:   
//
// Output:    
//
// Notes:   
//
CMSCryptoAPI::~CMSCryptoAPI()
{
  Uninitialize();
}

//-------------------------------------------------------------------------
// Purpose: Initialize the object
//
// Input:   na
//
// Output:  true/false restult
//
// Notes:   must be called first before using any of the other functions
//
bool CMSCryptoAPI::Initialize()
{
  // acquire the provider to use
  if (!CryptAcquireContext(&m_hCryptProv, NULL, MS_ENH_RSA_AES_PROV, PROV_RSA_AES, 0)) {
    // user may not have key set like the local system account so try machine set
    if (GetLastError()==NTE_BAD_KEYSET) {
      if (!CryptAcquireContext(&m_hCryptProv, NULL, MS_ENH_RSA_AES_PROV, PROV_RSA_AES, CRYPT_MACHINE_KEYSET)) {
        if (GetLastError()==NTE_BAD_KEYSET) {
          if (!CryptAcquireContext(&m_hCryptProv, NULL, MS_ENH_RSA_AES_PROV, PROV_RSA_AES, CRYPT_NEWKEYSET)) {
            CDebugPrint::DebugPrint(_T("Error %u during CryptAcquireContext for new keyset!\n"), GetLastError());
            return false;
          }
          else CDebugPrint::DebugPrint(_T("Created new keyset for CryptAcquireContext!\n"));
        }
        else {
          CDebugPrint::DebugPrint(_T("Error %u during CryptAcquireContext for machine set!\n"), GetLastError());
          return false;
        }
      }
      else CDebugPrint::DebugPrint(_T("Using the Machine Key Set\n"));
    }
    else {
      CDebugPrint::DebugPrint(_T("Error %u during CryptAcquireContext for user!\n"), GetLastError());
      return false;
    }
  }
  return true;
}

//-------------------------------------------------------------------------
// Purpose: Clean up object to release resources
//
// Input:   na
//
// Output:  true/false result
//
// Notes:   
//
bool CMSCryptoAPI::Uninitialize()
{
  if (DestroySessionKey()) {
  // now release crypto context
    if (m_hCryptProv) {
      if (!CryptReleaseContext(m_hCryptProv, 0)) {
        CDebugPrint::DebugPrint(_T("Error %u release context!\n"), GetLastError());
      }
      else m_hCryptProv=NULL;
    }
  }

  return m_hCryptProv==NULL;
}

//-------------------------------------------------------------------------
// Purpose: Destroy the session key if it exists
//
// Input:   na
//
// Output:  true / false result of it being destroyed
//
// Notes:   true is returned if it never existed
//
bool CMSCryptoAPI::DestroySessionKey()
{
  if (m_hSessionKey) {
    if (!CryptDestroyKey(m_hSessionKey)) {
      CDebugPrint::DebugPrint(_T("Error %u destroying session key!\n"), GetLastError());
    }
    else m_hSessionKey=NULL;
  }

  return m_hSessionKey==NULL;
}

//-------------------------------------------------------------------------
// Purpose: Create a session key using random data
//
// Input:   na
//
// Output:  true/false result
//
// Notes:   
//
bool CMSCryptoAPI::CreateRandomSessionKey()
{
  bool result=false;
  if (DestroySessionKey()) {
  // Create a random session key. 
    if (CryptGenKey(m_hCryptProv, CALG_AES_128, CRYPT_EXPORTABLE, &m_hSessionKey)) {
      // now populate IV
      result=SetIV();
    }
    else CDebugPrint::DebugPrint(_T("Error %u creating session key\n"), GetLastError());
  }

  return result;
}

//-------------------------------------------------------------------------
// Purpose: Import the session key from remote
//
// Input:   pkeyblob      - [i] the session key blob from remote
//          keyblobsize   - [i] size of the session key blob
//
// Output:  true/false result
//
// Notes:   
//
bool CMSCryptoAPI::ImportSessionKey(const BYTE* pkeyblob, DWORD keyblobsize)
{
  bool result=false;

  if (DestroySessionKey()) {
    // Get the handle to the exchange key. 
    HCRYPTKEY rsakeypair=CreateExchangeKey();
    if (rsakeypair) {
      // import the blob
      if (CryptImportKey(m_hCryptProv, pkeyblob, keyblobsize, rsakeypair, 0, &m_hSessionKey)) {
        result=true;
      }
      else CDebugPrint::DebugPrint(_T("Error %u importing session key\n"), GetLastError());
      // clean up
      if (!CryptDestroyKey(rsakeypair)) {
        CDebugPrint::DebugPrint(_T("Error %u destroying exchange key for import\n"), GetLastError());
      }
    }
  }

  return result;
}

//-------------------------------------------------------------------------
// Purpose: Export the session key for a remote to use
//
// Input:   publickeyblob       - [i] remote systems public key
//          publickeyblobsize   - [i] size of the public key
//          pblob               - [o] the created session key blob
//          blobsize            - [o] size of the created session key blob
//
// Output:  true/false result
//
// Notes:   
//
bool CMSCryptoAPI::ExportSessionKey(const BYTE* publickeyblob, DWORD publickeyblobsize, BYTE** pblob, DWORD *blobsize)
{
  // init output variables
  *pblob=NULL;
  *blobsize=0;

  bool result=false;

  // import the public key
  HCRYPTKEY hremotepubickey;
  if (!CryptImportKey(m_hCryptProv, publickeyblob, publickeyblobsize, NULL, AT_KEYEXCHANGE, &hremotepubickey)) {
    CDebugPrint::DebugPrint(_T("Error %u importing public key blob for keyexchange\n"), GetLastError());
  }
  else {
    // use that key to export the session key
    result=ExportKey(m_hSessionKey, hremotepubickey, SIMPLEBLOB, pblob, blobsize);

    // clean up
    if (!CryptDestroyKey(hremotepubickey)) {
      CDebugPrint::DebugPrint(_T("Error %u destroying remote public key\n"), GetLastError());
    }
  }
  return result;
}


//-------------------------------------------------------------------------
// Purpose: Create the RSA exchange key pair
//
// Input:   na
//
// Output:  the key handle or NULL if problem
//
// Notes:   Use CryptDestroyKey() when done with key
//
HCRYPTKEY CMSCryptoAPI::CreateExchangeKey()
{
  HCRYPTKEY rsakeypair=NULL;
  if (!CryptGetUserKey(m_hCryptProv, AT_KEYEXCHANGE, &rsakeypair)) {
    if (GetLastError()==NTE_NO_KEY) {
      // create a RSA private/public key pair
      CDebugPrint::DebugPrint(_T("Create RSA Key\n"));
      if (!CryptGenKey(m_hCryptProv, AT_KEYEXCHANGE, (KEYLENGTH<<KEYLENGTHSHIFT)|CRYPT_EXPORTABLE, &rsakeypair)) {
        CDebugPrint::DebugPrint(_T("Error %u creating RSA Key\n"), GetLastError());
      }
    }
    else CDebugPrint::DebugPrint(_T("Error %u getting user RSA key\n"), GetLastError());
  }

  return rsakeypair;
}


//-------------------------------------------------------------------------
// Purpose: Export a public key to a blob
//
// Input:   pbolb     - [o] the exported key blob
//          blobsize  - [o] size of the blob
//
// Output:  true/false result
//
// Notes:   The requester of a session key sends this over in order to
//          receive a protected session key
//
bool CMSCryptoAPI::ExportPublicKey(BYTE** pblob, DWORD* blobsize)
{
  bool result=false;

  // Get the handle to the exchange key. 
  HCRYPTKEY rsakeypair=CreateExchangeKey();
  if (rsakeypair) {
    // now export the public key
    result=ExportKey(rsakeypair, NULL, PUBLICKEYBLOB, pblob, blobsize);
    // clean up
    if (!CryptDestroyKey(rsakeypair)) {
      CDebugPrint::DebugPrint(_T("Error %u destroying rsa key\n"), GetLastError());
    }
  }

  return result;
}

//-------------------------------------------------------------------------
// Purpose: generic routine to export a key to a blob
//
// Input:   hkey        - [i] key to export
//          hexpkey     - [i] export key (remote public key)
//          blobtype    - [i] type of blob to create
//          pblob       - [o] blob created
//          blobsize    - [i] size of blob created
//
// Output:  true/false result
//
// Notes:   
//
bool CMSCryptoAPI::ExportKey(HCRYPTKEY hkey, HCRYPTKEY hexpkey, DWORD blobtype, BYTE** pblob, DWORD* blobsize)
{
  bool result=false;

  // get size of blob needed to export key
  if (!CryptExportKey(hkey, hexpkey, blobtype, 0, NULL, blobsize)) {
    CDebugPrint::DebugPrint(_T("Error %u getting size to export key with blob type %u\n"), GetLastError(), blobtype);
    // ensure stays zero
    *blobsize=0;
  }
  else {
    CDebugPrint::DebugPrint(_T("Export Key blob size %u\n"), *blobsize);
    // we have the size - create buffer for the exported session key
    if ((*pblob=new BYTE[*blobsize])!=NULL) {
      // success - now actually get the exported session key
      if (!CryptExportKey(hkey, hexpkey, blobtype, 0, *pblob, blobsize)) {
        CDebugPrint::DebugPrint(_T("Error %u exporting key with blob type %u\n"), GetLastError(), blobtype);
        // clean up
        delete[](*pblob);
        // return nothing
        *pblob=NULL;
        *blobsize=0;
      }
      else result=true;
    }
    else {
      CDebugPrint::DebugPrint(_T("Error allocating %u byte buffer to export key with blob type %u\n"), *blobsize, blobtype);
      *blobsize=0;
    }
  }
  return result;
}

//-------------------------------------------------------------------------
// Purpose: Encrypt a block of data
//
// Input:   pplaintext    - [i] plain text to encrypt
//          plaintextsize - [i] size of plain text
//          pciphertext   - [o][opt] buffer with cipher data
//          ciphertextsize- [io] size of cipher data
//
// Output:  true/false result
//
// Notes:   If pciphertext is NULL encryption is done in-place and the
//          ciphertextsize gives the total size of the input buffer
//
bool CMSCryptoAPI::EncryptData(BYTE* pplaintext, DWORD plaintextsize, BYTE** pciphertext, DWORD* ciphertextsize)
{
  bool result=false;

  if (pciphertext) {
    *ciphertextsize=0;
    *pciphertext=NULL;
    // Get the size of the output buffer needed
    if (CryptEncrypt(m_hSessionKey, NULL, TRUE, 0, NULL, ciphertextsize, 0)) {
      assert(*ciphertextsize>=plaintextsize);
      // create the output buffer
      if ((*pciphertext=new BYTE[*ciphertextsize])!=NULL) {
        // created buffer - copy over data
        memcpy(*pciphertext, pplaintext, plaintextsize);
      }
    }
  }
  else pciphertext=&pplaintext;

  if (*pciphertext!=NULL) {
    // do the encryption
    if (CryptEncrypt(m_hSessionKey, NULL, TRUE, 0, *pciphertext, &plaintextsize, *ciphertextsize)) {
      // ensure result of encryption matches size returned
      assert(plaintextsize==*ciphertextsize);
      result=true;
    }
    else CDebugPrint::DebugPrint(_T("Error %u encrypting data\n"), GetLastError());
  }
  else CDebugPrint::DebugPrint(_T("Unable to create buffer of %u bytes to encrypt data\n"), *ciphertextsize);
  
  return result;
}

//-------------------------------------------------------------------------
// Purpose: Decrypt a block of data
//
// Input:   pciphertext   - [i] buffer with cipher data
//          ciphertextsize- [i] size of cipher data
//          pplaintext    - [o][opt] plain text 
//          plaintextsize - [o] size of plain text
//
// Output:  true/false result
//
// Notes:   if plaintext is NULL the decryption occurs in-place
//
bool CMSCryptoAPI::DecryptData(BYTE* pciphertext, DWORD ciphertextsize, BYTE** pplaintext, DWORD* plaintextsize)
{
  bool result=false;

  *plaintextsize=0;
  if (pplaintext) {
    if ((*pplaintext=new BYTE[ciphertextsize])!=NULL) {
      // copy over data
      memcpy(*pplaintext, pciphertext, ciphertextsize);
    }
  }
  else pplaintext=&pciphertext;

  if (*pplaintext!=NULL) {
    // do the decryption
    *plaintextsize=ciphertextsize;
    if (CryptDecrypt(m_hSessionKey, NULL, TRUE, 0, *pplaintext, plaintextsize)) {
      result=true;
    }
    else CDebugPrint::DebugPrint(_T("Error %u decrypting data\n"), GetLastError());
  }
  else CDebugPrint::DebugPrint(_T("Unable to create buffer of %u bytes to decrypt data\n"), ciphertextsize);

  return result;
}

//-------------------------------------------------------------------------
// Purpose: Setup the IV of a key
//
// Input:   pbiv   - [i][opt] IV data to use
//
// Output:  true/false result
//
// Notes:   Random data is used if pbiv is not given
//
bool CMSCryptoAPI::SetIV(BYTE* pbiv)
{
  bool result=false;

  // check if we put in random data or use callers value
  if (pbiv==NULL) {
    // get block length
    DWORD dwblocklen=GetSessionKeyBlockSize();
    if (dwblocklen!=0) {
      // create buffer for random data
      BYTE* pbtemp;
      if ((pbtemp=new BYTE[dwblocklen])!=NULL) {
        // create random data
        if (CryptGenRandom(m_hCryptProv, dwblocklen, pbtemp)) {
          // set the IV data
          if (CryptSetKeyParam(m_hSessionKey, KP_IV, pbtemp, 0)) {
            result=true;
          }
          else CDebugPrint::DebugPrint(_T("Error %u on CryptSetKeyParam KP_IV\n"), GetLastError());
        }
        else CDebugPrint::DebugPrint(_T("Error %u on CryptGenRandom\n"), GetLastError());

        // clean up
        delete[] pbtemp;
      }
      else CDebugPrint::DebugPrint(_T("Unable to create buffer of %u bytes for rng\n"), dwblocklen);
    }
  }
  else {
    if (CryptSetKeyParam(m_hSessionKey, KP_IV, pbiv, 0)) {
      result=true;
    }
    else CDebugPrint::DebugPrint(_T("Error %u on CryptSetKeyParam KP_VI caller buffer\n"), GetLastError());
  }

  return result;
}

//-------------------------------------------------------------------------
// Purpose: Calculate size of buffer that can used for in place encryption
//
// Input:   plaintextsize  - [i] size of the plain text
//
// Output:  size of aligned buffer that can be used for in place encryption
//
// Notes:   
//
DWORD CMSCryptoAPI::CalcSizeForInPlaceEncryption(DWORD plaintextsize)
{
  // get the block size
  DWORD dwblocksize=GetSessionKeyBlockSize();
  // align up to it
  return (((plaintextsize+dwblocksize-1)/dwblocksize)*dwblocksize);
  // if always a factor of 2 then 
  // return ((plaintextsize+dwblocksize-1) & ~(dwblocksize-1));
}

//-------------------------------------------------------------------------
// Purpose: Get the block size for the session key
//
// Input:   na
//
// Output:  size of aligned buffer that can be used for in place encryption
//
// Notes:   
//
DWORD CMSCryptoAPI::GetSessionKeyBlockSize()
{
  // get block length
  DWORD dwblocklen=0;
  DWORD dwdatalen=sizeof(dwblocklen);
  if (CryptGetKeyParam(m_hSessionKey, KP_BLOCKLEN, (BYTE*) &dwblocklen, &dwdatalen, 0)) {
    // convert bits to bytes
    dwblocklen/=8;
  }
  else CDebugPrint::DebugPrint(_T("Error %u on CryptGetKeyParam KP_BLOCKLEN\n"), GetLastError());

  return dwblocklen;
}

Answer 1

Okay, I wondered if it could be the IV. I changed to not set it when creating the session key so it looks like:

bool CMSCryptoAPI::CreateRandomSessionKey()
{
  bool result=false;
  if (DestroySessionKey()) {
  // Create a random session key. 
    if (CryptGenKey(m_hCryptProv, CALG_AES_128, CRYPT_EXPORTABLE, &m_hSessionKey)) {
      // now populate IV
      //result=SetIV(); REMOVED
      result=true;
    }
    else CDebugPrint::DebugPrint(_T("Error %u creating session key\n"), GetLastError());
  }

  return result;
}

Now it works on both sides, but then that makes me wonder, should I be setting some fixed IV or does Windows handle that or is it always 0 or something like that?

By the way, enjoy the class for those like me who needed something, others may help improve on it. I tend to use nothrownew.obj but others probably using exception handling (which I'm not all that up to speed with).