Openssl and Windows CryptoAPI compatibility issue

Question

I have problem in encrypted communication between windows application and server app. Client app is running at Windows, written in C and uses CryptoAPI. Server application uses PHP and Openssl extension. As cipher algorithm AES-256-CBC was chosen. Using the same algorithm Openssl and CryptoAPI produces different results. The same problem I had with RC2-CBC algorithm. This online tool http://asecuritysite.com/Encryption/openssl generates the same result as openssl, so I conclude that bug in C code.

PHP code:

<?php
//$flag = OPENSSL_RAW_DATA;
$flag = false;
//this string will encode
$dataString = 'some data string';
$pass = "1234567812345678";
$method = "aes-256-cbc";

$iv = "Zievrs8NZievrs8N";
echo "original:\n";
var_dump($dataString);
$encryptedMessage = openssl_encrypt($dataString, $method, $pass, $flag, $iv);
echo "after encrypt:\n";
var_dump($encryptedMessage);
echo "vector:\n";
var_dump($iv);
$decryptedMessage = openssl_decrypt($encryptedMessage, $method, $pass, $flag, $iv);
echo "after decrypt:\n";
var_dump($decryptedMessage);

Output:

original:
string(16) "some data string"
after encrypt:
string(44) "9O8UAaRRCfneeRbRCeiYi9nOM8F2KA6gtkAsvPliUdA="
vector:
string(16) "Zievrs8NZievrs8N"
after decrypt:
string(16) "some data string"

C code:

BOOL SetKey(BYTE* szKey, DWORD dwKeySize, HCRYPTPROV* m_hProv, HCRYPTHASH* m_hHash, HCRYPTKEY* m_hKey)
{
    BOOL m_fOK= TRUE;
    if (*m_hProv == 0) {
        m_fOK = CryptAcquireContextA(m_hProv, NULL, 
            NULL, //MS_DEF_PROV_A, 
            PROV_RSA_AES, 
            CRYPT_VERIFYCONTEXT 
        );
    }
    if (m_fOK && (*m_hHash != 0)) {
        m_fOK = CryptDestroyHash(*m_hHash); 
        m_hHash = 0;
    }
    if (m_fOK && (*m_hHash == 0)) {
        m_fOK = CryptCreateHash(*m_hProv, CALG_SHA_256, 0, 0, m_hHash);
    }
    if (m_fOK) {
        m_fOK = CryptHashData(*m_hHash, (BYTE*)szKey, dwKeySize, 0);
    }
    if (m_fOK) {
        m_fOK = CryptDeriveKey(*m_hProv, CALG_AES_256, *m_hHash, CRYPT_EXPORTABLE | CRYPT_NO_SALT, m_hKey);
    }
    if (m_fOK) {
        DWORD mode = CRYPT_MODE_CBC;
        m_fOK = CryptSetKeyParam(*m_hKey, KP_MODE, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        BYTE iv[] = {'Z','i','e','v','r','s','8','N','Z','i','e','v','r','s','8','N',0};
        m_fOK = CryptSetKeyParam(*m_hKey, KP_IV, (BYTE*)iv, 0);
    }

    return m_fOK;
}

BOOL EncryptDecrypt(BYTE* pData, BYTE** pRes, DWORD* dwDataLen, BYTE* pKey, DWORD dwKeySize, BOOL fEncrypt)
{
    HCRYPTPROV m_hProv = 0;
    HCRYPTHASH m_hHash = 0;
    HCRYPTKEY  m_hKey  = 0;

    BOOL m_fOK= TRUE;
    m_fOK = SetKey(pKey, dwKeySize, &m_hProv, &m_hHash, &m_hKey);
    if (fEncrypt) {
        DWORD dwTotalBufferSize = 0;
        DWORD dwNewLen = *dwDataLen;
        if((m_fOK = CryptEncrypt(m_hKey, 0, TRUE, 0, NULL, &dwNewLen, dwTotalBufferSize))) {
            *pRes = (BYTE*)malloc(dwNewLen);
            memcpy(*pRes, pData, *dwDataLen);
            dwTotalBufferSize = dwNewLen;
            if(!(m_fOK = CryptEncrypt(m_hKey, 0, TRUE, 0, *pRes, dwDataLen, dwTotalBufferSize))) {
                free(*pRes);
                *pRes = NULL;
                *dwDataLen = 0;
            }
        }
    }
    else  {
        *pRes = (BYTE*)malloc(*dwDataLen);
        memcpy(*pRes, pData, *dwDataLen);
        if(!(m_fOK = CryptDecrypt(m_hKey, 0, TRUE, 0, *pRes, dwDataLen))) {
            DWORD err = GetLastError();
            char msg[100];
            wsprintfA(msg, "err = %d\n", err);
            OutputDebugStringA(msg);
            free(*pRes);
            *pRes = NULL;
            *dwDataLen = 0;
        }
    }

    if (m_hKey)  CryptDestroyKey(m_hKey); 
    if (m_hHash) CryptDestroyHash(m_hHash); 
    if (m_hProv) CryptReleaseContext(m_hProv, 0); 

    return m_fOK;
}

void main() {
    const char* data = "some data string";
    BYTE* res = NULL;
    DWORD len = strlen(data);
    EncryptDecrypt((BYTE*)data, &res, &len, (BYTE*)"1234567812345678", 16, TRUE);
    size_t len_en = 0;
    char* base64 = base64_encode(res, len, &len_en);
    printf("base64 = %s\n", base64);
}

Output:

base64 = miFMwk4/ZwjMLsnV4Po9UdWxix32TrK5BcSgSKYr384=

Answer 1

Encrypted output is different. It means that key which is ultimately used is different or data is different. But data is same, hence key must be different.

It means in the process of key generation, something is different. It might be possible that OpenSSL may be using some other key deriving function which is not visible here. Try to use some standard algorithm for key generation. Instead of hashing, try not to use hash.

Answer 2

I finally got proper results from CryptoAPI by importing key as PLAINTEXTBLOB.

Here is not final, but workable fixes to SetKey function.

#include <WinCrypt.h>

typedef struct {
  BLOBHEADER hdr;
  DWORD      dwKeySize;
  BYTE       rgbKeyData[16];
} PLAINTEXTKEYBLOB_t;

BOOL SetKey(BYTE* szKey, DWORD dwKeySize, HCRYPTPROV* m_hProv, HCRYPTHASH* m_hHash, HCRYPTKEY* m_hKey)
{
    BOOL m_fOK= TRUE;
    if (*m_hProv == 0) {
        m_fOK = CryptAcquireContextW(m_hProv, NULL, 
            MS_ENH_RSA_AES_PROV, //MS_DEF_PROV_A, 
            PROV_RSA_AES, 
            CRYPT_NEWKEYSET | CRYPT_VERIFYCONTEXT 
        );
    }
    if (m_fOK && (*m_hHash != 0)) {
        m_fOK = CryptDestroyHash(*m_hHash); 
        m_hHash = 0;
    }

    if(m_fOK) {
        BYTE key[] = {0x06, 0xa9, 0x21, 0x40, 0x36, 0xb8, 0xa1, 0x5b, 0x51, 0x2e, 0x03, 0xd5, 0x34, 0x12, 0x00, 0x06};

        PLAINTEXTKEYBLOB_t blob;
        blob.hdr.bType = PLAINTEXTKEYBLOB;
        blob.hdr.bVersion = 2;
        blob.hdr.reserved = 0;
        blob.hdr.aiKeyAlg = CALG_AES_128;
        blob.dwKeySize = 16;
        for(int i=0; i<16; i++) {
            //blob.rgbKeyData[16-1-i] = key[i];
            blob.rgbKeyData[i] = key[i];
        }
        m_fOK = CryptImportKey(*m_hProv, (BYTE*)&blob, sizeof(PLAINTEXTKEYBLOB_t), 0, NULL, m_hKey);
    }

    if (m_fOK) {
        DWORD mode = CRYPT_MODE_CBC;
        m_fOK = CryptSetKeyParam(*m_hKey, KP_MODE, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        DWORD mode = 0;
        DWORD dwDataLen = sizeof(mode);
        m_fOK = CryptGetKeyParam(*m_hKey, KP_PADDING, (BYTE*)&mode, &dwDataLen, 0);
        mode = 0;
        //m_fOK = CryptSetKeyParam(*m_hKey, KP_PADDING, (BYTE*)&mode, 0);
    }
    if (m_fOK) {
        BYTE iv[] = {0x3d, 0xaf, 0xba, 0x42, 0x9d, 0x9e, 0xb4, 0x30, 0xb4, 0x22, 0xda, 0x80, 0x2c, 0x9f, 0xac, 0x41};
        m_fOK = CryptSetKeyParam(*m_hKey, KP_IV, (BYTE*)iv, 0);
    }

    return m_fOK;
}