C++ Read and use RSA Private key in OpenSSL

Question

Assume that there's a program which uses OpenSSL RSA for it's security and I have the private key that it uses to decrypt/encrypt it's data.

This is a quote from whom that wrote how that programs protocol exactly works :

For the session opening, the client generates a 256-bits AES session key, and a 128-bits AES session IV (Initialisation Vector).

The client sends a GSP SESSION INIT message to the server, encrypted with the client's RSA private key (that is, encrypted with the private key). The RSA private key is the same for all Garena Clients, has been ripped from the windows EXE, and is given at the end of this file :) Because of this, the encryption is worthless: if someone is able to sniff the GSP session, he can get the encrypted GSP SESSION INIT message, decrypt it with the RSA public key (which can be derived from the RSA private key very easily with OpenSSL), and so get the AES session key and IV.

After the GSP SESSION INIT message, all the other GSP messages are encrypted in AES CBC with the session key and IV. The key and IV stay the same during all the session, in both directions (that is, the IV used for the encryption of each subsequent message is really the starting IV and not the last ciphertext block like it should be).

As he mentioned I just need RSA for once (encrypt GSP SESSION INIT) and after that we communicate with AES CBC.

This is the SSL Class that I use : (This is just RSA Stuff)

CGarenaEncrypt::CGarenaEncrypt() {
localKeypair = NULL;
remotePubKey = NULL;

#ifdef PSUEDO_CLIENT
    genTestClientKey();
#endif

init();
}

int CGarenaEncrypt::rsaEncrypt(const unsigned char *msg, size_t msgLen, unsigned char **encMsg, unsigned char **ek, size_t *ekl, unsigned char **iv, size_t *ivl) {
size_t encMsgLen = 0;
size_t blockLen = 0;

*ek = (unsigned char*)malloc(EVP_PKEY_size(localKeypair));
*iv = (unsigned char*)malloc(EVP_MAX_IV_LENGTH);
if(*ek == NULL || *iv == NULL) return FAILURE;
*ivl = EVP_MAX_IV_LENGTH;

*encMsg = (unsigned char*)malloc(msgLen + EVP_MAX_IV_LENGTH);
if(encMsg == NULL) return FAILURE;

if(!EVP_SealInit(rsaEncryptCtx, EVP_aes_256_cbc(), ek, (int*)ekl, *iv, &localKeypair, 1)) {
    return FAILURE;
}

if(!EVP_SealUpdate(rsaEncryptCtx, *encMsg + encMsgLen, (int*)&blockLen, (const unsigned char*)msg, (int)msgLen)) {
    return FAILURE;
}
encMsgLen += blockLen;

if(!EVP_SealFinal(rsaEncryptCtx, *encMsg + encMsgLen, (int*)&blockLen)) {
    return FAILURE;
}
encMsgLen += blockLen;

EVP_CIPHER_CTX_cleanup(rsaEncryptCtx);

return (int)encMsgLen;
}

int CGarenaEncrypt::rsaDecrypt(unsigned char *encMsg, size_t encMsgLen, unsigned char *ek, size_t ekl, unsigned char *iv, size_t ivl, unsigned char **decMsg) {
size_t decLen = 0;
size_t blockLen = 0;
EVP_PKEY *key;

*decMsg = (unsigned char*)malloc(encMsgLen + ivl);
if(decMsg == NULL) return FAILURE;

#ifdef PSUEDO_CLIENT
    key = remotePubKey;
#else
    key = localKeypair;
#endif

if(!EVP_OpenInit(rsaDecryptCtx, EVP_aes_256_cbc(), ek, ekl, iv, key)) {
    return FAILURE;
}

if(!EVP_OpenUpdate(rsaDecryptCtx, (unsigned char*)*decMsg + decLen, (int*)&blockLen, encMsg, (int)encMsgLen)) {
    return FAILURE;
}
decLen += blockLen;

if(!EVP_OpenFinal(rsaDecryptCtx, (unsigned char*)*decMsg + decLen, (int*)&blockLen)) {
    return FAILURE;
}
decLen += blockLen;

EVP_CIPHER_CTX_cleanup(rsaDecryptCtx);

return (int)decLen;
}

int CGarenaEncrypt::readPrivateKey(FILE *fd) {
if(!PEM_read_PrivateKey(fd, &localKeypair, NULL, NULL))
    return FAILURE;

return SUCCESS;
}

int CGarenaEncrypt::init() {
// Initalize contexts
rsaEncryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));
aesEncryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));

rsaDecryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));
aesDecryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX));

// Always a good idea to check if malloc failed
if(rsaEncryptCtx == NULL || aesEncryptCtx == NULL || rsaDecryptCtx == NULL || aesDecryptCtx == NULL) {
    return FAILURE;
}

// Init these here to make valgrind happy
EVP_CIPHER_CTX_init(rsaEncryptCtx);
EVP_CIPHER_CTX_init(aesEncryptCtx);

EVP_CIPHER_CTX_init(rsaDecryptCtx);
EVP_CIPHER_CTX_init(aesDecryptCtx);

// Init RSA
EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);

if(EVP_PKEY_keygen_init(ctx) <= 0) {
    return FAILURE;
}

if(EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, RSA_KEYLEN) <= 0) {
    return FAILURE;
}

if(EVP_PKEY_keygen(ctx, &localKeypair) <= 0) {
    return FAILURE;
}

EVP_PKEY_CTX_free(ctx);

// Init AES
aesKey = (unsigned char*)malloc(AES_KEYLEN/8);
aesIV = (unsigned char*)malloc(AES_KEYLEN/16);

unsigned char *aesPass = (unsigned char*)malloc(AES_KEYLEN/8);
unsigned char *aesSalt = (unsigned char*)malloc(8);

if(aesKey == NULL || aesIV == NULL || aesPass == NULL || aesSalt == NULL) {
    return FAILURE;
}

// For the AES key we have the option of using a PBKDF (password-baswed key derivation formula)
// or just using straight random data for the key and IV. Depending on your use case, you will
// want to pick one or another.
#ifdef USE_PBKDF
    // Get some random data to use as the AES pass and salt
    if(RAND_bytes(aesPass, AES_KEYLEN/8) == 0) {
        return FAILURE;
    }

    if(RAND_bytes(aesSalt, 8) == 0) {
        return FAILURE;
    }

    if(EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha256(), aesSalt, aesPass, AES_KEYLEN/8, AES_ROUNDS, aesKey, aesIV) == 0) {
        return FAILURE;
    }
#else
    if(RAND_bytes(aesKey, AES_KEYLEN/8) == 0) {
        return FAILURE;
    }

    if(RAND_bytes(aesIV, AES_KEYLEN/16) == 0) {
        return FAILURE;
    }
#endif

free(aesPass);
free(aesSalt);

return SUCCESS;
}

And this is how I tried to do all the things :

if (!AfxSocketInit())
Util_Log(0, 2, _T("AfxSocketInit() failed."));

FILE* privKey;

int err  = fopen_s( &privKey, "E:\\gkey.pem", "r" );

if( err != 0 ){
    Util_Log(0, 2, _T("Failed to open 'gkey.pem' file."));
    return;
}

err = m_encrypt.readPrivateKey(privKey);

fclose(privKey);

if( err != 0 ){
    Util_Log(0, 2, _T("readPrivateKey Failed."));
    return;
}


sendGSPSessionInit();

bool CGarenaInterface::sendGSPSessionInit(void)
{
FILE *f;
if(AllocConsole()) {
freopen_s(&f, "CONOUT$", "wt", stdout);
SetConsoleTitle(_T("Debug Console"));
SetConsoleTextAttribute(GetStdHandle(STD_OUTPUT_HANDLE), FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED);
}

Util_Log(0, 0, _T("Sending GSP session init ..."));

int nEncSize = 0;
int nDecSize = 0;
uint8_t* EncOut = NULL;
uint8_t* DecOut = NULL;
unsigned char *ek;
unsigned char *iv;
size_t ekl;
size_t ivl;

CByteBuffer *bytebuffer = new CByteBuffer();
CByteBuffer *bytebuffer2 = new CByteBuffer();
bytebuffer->Allocate(50);
bytebuffer->PutBytes(m_encrypt.aesKey, 32);
bytebuffer->PutBytes(m_encrypt.aesIV, 16);
bytebuffer->PutShort(0xF00F);

printf("\n original:\n");
Util_HexPrint(*bytebuffer, bytebuffer->ByteBuffer->len);

nEncSize = m_encrypt.rsaEncrypt(*bytebuffer, bytebuffer->ByteBuffer->len, &EncOut, &ek, &ekl, &iv, &ivl);

printf("encrypted:\n");
Util_HexPrint(EncOut, nEncSize);

nDecSize = m_encrypt.rsaDecrypt(EncOut, nEncSize, ek, ekl, iv, ivl, &DecOut);

printf("decrypted:\n");
Util_HexPrint(DecOut, nDecSize);

bytebuffer2->Allocate(nEncSize + 6);
bytebuffer2->PutInt(258);
bytebuffer2->PutShort(0x00AD);
bytebuffer2->PutBytes(EncOut, nEncSize);

printf("\n final:\n");
Util_HexPrint(*bytebuffer2, bytebuffer2->ByteBuffer->len);

//  m_socket.Create();
//  m_socket.m_buffer = *bytebuffer2;
//  m_socket.nBufLen = bytebuffer2->ByteBuffer->len;
//  m_socket.Connect(GARENA_MAIN_SERVER_ADDRESS, 7456);


delete bytebuffer;
bytebuffer = NULL;
//  delete bytebuffer2;
//  bytebuffer2 = NULL;

free(EncOut);
free(DecOut);
free(ek);
free(iv);
EncOut = NULL;
DecOut = NULL;
ek = NULL;
iv = NULL;


return true;
}

As the picture shows everything works correctly BUT the problem is the server that I connect to, expects a packet with ~200-250 length WHILE mine is ~70. That means when they encrypt their data (which is same between what I use and what original client uses) the encrypted length will be a large number (~240). Also there is an alternative for the original client (What I am actually do) which is written in java generates the correct encrypted data with correct length.

GCB

I think there should be a problem while reading the private key because when I call
PEM_write_PrivateKey(stdout, localKeypair, NULL, NULL, 0, 0, NULL) The key that prints in console in completely different from the private key that I want to read OR I'm using wrong key length ?! I don't know.

Any help would be most welcome

Answer 1

first, you have wrong information. EVP_Seal... operation does not use rsa private encryption. EVP_Seal.. use rsa_public_encryption and EVP_Open.. use rsa_private_decryption.

seconds, size of encrypted key depends on RSA key size. in your code you use RSA_KEYLEN as key size. i think your server's key size maybe 2048(bits) long. you use 512(bits) long.