How to verify private RSA signed signature with corresponding X509 certificate

Question

I have generated private key - certificate pair with this command on linux:

openssl req -x509 -newkey rsa:1024 -keyout key.pem -out certificate.pem -days 730 -nodes

Now in C++ I want to sign some data with private key, SHA1, and then verify the signature using certificate: RSA_sign vs RSA_verify

Here is the complete code (error cases omitted) that I run:

#include <openssl/err.h>
#include <openssl/rsa.h>
#include <openssl/sha.h>
#include <openssl/pem.h>
#include <QByteArray>
#include <QString>

QByteArray sign(const QString& data)
{
    FILE * key_fd = NULL;
    EVP_PKEY * key = NULL;
    unsigned char * msg_to_sign = NULL;

    QString key_file = "key.pem";
    key_fd = fopen(key_file.toLatin1().data(), "rt");

    key = PEM_read_PrivateKey(key_fd, NULL, NULL, NULL);

    int data_len = data.size();
    msg_to_sign = new unsigned char[data_len];
    memcpy(msg_to_sign, data.toUtf8().constData(), data_len);

    unsigned char msg_to_sign_sha1[SHA_DIGEST_LENGTH];

    SHA1(msg_to_sign, data_len, msg_to_sign_sha1))

    unsigned char signed_msg_hash[KEY_LENGTH_IN_BITS] = {0};
    unsigned int signed_msg_hash_length = 0;
    int ret = RSA_sign(NID_sha1,
                       msg_to_sign_sha1, SHA_DIGEST_LENGTH,
                       signed_msg_hash, &signed_msg_hash_length,
                       key->pkey.rsa);

    QByteArray ba;
    ba.append((char *) signed_msg_hash);
    return ba;
}

bool verify(QString& data, QByteArray& signature)
{
    FILE * certificate_fd = NULL;
    unsigned char * msg_to_verify = NULL;
    unsigned char * msg_signature = NULL;

    QString cert_file("certificate.pem");
    certificate_fd = fopen(cert_file.toLatin1().data(), "rt");

    X509 * cert = PEM_read_X509(certificate_fd, NULL, NULL, NULL);

    EVP_PKEY * evp_pubkey;
    evp_pubkey  = X509_get_pubkey(cert);

    RSA * rsa_pubkey;
    rsa_pubkey  = EVP_PKEY_get1_RSA(evp_pubkey);

    int signature_len = signature.size();
    msg_signature = new unsigned char[signature_len];
    memcpy(msg_signature, signature.constData(), signature_len);

    int data_len = data.toUtf8().size();
    msg_to_verify = new unsigned char[data_len];
    memcpy(msg_to_verify, data.toUtf8().constData(), data_len);

    int RESULT = RSA_verify(NID_sha1, msg_to_verify, data_len, msg_signature, 
    signature_len, rsa_pubkey);

    return RESULT == 1;
}


int main()
{
    QString dataToSign("a");
    QByteArray signature = sign(dataToSign);

    bool result =  verify(dataToSign, signature);
    // RESULT is FALSE

    return 0;
}

The result of verification is FALSE .

ERR_get_error() returns 67702888 .

What am I doing wrong?

Answer 1

I was signing msg-SHA1 and verifying only msg. When they are consistent, it is OK now.