Commit 37f49287 authored by Richard Levitte's avatar Richard Levitte Committed by Matt Caswell
Browse files

CAPI engine: add support for RSA_NO_PADDING 



Since the SSL code started using RSA_NO_PADDING, the CAPI engine became
unusable.  This change fixes that.

Fixes #7131

Reviewed-by: default avatarTim Hudson <tjh@openssl.org>
Reviewed-by: default avatarMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7174)
parent 512d8117

engines/e_capi.c

+21 −7
Original line number Diff line number Diff line
@@ -917,6 +917,7 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 
    unsigned char *tmpbuf;

    CAPI_KEY *capi_key;

    CAPI_CTX *ctx;

    DWORD flags = 0;

    DWORD dlen;



    if (flen <= 0)
@@ -932,13 +933,24 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 
        return -1;

    }



    if (padding != RSA_PKCS1_PADDING) {

    switch (padding) {

    case RSA_PKCS1_PADDING:

        /* Nothing to do */

        break;

#ifdef CRYPT_DECRYPT_RSA_NO_PADDING_CHECK

    case RSA_NO_PADDING:

        flags = CRYPT_DECRYPT_RSA_NO_PADDING_CHECK;

        break;

#endif

    default:

        {

            char errstr[10];

            BIO_snprintf(errstr, 10, "%d", padding);

            CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_UNSUPPORTED_PADDING);

            ERR_add_error_data(2, "padding=", errstr);

            return -1;

        }

    }



    /* Create temp reverse order version of input */

    if ((tmpbuf = OPENSSL_malloc(flen)) == NULL) {
@@ -950,14 +962,16 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 


    /* Finally decrypt it */

    dlen = flen;

    if (!CryptDecrypt(capi_key->key, 0, TRUE, 0, tmpbuf, &dlen)) {

    if (!CryptDecrypt(capi_key->key, 0, TRUE, flags, tmpbuf, &dlen)) {

        CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_DECRYPT_ERROR);

        capi_addlasterror();

        OPENSSL_cleanse(tmpbuf, dlen);

        OPENSSL_free(tmpbuf);

        return -1;

    } else {

        memcpy(to, tmpbuf, (flen = (int)dlen));

    }

    OPENSSL_cleanse(tmpbuf, flen);

    OPENSSL_free(tmpbuf);



    return flen;