Commit fb953d29 authored by Richard Levitte's avatar Richard Levitte Committed by Matt Caswell
Browse files

CAPI engine: add support for RSA_NO_PADDING 



Since the SSL code started using RSA_NO_PADDING, the CAPI engine became
unusable.  This change fixes that.

Fixes #7131

Reviewed-by: default avatarBernd Edlinger <bernd.edlinger@hotmail.de>
Reviewed-by: default avatarTim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7132)
parent 4a98bb00

engines/e_capi.c

+24 −8
Original line number Diff line number Diff line
@@ -900,6 +900,8 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 
    unsigned char *tmpbuf;

    CAPI_KEY *capi_key;

    CAPI_CTX *ctx;

    DWORD flags = 0;



    ctx = ENGINE_get_ex_data(rsa->engine, capi_idx);



    CAPI_trace(ctx, "Called capi_rsa_priv_dec()\n");
@@ -910,13 +912,24 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 
        return -1;

    }



    if (padding != RSA_PKCS1_PADDING) {

    switch (padding) {

    case RSA_PKCS1_PADDING:

        /* Nothing to do */

        break;

#ifdef CRYPT_DECRYPT_RSA_NO_PADDING_CHECK

    case RSA_NO_PADDING:

        flags = CRYPT_DECRYPT_RSA_NO_PADDING_CHECK;

        break;

#endif

    default:

        {

            char errstr[10];

            BIO_snprintf(errstr, 10, "%d", padding);

            CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_UNSUPPORTED_PADDING);

            ERR_add_error_data(2, "padding=", errstr);

            return -1;

        }

    }



    /* Create temp reverse order version of input */

    if (!(tmpbuf = OPENSSL_malloc(flen))) {
@@ -927,14 +940,17 @@ int capi_rsa_priv_dec(int flen, const unsigned char *from, 
        tmpbuf[flen - i - 1] = from[i];



    /* Finally decrypt it */

    if (!CryptDecrypt(capi_key->key, 0, TRUE, 0, tmpbuf, &flen)) {

    if (!CryptDecrypt(capi_key->key, 0, TRUE, flags, tmpbuf, &flen)) {

        CAPIerr(CAPI_F_CAPI_RSA_PRIV_DEC, CAPI_R_DECRYPT_ERROR);

        capi_addlasterror();

        OPENSSL_cleanse(tmpbuf, flen);

        OPENSSL_free(tmpbuf);

        return -1;

    } else

    } else {

        memcpy(to, tmpbuf, flen);

    }



    OPENSSL_cleanse(tmpbuf, flen);

    OPENSSL_free(tmpbuf);



    return flen;