Commit 04cf3920 authored by Bernd Edlinger's avatar Bernd Edlinger Committed by Andy Polyakov
Browse files

Fix a crash or unbounded allocation in RSA_padding_add_PKCS1_PSS_mgf1 


and RSA_verify_PKCS1_PSS_mgf1 with 512-bit RSA vs. sha-512.

Reviewed-by: default avatarAndy Polyakov <appro@openssl.org>
Reviewed-by: default avatarRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2881)

(cherry picked from commit e653b6cd)
parent d843779d

crypto/rsa/rsa_pss.c

+11 −2
Original line number Diff line number Diff line
@@ -122,7 +122,11 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash, 
        EM++;

        emLen--;

    }

    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */

    if (emLen < hLen + 2) {

        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);

        goto err;

    }

    if (sLen > emLen - hLen - 2) { /* sLen can be small negative */

        RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);

        goto err;

    }
@@ -222,9 +226,14 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM, 
        *EM++ = 0;

        emLen--;

    }

    if (emLen < hLen + 2) {

        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,

               RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);

        goto err;

    }

    if (sLen == -2) {

        sLen = emLen - hLen - 2;

    } else if (emLen < (hLen + sLen + 2)) {

    } else if (sLen > emLen - hLen - 2) {

        RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,

               RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);

        goto err;