Commit 32492093 authored by Rich Salz's avatar Rich Salz Committed by Matt Caswell
Browse files

Fix an endless loop in rsa_builtin_keygen. 



Cherry-picked by Matt Caswell from 69795831.

Reviewed-by: default avatarRichard Levitte <levitte@openssl.org>
Reviewed-by: default avatarMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4670)
parent 950d49d4

crypto/rsa/rsa_gen.c

+11 −12
Original line number Diff line number Diff line
@@ -110,6 +110,16 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, 
    int bitsp, bitsq, ok = -1, n = 0;

    BN_CTX *ctx = NULL;



    /*

     * When generating ridiculously small keys, we can get stuck

     * continually regenerating the same prime values.

     */

    if (bits < 16) {

        ok = 0;             /* we set our own err */

        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);

        goto err;

    }



    ctx = BN_CTX_new();

    if (ctx == NULL)

        goto err;
@@ -161,21 +171,10 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, 
    if (!BN_GENCB_call(cb, 3, 0))

        goto err;

    for (;;) {

        /*

         * When generating ridiculously small keys, we can get stuck

         * continually regenerating the same prime values. Check for this and

         * bail if it happens 3 times.

         */

        unsigned int degenerate = 0;

        do {

            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))

                goto err;

        } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));

        if (degenerate == 3) {

            ok = 0;             /* we set our own err */

            RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);

            goto err;

        }

        } while (BN_cmp(rsa->p, rsa->q) == 0);

        if (!BN_sub(r2, rsa->q, BN_value_one()))

            goto err;

        if (!BN_gcd(r1, r2, rsa->e, ctx))