Commit 9fd44200 authored by Bernd Edlinger's avatar Bernd Edlinger
Browse files

Fix an endless loop in BN_generate_prime_ex 



Happens when trying to generate 4 or 5 bit safe primes.

[extended tests]

Reviewed-by: default avatarPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9311)

(cherry picked from commit 291f616ced45c924d639d97fc9ca2cbeaad096cf)
parent 78af3f6f

crypto/bn/bn_prime.c

+6 −2
Original line number Diff line number Diff line
@@ -63,8 +63,12 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, 
        /* There are no prime numbers this small. */

        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);

        return 0;

    } else if (bits == 2 && safe) {

        /* The smallest safe prime (7) is three bits. */

    } else if (add == NULL && safe && bits < 6 && bits != 3) {

        /*

         * The smallest safe prime (7) is three bits.

         * But the following two safe primes with less than 6 bits (11, 23)

         * are unreachable for BN_rand with BN_RAND_TOP_TWO.

         */

        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);

        return 0;

    }

test/bntest.c

+40 −7
Original line number Diff line number Diff line
@@ -2169,17 +2169,49 @@ static int test_expmodone(void) 
    return ret;

}



static int test_smallprime(void)

static int test_smallprime(int kBits)

{

    static const int kBits = 10;

    BIGNUM *r;

    int st = 0;



    if (!TEST_ptr(r = BN_new())

            || !TEST_true(BN_generate_prime_ex(r, (int)kBits, 0,

    if (!TEST_ptr(r = BN_new()))

        goto err;



    if (kBits <= 1) {

        if (!TEST_false(BN_generate_prime_ex(r, kBits, 0,

                                             NULL, NULL, NULL)))

            goto err;

    } else {

        if (!TEST_true(BN_generate_prime_ex(r, kBits, 0,

                                            NULL, NULL, NULL))

                || !TEST_int_eq(BN_num_bits(r), kBits))

            goto err;

    }



    st = 1;

 err:

    BN_free(r);

    return st;

}



static int test_smallsafeprime(int kBits)

{

    BIGNUM *r;

    int st = 0;



    if (!TEST_ptr(r = BN_new()))

        goto err;



    if (kBits <= 5 && kBits != 3) {

        if (!TEST_false(BN_generate_prime_ex(r, kBits, 1,

                                             NULL, NULL, NULL)))

            goto err;

    } else {

        if (!TEST_true(BN_generate_prime_ex(r, kBits, 1,

                                            NULL, NULL, NULL))

                || !TEST_int_eq(BN_num_bits(r), kBits))

            goto err;

    }



    st = 1;

 err:
@@ -2405,7 +2437,8 @@ int setup_tests(void) 
        ADD_TEST(test_badmod);

        ADD_TEST(test_expmodzero);

        ADD_TEST(test_expmodone);

        ADD_TEST(test_smallprime);

        ADD_ALL_TESTS(test_smallprime, 16);

        ADD_ALL_TESTS(test_smallsafeprime, 16);

        ADD_TEST(test_swap);

        ADD_TEST(test_ctx_consttime_flag);

#ifndef OPENSSL_NO_EC2M