Commit 69795831 authored by Rich Salz's avatar Rich Salz
Browse files

Fix an endless loop in rsa_builtin_keygen. 



And add a test case.

Reviewed-by: default avatarRichard Levitte <levitte@openssl.org>
Reviewed-by: default avatarRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2757)
parent e498d954

crypto/rsa/rsa_gen.c

+11 −12
Original line number Diff line number Diff line
@@ -43,6 +43,16 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, 
    int bitsp, bitsq, ok = -1, n = 0;

    BN_CTX *ctx = NULL;



    /*

     * When generating ridiculously small keys, we can get stuck

     * continually regenerating the same prime values.

     */

    if (bits < 16) {

        ok = 0;             /* we set our own err */

        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);

        goto err;

    }



    ctx = BN_CTX_new();

    if (ctx == NULL)

        goto err;
@@ -94,21 +104,10 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, 
    if (!BN_GENCB_call(cb, 3, 0))

        goto err;

    for (;;) {

        /*

         * When generating ridiculously small keys, we can get stuck

         * continually regenerating the same prime values. Check for this and

         * bail if it happens 3 times.

         */

        unsigned int degenerate = 0;

        do {

            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))

                goto err;

        } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));

        if (degenerate == 3) {

            ok = 0;             /* we set our own err */

            RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);

            goto err;

        }

        } while (BN_cmp(rsa->p, rsa->q) == 0);

        if (!BN_sub(r2, rsa->q, BN_value_one()))

            goto err;

        if (!BN_gcd(r1, r2, rsa->e, ctx))

test/recipes/15-test_genrsa.t

0 → 100644
+26 −0
Original line number Diff line number Diff line 
#! /usr/bin/env perl

# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.

#

# Licensed under the OpenSSL license (the "License").  You may not use

# this file except in compliance with the License.  You can obtain a copy

# in the file LICENSE in the source distribution or at

# https://www.openssl.org/source/license.html





use strict;

use warnings;



use File::Spec;

use OpenSSL::Test qw/:DEFAULT srctop_file/;

use OpenSSL::Test::Utils;



setup("test_genrsa");



plan tests => 5;



is(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '8'])), 0, "genrsa -3 8");

ok(run(app([ 'openssl', 'genrsa', '-3', '-out', 'genrsatest.pem', '16'])), "genrsa -3 16");

ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");

ok(run(app([ 'openssl', 'genrsa', '-f4', '-out', 'genrsatest.pem', '16'])), "genrsa -f4 16");

ok(run(app([ 'openssl', 'rsa', '-check', '-in', 'genrsatest.pem', '-noout'])), "rsa -check");

unlink 'genrsatest.pem';