Commit 8e704858 authored by Rich Salz's avatar Rich Salz Committed by Rich Salz
Browse files

RT3955: Reduce some stack usage 



Use malloc/free instead of big onstack buffers.

Reviewed-by: default avatarTim Hudson <tjh@openssl.org>
parent ecdaa1ae

crypto/bn/bn_prime.c

+9 −5
Original line number Diff line number Diff line
@@ -131,7 +131,7 @@ 
static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,

                   const BIGNUM *a1_odd, int k, BN_CTX *ctx,

                   BN_MONT_CTX *mont);

static int probable_prime(BIGNUM *rnd, int bits);

static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods);

static int probable_prime_dh_safe(BIGNUM *rnd, int bits,

                                  const BIGNUM *add, const BIGNUM *rem,

                                  BN_CTX *ctx);
@@ -211,9 +211,13 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, 
    BIGNUM *t;

    int found = 0;

    int i, j, c1 = 0;

    BN_CTX *ctx;

    BN_CTX *ctx = NULL;

    prime_t *mods = NULL;

    int checks = BN_prime_checks_for_size(bits);



    mods = OPENSSL_zalloc(sizeof(*mods) * NUMPRIMES);

    if (mods == NULL)

        goto err;

    if (bits < 2) {

        /* There are no prime numbers this small. */

        BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
@@ -234,7 +238,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, 
 loop:

    /* make a random number and set the top and bottom bits */

    if (add == NULL) {

        if (!probable_prime(ret, bits))

        if (!probable_prime(ret, bits, mods))

            goto err;

    } else {

        if (safe) {
@@ -285,6 +289,7 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, 
    /* we have a prime :-) */

    found = 1;

 err:

    OPENSSL_free(mods);

    if (ctx != NULL)

        BN_CTX_end(ctx);

    BN_CTX_free(ctx);
@@ -497,10 +502,9 @@ static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1, 
    return 1;

}



static int probable_prime(BIGNUM *rnd, int bits)

static int probable_prime(BIGNUM *rnd, int bits, prime_t *mods)

{

    int i;

    prime_t mods[NUMPRIMES];

    BN_ULONG delta;

    BN_ULONG maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];

    char is_single_word = bits <= BN_BITS2;

crypto/pkcs7/pk7_smime.c

+34 −20
Original line number Diff line number Diff line
@@ -64,6 +64,9 @@ 
#include <openssl/x509.h>

#include <openssl/x509v3.h>





#define BUFFERSIZE 4096



static int pkcs7_copy_existing_digest(PKCS7 *p7, PKCS7_SIGNER_INFO *si);



PKCS7 *PKCS7_sign(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,
@@ -113,6 +116,7 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags) 
{

    BIO *p7bio;

    int ret = 0;



    if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) {

        PKCS7err(PKCS7_F_PKCS7_FINAL, ERR_R_MALLOC_FAILURE);

        return 0;
@@ -253,7 +257,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, 
    STACK_OF(PKCS7_SIGNER_INFO) *sinfos;

    PKCS7_SIGNER_INFO *si;

    X509_STORE_CTX cert_ctx;

    char buf[4096];

    char *buf = NULL;

    int i, j = 0, k, ret = 0;

    BIO *p7bio = NULL;

    BIO *tmpin = NULL, *tmpout = NULL;
@@ -355,8 +359,12 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, 
        tmpout = out;



    /* We now have to 'read' from p7bio to calculate digests etc. */

    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {

        PKCS7err(PKCS7_F_PKCS7_VERIFY, ERR_R_MALLOC_FAILURE);

        goto err;

    }

    for (;;) {

        i = BIO_read(p7bio, buf, sizeof(buf));

        i = BIO_read(p7bio, buf, BUFFERSIZE);

        if (i <= 0)

            break;

        if (tmpout)
@@ -387,6 +395,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, 
    ret = 1;



 err:

    OPENSSL_free(buf);

    if (tmpin == indata) {

        if (indata)

            BIO_pop(p7bio);
@@ -505,7 +514,7 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) 
{

    BIO *tmpmem;

    int ret, i;

    char buf[4096];

    char *buf = NULL;



    if (!p7) {

        PKCS7err(PKCS7_F_PKCS7_DECRYPT, PKCS7_R_INVALID_NULL_POINTER);
@@ -549,9 +558,13 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) 
        }

        BIO_free_all(bread);

        return ret;

    } else {

    }

    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {

        PKCS7err(PKCS7_F_PKCS7_DECRYPT, ERR_R_MALLOC_FAILURE);

        goto err;

    }

    for (;;) {

            i = BIO_read(tmpmem, buf, sizeof(buf));

        i = BIO_read(tmpmem, buf, BUFFERSIZE);

        if (i <= 0) {

            ret = 1;

            if (BIO_method_type(tmpmem) == BIO_TYPE_CIPHER) {
@@ -566,7 +579,8 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags) 
            break;

        }

    }

err:

    OPENSSL_free(buf);

    BIO_free_all(tmpmem);

    return ret;

}
 
}