Commit 9716a8f9 authored by Dr. Stephen Henson's avatar Dr. Stephen Henson
Browse files

Fix to PKCS#7 routines so it can decrypt some oddball RC2 handling.

parent 74400f73

CHANGES

+12 −0
Original line number Diff line number Diff line
@@ -4,6 +4,18 @@ 


 Changes between 0.9.4 and 0.9.5  [xx XXX 1999]



  *) Hack to fix PKCS#7 decryption when used with some unorthodox RC2 

     handling. Most clients have the effective key size in bits equal to

     the key length in bits: so a 40 bit RC2 key uses a 40 bit (5 byte) key.

     A few however don't do this and instead use the size of the decrypted key

     to determine the RC2 key length and the AlgorithmIdentifier to determine

     the effective key length. In this case the effective key lenth can still

     be 40 bits but the key length can be 168 bits for example. This is fixed

     by manually forcing an RC2 key into the EVP_PKEY structure because the

     EVP code can't currently handle unusual RC2 key sizes: it always assumes

     the key length and effective key length are equal.

     [Steve Henson]



  *) Add a bunch of functions that should simplify the creation of 

     X509_NAME structures. Now you should be able to do:

     X509_NAME_add_entry_by_txt(nm, "CN", MBSTRING_ASC, "Steve", -1, -1, 0);

crypto/pkcs7/pk7_doit.c

+22 −11
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) 
	{

	int i,j;

	BIO *out=NULL,*btmp=NULL,*etmp=NULL,*bio=NULL;

	char *tmp=NULL;

	unsigned char *tmp=NULL;

	X509_ALGOR *xa;

	ASN1_OCTET_STRING *data_body=NULL;

	const EVP_MD *evp_md;
@@ -262,6 +262,7 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) 
	STACK_OF(PKCS7_RECIP_INFO) *rsk=NULL;

	X509_ALGOR *xalg=NULL;

	PKCS7_RECIP_INFO *ri=NULL;

	char is_rc2 = 0;

/*	EVP_PKEY *pkey; */

#if 0

	X509_STORE_CTX s_ctx;
@@ -306,6 +307,8 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) 
	        goto err;

		}



	if(EVP_CIPHER_nid(evp_cipher) == NID_rc2_cbc) is_rc2 = 1; 



	/* We will be checking the signature */

	if (md_sk != NULL)

		{
@@ -375,17 +378,15 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) 
		}



		jj=EVP_PKEY_size(pkey);

		tmp=Malloc(jj+10);

		tmp=(unsigned char *)Malloc(jj+10);

		if (tmp == NULL)

			{

			PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_MALLOC_FAILURE);

			goto err;

			}



		jj=EVP_PKEY_decrypt((unsigned char *)tmp,

			M_ASN1_STRING_data(ri->enc_key),

			M_ASN1_STRING_length(ri->enc_key),

			pkey);

		jj=EVP_PKEY_decrypt(tmp, M_ASN1_STRING_data(ri->enc_key),

			M_ASN1_STRING_length(ri->enc_key), pkey);

		if (jj <= 0)

			{

			PKCS7err(PKCS7_F_PKCS7_DATADECODE,ERR_R_EVP_LIB);
@@ -398,13 +399,23 @@ BIO *PKCS7_dataDecode(PKCS7 *p7, EVP_PKEY *pkey, BIO *in_bio, X509 *pcert) 
		if (EVP_CIPHER_asn1_to_param(evp_ctx,enc_alg->parameter) < 0)

			return(NULL);



		if (jj != EVP_CIPHER_CTX_key_length(evp_ctx))

			{

		if (jj != EVP_CIPHER_CTX_key_length(evp_ctx)) {

			/* HACK: some S/MIME clients don't use the same key

			 * and effective key length. The key length is

			 * determined by the size of the decrypted RSA key.

			 * So we hack things to manually set the RC2 key

			 * because we currently can't do this with the EVP

			 * interface.

			 */

			if(is_rc2) RC2_set_key(&(evp_ctx->c.rc2_ks),jj, tmp,

					EVP_CIPHER_CTX_key_length(evp_ctx)*8);

			else {



				PKCS7err(PKCS7_F_PKCS7_DATADECODE,

					PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH);

				goto err;

			}

		EVP_CipherInit(evp_ctx,NULL,(unsigned char *)tmp,NULL,0);

		} else EVP_CipherInit(evp_ctx,NULL,tmp,NULL,0);



		memset(tmp,0,jj);