Improve error detection when streaming S/MIME.

Only use streaming when appropriate for detached data in smime utility.

apps/smime.c

@@ -688,15 +688,22 @@ int MAIN(int argc, char **argv)
 	else if (operation & SMIME_SIGNERS)
 		{
 		int i;
-		/* If detached data and SMIME output enable partial
-		 * signing.
+		/* If detached data content we only enable streaming if
+		 * S/MIME output format.
 		 */
 		if (operation == SMIME_SIGN)
 			{
-			if (indef || (flags & PKCS7_DETACHED))
+			if (flags & PKCS7_DETACHED)
+				{
+				if (outformat == FORMAT_SMIME)
+					flags |= PKCS7_STREAM;
+				}
+			else if (indef)
 				flags |= PKCS7_STREAM;
 			flags |= PKCS7_PARTIAL;
 			p7 = PKCS7_sign(NULL, NULL, other, in, flags);
+			if (!p7)
+				goto end;
 			}
 		else
 			flags |= PKCS7_REUSE_DIGEST;

crypto/pkcs7/bio_pk7.c

@@ -114,13 +114,17 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
 	out = BIO_push(asn_bio, out);
 
+	if (!p7aux || !asn_bio || !out)
+		goto err;
+
 	BIO_asn1_set_prefix(asn_bio, pkcs7_prefix, pkcs7_prefix_free);
 	BIO_asn1_set_suffix(asn_bio, pkcs7_suffix, pkcs7_suffix_free);
 
 	/* Now initialize BIO for PKCS#7 output */
 
 	p7bio = PKCS7_dataInit(p7, out);
-	PKCS7_stream(&boundary, p7);
+	if (!p7bio || !PKCS7_stream(&boundary, p7))
+		goto err;
 
 	p7aux->p7 = p7;
 	p7aux->p7bio = p7bio;
@@ -131,6 +135,14 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
 	return p7bio;
 
+	err:
+	if (p7bio)
+		BIO_free(p7bio);
+	if (asn_bio)
+		BIO_free(asn_bio);
+	if (p7aux)
+		OPENSSL_free(p7aux);
+	return NULL;
 	}
 
 static int pkcs7_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg)

crypto/pkcs7/pk7_lib.c

@@ -665,8 +665,3 @@ int PKCS7_stream(unsigned char ***boundary, PKCS7 *p7)
 
 	return 1;
 	}
-
-
-
-
-