Commit 15972296 authored by David Benjamin's avatar David Benjamin Committed by Bernd Edlinger
Browse files

Fix calling convention bug in ecp_nistz256_ord_sqr_mont 



The rep parameter takes an int in C, but the assembly implementation
looks at the upper bits. While it's unlikely to happen here, where all
calls pass a constant, in other scenarios x86_64 compilers will leave
arbitrary values in the upper half.

Fix this by making the C prototype match the assembly. (This aspect of
the calling convention implies smaller-than-word arguments in assembly
functions should be avoided. There are far fewer things to test if
everything consistently takes word-sized arguments.)

This was found as part of ABI testing work in BoringSSL.

Reviewed-by: default avatarPaul Dale <paul.dale@oracle.com>
Reviewed-by: default avatarBernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/8108)
parent 3051bf2a

crypto/ec/asm/ecp_nistz256-armv8.pl

+1 −1
Original line number Diff line number Diff line
@@ -1488,7 +1488,7 @@ $code.=<<___; 


////////////////////////////////////////////////////////////////////////

// void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],

//                                int rep);

//                                uint64_t rep);

.globl	ecp_nistz256_ord_sqr_mont

.type	ecp_nistz256_ord_sqr_mont,%function

.align	4

crypto/ec/asm/ecp_nistz256-ppc64.pl

+1 −1
Original line number Diff line number Diff line
@@ -1919,7 +1919,7 @@ $code.=<<___; 


################################################################################

# void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],

#                                int rep);

#                                uint64_t rep);

.globl	ecp_nistz256_ord_sqr_mont

.align	5

ecp_nistz256_ord_sqr_mont:

crypto/ec/asm/ecp_nistz256-x86_64.pl

+1 −1
Original line number Diff line number Diff line
@@ -826,7 +826,7 @@ $code.=<<___; 
# void ecp_nistz256_ord_sqr_mont(

#   uint64_t res[4],

#   uint64_t a[4],

#   int rep);

#   uint64_t rep);



.globl	ecp_nistz256_ord_sqr_mont

.type	ecp_nistz256_ord_sqr_mont,\@function,3

crypto/ec/ecp_nistz256.c

+1 −1
Original line number Diff line number Diff line
@@ -1467,7 +1467,7 @@ void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], 
                               const BN_ULONG b[P256_LIMBS]);

void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS],

                               const BN_ULONG a[P256_LIMBS],

                               int rep);

                               BN_ULONG rep);



static int ecp_nistz256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r,

                                    const BIGNUM *x, BN_CTX *ctx)