Commit 9833757b authored by Andy Polyakov's avatar Andy Polyakov
Browse files

s390x assembler pack update from HEAD.

parent 4195343c

crypto/aes/asm/aes-s390x.pl

+991 −73

File changed.

Preview size limit exceeded, changes collapsed.

crypto/bn/asm/s390x-gf2m.pl

0 → 100644
+221 −0
Original line number Diff line number Diff line 
#!/usr/bin/env perl

#

# ====================================================================

# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

# project. The module is, however, dual licensed under OpenSSL and

# CRYPTOGAMS licenses depending on where you obtain it. For further

# details see http://www.openssl.org/~appro/cryptogams/.

# ====================================================================

#

# May 2011

#

# The module implements bn_GF2m_mul_2x2 polynomial multiplication used

# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for

# the time being... gcc 4.3 appeared to generate poor code, therefore

# the effort. And indeed, the module delivers 55%-90%(*) improvement

# on haviest ECDSA verify and ECDH benchmarks for 163- and 571-bit

# key lengths on z990, 30%-55%(*) - on z10, and 70%-110%(*) - on z196.

# This is for 64-bit build. In 32-bit "highgprs" case improvement is

# even higher, for example on z990 it was measured 80%-150%. ECDSA

# sign is modest 9%-12% faster. Keep in mind that these coefficients

# are not ones for bn_GF2m_mul_2x2 itself, as not all CPU time is

# burnt in it...

#

# (*)	gcc 4.1 was observed to deliver better results than gcc 4.3,

#	so that improvement coefficients can vary from one specific

#	setup to another.



$flavour = shift;



if ($flavour =~ /3[12]/) {

        $SIZE_T=4;

        $g="";

} else {

        $SIZE_T=8;

        $g="g";

}



while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}

open STDOUT,">$output";



$stdframe=16*$SIZE_T+4*8;



$rp="%r2";

$a1="%r3";

$a0="%r4";

$b1="%r5";

$b0="%r6";



$ra="%r14";

$sp="%r15";



@T=("%r0","%r1");

@i=("%r12","%r13");



($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(6..11));

($lo,$hi,$b)=map("%r$_",(3..5)); $a=$lo; $mask=$a8;



$code.=<<___;

.text



.type	_mul_1x1,\@function

.align	16

_mul_1x1:

	lgr	$a1,$a

	sllg	$a2,$a,1

	sllg	$a4,$a,2

	sllg	$a8,$a,3



	srag	$lo,$a1,63			# broadcast 63rd bit

	nihh	$a1,0x1fff

	srag	@i[0],$a2,63			# broadcast 62nd bit

	nihh	$a2,0x3fff

	srag	@i[1],$a4,63			# broadcast 61st bit

	nihh	$a4,0x7fff

	ngr	$lo,$b

	ngr	@i[0],$b

	ngr	@i[1],$b



	lghi	@T[0],0

	lgr	$a12,$a1

	stg	@T[0],`$stdframe+0*8`($sp)	# tab[0]=0

	xgr	$a12,$a2

	stg	$a1,`$stdframe+1*8`($sp)	# tab[1]=a1

	 lgr	$a48,$a4

	stg	$a2,`$stdframe+2*8`($sp)	# tab[2]=a2

	 xgr	$a48,$a8

	stg	$a12,`$stdframe+3*8`($sp)	# tab[3]=a1^a2

	 xgr	$a1,$a4



	stg	$a4,`$stdframe+4*8`($sp)	# tab[4]=a4

	xgr	$a2,$a4

	stg	$a1,`$stdframe+5*8`($sp)	# tab[5]=a1^a4

	xgr	$a12,$a4

	stg	$a2,`$stdframe+6*8`($sp)	# tab[6]=a2^a4

	 xgr	$a1,$a48

	stg	$a12,`$stdframe+7*8`($sp)	# tab[7]=a1^a2^a4

	 xgr	$a2,$a48



	stg	$a8,`$stdframe+8*8`($sp)	# tab[8]=a8

	xgr	$a12,$a48

	stg	$a1,`$stdframe+9*8`($sp)	# tab[9]=a1^a8

	 xgr	$a1,$a4

	stg	$a2,`$stdframe+10*8`($sp)	# tab[10]=a2^a8

	 xgr	$a2,$a4

	stg	$a12,`$stdframe+11*8`($sp)	# tab[11]=a1^a2^a8



	xgr	$a12,$a4

	stg	$a48,`$stdframe+12*8`($sp)	# tab[12]=a4^a8

	 srlg	$hi,$lo,1

	stg	$a1,`$stdframe+13*8`($sp)	# tab[13]=a1^a4^a8

	 sllg	$lo,$lo,63

	stg	$a2,`$stdframe+14*8`($sp)	# tab[14]=a2^a4^a8

	 srlg	@T[0],@i[0],2

	stg	$a12,`$stdframe+15*8`($sp)	# tab[15]=a1^a2^a4^a8



	lghi	$mask,`0xf<<3`

	sllg	$a1,@i[0],62

	 sllg	@i[0],$b,3

	srlg	@T[1],@i[1],3

	 ngr	@i[0],$mask

	sllg	$a2,@i[1],61

	 srlg	@i[1],$b,4-3

	xgr	$hi,@T[0]

	 ngr	@i[1],$mask

	xgr	$lo,$a1

	xgr	$hi,@T[1]

	xgr	$lo,$a2



	xg	$lo,$stdframe(@i[0],$sp)

	srlg	@i[0],$b,8-3

	ngr	@i[0],$mask

___

for($n=1;$n<14;$n++) {

$code.=<<___;

	lg	@T[1],$stdframe(@i[1],$sp)

	srlg	@i[1],$b,`($n+2)*4`-3

	sllg	@T[0],@T[1],`$n*4`

	ngr	@i[1],$mask

	srlg	@T[1],@T[1],`64-$n*4`

	xgr	$lo,@T[0]

	xgr	$hi,@T[1]

___

	push(@i,shift(@i)); push(@T,shift(@T));

}

$code.=<<___;

	lg	@T[1],$stdframe(@i[1],$sp)

	sllg	@T[0],@T[1],`$n*4`

	srlg	@T[1],@T[1],`64-$n*4`

	xgr	$lo,@T[0]

	xgr	$hi,@T[1]



	lg	@T[0],$stdframe(@i[0],$sp)

	sllg	@T[1],@T[0],`($n+1)*4`

	srlg	@T[0],@T[0],`64-($n+1)*4`

	xgr	$lo,@T[1]

	xgr	$hi,@T[0]



	br	$ra

.size	_mul_1x1,.-_mul_1x1



.globl	bn_GF2m_mul_2x2

.type	bn_GF2m_mul_2x2,\@function

.align	16

bn_GF2m_mul_2x2:

	stm${g}	%r3,%r15,3*$SIZE_T($sp)



	lghi	%r1,-$stdframe-128

	la	%r0,0($sp)

	la	$sp,0(%r1,$sp)			# alloca

	st${g}	%r0,0($sp)			# back chain

___

if ($SIZE_T==8) {

my @r=map("%r$_",(6..9));

$code.=<<___;

	bras	$ra,_mul_1x1			# a1b1

	stmg	$lo,$hi,16($rp)



	lg	$a,`$stdframe+128+4*$SIZE_T`($sp)

	lg	$b,`$stdframe+128+6*$SIZE_T`($sp)

	bras	$ra,_mul_1x1			# a0b0

	stmg	$lo,$hi,0($rp)



	lg	$a,`$stdframe+128+3*$SIZE_T`($sp)

	lg	$b,`$stdframe+128+5*$SIZE_T`($sp)

	xg	$a,`$stdframe+128+4*$SIZE_T`($sp)

	xg	$b,`$stdframe+128+6*$SIZE_T`($sp)

	bras	$ra,_mul_1x1			# (a0+a1)(b0+b1)

	lmg	@r[0],@r[3],0($rp)



	xgr	$lo,$hi

	xgr	$hi,@r[1]

	xgr	$lo,@r[0]

	xgr	$hi,@r[2]

	xgr	$lo,@r[3]	

	xgr	$hi,@r[3]

	xgr	$lo,$hi

	stg	$hi,16($rp)

	stg	$lo,8($rp)

___

} else {

$code.=<<___;

	sllg	%r3,%r3,32

	sllg	%r5,%r5,32

	or	%r3,%r4

	or	%r5,%r6

	bras	$ra,_mul_1x1

	rllg	$lo,$lo,32

	rllg	$hi,$hi,32

	stmg	$lo,$hi,0($rp)

___

}

$code.=<<___;

	lm${g}	%r6,%r15,`$stdframe+128+6*$SIZE_T`($sp)

	br	$ra

.size	bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2

.string	"GF(2^m) Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>"

___



$code =~ s/\`([^\`]*)\`/eval($1)/gem;

print $code;

close STDOUT;

crypto/bn/asm/s390x-mont.pl

+77 −25
Original line number Diff line number Diff line
@@ -32,6 +32,33 @@ 
# Reschedule to minimize/avoid Address Generation Interlock hazard,

# make inner loops counter-based.



# November 2010.

#

# Adapt for -m31 build. If kernel supports what's called "highgprs"

# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit

# instructions and achieve "64-bit" performance even in 31-bit legacy

# application context. The feature is not specific to any particular

# processor, as long as it's "z-CPU". Latter implies that the code

# remains z/Architecture specific. Compatibility with 32-bit BN_ULONG

# is achieved by swapping words after 64-bit loads, follow _dswap-s.

# On z990 it was measured to perform 2.6-2.2 times better than

# compiler-generated code, less for longer keys...



$flavour = shift;



if ($flavour =~ /3[12]/) {

	$SIZE_T=4;

	$g="";

} else {

	$SIZE_T=8;

	$g="g";

}



while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}

open STDOUT,">$output";



$stdframe=16*$SIZE_T+4*8;



$mn0="%r0";

$num="%r1";
@@ -60,34 +87,44 @@ $code.=<<___; 
.globl	bn_mul_mont

.type	bn_mul_mont,\@function

bn_mul_mont:

	lgf	$num,164($sp)	# pull $num

	sla	$num,3		# $num to enumerate bytes

	lgf	$num,`$stdframe+$SIZE_T-4`($sp)	# pull $num

	sla	$num,`log($SIZE_T)/log(2)`	# $num to enumerate bytes

	la	$bp,0($num,$bp)



	stg	%r2,16($sp)

	st${g}	%r2,2*$SIZE_T($sp)



	cghi	$num,16		#

	lghi	%r2,0		#

	blr	%r14		# if($num<16) return 0;

___

$code.=<<___ if ($flavour =~ /3[12]/);

	tmll	$num,4

	bnzr	%r14		# if ($num&1) return 0;

___

$code.=<<___ if ($flavour !~ /3[12]/);

	cghi	$num,96		#

	bhr	%r14		# if($num>96) return 0;

___

$code.=<<___;

	stm${g}	%r3,%r15,3*$SIZE_T($sp)



	stmg	%r3,%r15,24($sp)



	lghi	$rp,-160-8	# leave room for carry bit

	lghi	$rp,-$stdframe-8	# leave room for carry bit

	lcgr	$j,$num		# -$num

	lgr	%r0,$sp

	la	$rp,0($rp,$sp)

	la	$sp,0($j,$rp)	# alloca

	stg	%r0,0($sp)	# back chain

	st${g}	%r0,0($sp)	# back chain



	sra	$num,3		# restore $num

	la	$bp,0($j,$bp)	# restore $bp

	ahi	$num,-1		# adjust $num for inner loop

	lg	$n0,0($n0)	# pull n0

	_dswap	$n0



	lg	$bi,0($bp)

	_dswap	$bi

	lg	$alo,0($ap)

	_dswap	$alo

	mlgr	$ahi,$bi	# ap[0]*bp[0]

	lgr	$AHI,$ahi
@@ -95,6 +132,7 @@ bn_mul_mont: 
	msgr	$mn0,$n0



	lg	$nlo,0($np)	#

	_dswap	$nlo

	mlgr	$nhi,$mn0	# np[0]*m1

	algr	$nlo,$alo	# +="tp[0]"

	lghi	$NHI,0
@@ -106,12 +144,14 @@ bn_mul_mont: 
.align	16

.L1st:

	lg	$alo,0($j,$ap)

	_dswap	$alo

	mlgr	$ahi,$bi	# ap[j]*bp[0]

	algr	$alo,$AHI

	lghi	$AHI,0

	alcgr	$AHI,$ahi



	lg	$nlo,0($j,$np)

	_dswap	$nlo

	mlgr	$nhi,$mn0	# np[j]*m1

	algr	$nlo,$NHI

	lghi	$NHI,0
@@ -119,22 +159,24 @@ bn_mul_mont: 
	algr	$nlo,$alo

	alcgr	$NHI,$nhi



	stg	$nlo,160-8($j,$sp)	# tp[j-1]=

	stg	$nlo,$stdframe-8($j,$sp)	# tp[j-1]=

	la	$j,8($j)	# j++

	brct	$count,.L1st



	algr	$NHI,$AHI

	lghi	$AHI,0

	alcgr	$AHI,$AHI	# upmost overflow bit

	stg	$NHI,160-8($j,$sp)

	stg	$AHI,160($j,$sp)

	stg	$NHI,$stdframe-8($j,$sp)

	stg	$AHI,$stdframe($j,$sp)

	la	$bp,8($bp)	# bp++



.Louter:

	lg	$bi,0($bp)	# bp[i]

	_dswap	$bi

	lg	$alo,0($ap)

	_dswap	$alo

	mlgr	$ahi,$bi	# ap[0]*bp[i]

	alg	$alo,160($sp)	# +=tp[0]

	alg	$alo,$stdframe($sp)	# +=tp[0]

	lghi	$AHI,0

	alcgr	$AHI,$ahi
@@ -142,6 +184,7 @@ bn_mul_mont: 
	msgr	$mn0,$n0	# tp[0]*n0



	lg	$nlo,0($np)	# np[0]

	_dswap	$nlo

	mlgr	$nhi,$mn0	# np[0]*m1

	algr	$nlo,$alo	# +="tp[0]"

	lghi	$NHI,0
@@ -153,14 +196,16 @@ bn_mul_mont: 
.align	16

.Linner:

	lg	$alo,0($j,$ap)

	_dswap	$alo

	mlgr	$ahi,$bi	# ap[j]*bp[i]

	algr	$alo,$AHI

	lghi	$AHI,0

	alcgr	$ahi,$AHI

	alg	$alo,160($j,$sp)# +=tp[j]

	alg	$alo,$stdframe($j,$sp)# +=tp[j]

	alcgr	$AHI,$ahi



	lg	$nlo,0($j,$np)

	_dswap	$nlo

	mlgr	$nhi,$mn0	# np[j]*m1

	algr	$nlo,$NHI

	lghi	$NHI,0
@@ -168,31 +213,33 @@ bn_mul_mont: 
	algr	$nlo,$alo	# +="tp[j]"

	alcgr	$NHI,$nhi



	stg	$nlo,160-8($j,$sp)	# tp[j-1]=

	stg	$nlo,$stdframe-8($j,$sp)	# tp[j-1]=

	la	$j,8($j)	# j++

	brct	$count,.Linner



	algr	$NHI,$AHI

	lghi	$AHI,0

	alcgr	$AHI,$AHI

	alg	$NHI,160($j,$sp)# accumulate previous upmost overflow bit

	alg	$NHI,$stdframe($j,$sp)# accumulate previous upmost overflow bit

	lghi	$ahi,0

	alcgr	$AHI,$ahi	# new upmost overflow bit

	stg	$NHI,160-8($j,$sp)

	stg	$AHI,160($j,$sp)

	stg	$NHI,$stdframe-8($j,$sp)

	stg	$AHI,$stdframe($j,$sp)



	la	$bp,8($bp)	# bp++

	clg	$bp,160+8+32($j,$sp)	# compare to &bp[num]

	cl${g}	$bp,`$stdframe+8+4*$SIZE_T`($j,$sp)	# compare to &bp[num]

	jne	.Louter



	lg	$rp,160+8+16($j,$sp)	# reincarnate rp

	la	$ap,160($sp)

	l${g}	$rp,`$stdframe+8+2*$SIZE_T`($j,$sp)	# reincarnate rp

	la	$ap,$stdframe($sp)

	ahi	$num,1		# restore $num, incidentally clears "borrow"



	la	$j,0(%r0)

	lr	$count,$num

.Lsub:	lg	$alo,0($j,$ap)

	slbg	$alo,0($j,$np)

	lg	$nlo,0($j,$np)

	_dswap	$nlo

	slbgr	$alo,$nlo

	stg	$alo,0($j,$rp)

	la	$j,8($j)

	brct	$count,.Lsub
@@ -208,18 +255,23 @@ bn_mul_mont: 
	la	$j,0(%r0)

	lgr	$count,$num

.Lcopy:	lg	$alo,0($j,$ap)		# copy or in-place refresh

	stg	$j,160($j,$sp)	# zap tp

	_dswap	$alo

	stg	$j,$stdframe($j,$sp)	# zap tp

	stg	$alo,0($j,$rp)

	la	$j,8($j)

	brct	$count,.Lcopy



	la	%r1,160+8+48($j,$sp)

	lmg	%r6,%r15,0(%r1)

	la	%r1,`$stdframe+8+6*$SIZE_T`($j,$sp)

	lm${g}	%r6,%r15,0(%r1)

	lghi	%r2,1		# signal "processed"

	br	%r14

.size	bn_mul_mont,.-bn_mul_mont

.string	"Montgomery Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>"

___



print $code;

foreach (split("\n",$code)) {

	s/\`([^\`]*)\`/eval $1/ge;

	s/_dswap\s+(%r[0-9]+)/sprintf("rllg\t%s,%s,32",$1,$1) if($SIZE_T==4)/e;

	print $_,"\n";

}

close STDOUT;

crypto/rc4/asm/rc4-s390x.pl

+39 −10
Original line number Diff line number Diff line
@@ -13,6 +13,29 @@ 
# "cluster" Address Generation Interlocks, so that one pipeline stall

# resolves several dependencies.



# November 2010.

#

# Adapt for -m31 build. If kernel supports what's called "highgprs"

# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit

# instructions and achieve "64-bit" performance even in 31-bit legacy

# application context. The feature is not specific to any particular

# processor, as long as it's "z-CPU". Latter implies that the code

# remains z/Architecture specific. On z990 it was measured to perform

# 50% better than code generated by gcc 4.3.



$flavour = shift;



if ($flavour =~ /3[12]/) {

	$SIZE_T=4;

	$g="";

} else {

	$SIZE_T=8;

	$g="g";

}



while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}

open STDOUT,">$output";



$rp="%r14";

$sp="%r15";

$code=<<___;
@@ -39,7 +62,12 @@ $code.=<<___; 
.type	RC4,\@function

.align	64

RC4:

	stmg	%r6,%r11,48($sp)

	stm${g}	%r6,%r11,6*$SIZE_T($sp)

___

$code.=<<___ if ($flavour =~ /3[12]/);

	llgfr	$len,$len

___

$code.=<<___;

	llgc	$XX[0],0($key)

	llgc	$YY,1($key)

	la	$XX[0],1($XX[0])
@@ -90,7 +118,7 @@ $code.=<<___; 
	xgr	$acc,$TX[1]

	stg	$acc,0($out)

	la	$out,8($out)

	brct	$cnt,.Loop8

	brctg	$cnt,.Loop8



.Lshort:

	lghi	$acc,7
@@ -122,7 +150,7 @@ $code.=<<___; 
	ahi	$XX[0],-1

	stc	$XX[0],0($key)

	stc	$YY,1($key)

	lmg	%r6,%r11,48($sp)

	lm${g}	%r6,%r11,6*$SIZE_T($sp)

	br	$rp

.size	RC4,.-RC4

.string	"RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
@@ -130,7 +158,7 @@ $code.=<<___; 
___

}



# void private_RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)

# void RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)

{

$cnt="%r0";

$idx="%r1";
@@ -143,11 +171,11 @@ $ikey="%r7"; 
$iinp="%r8";



$code.=<<___;

.globl	private_RC4_set_key

.type	private_RC4_set_key,\@function

.globl	RC4_set_key

.type	RC4_set_key,\@function

.align	64

private_RC4_set_key:

	stmg	%r6,%r8,48($sp)

RC4_set_key:

	stm${g}	%r6,%r8,6*$SIZE_T($sp)

	lhi	$cnt,256

	la	$idx,0(%r0)

	sth	$idx,0($key)
@@ -180,9 +208,9 @@ private_RC4_set_key: 
	la	$iinp,0(%r0)

	j	.L2ndloop

.Ldone:

	lmg	%r6,%r8,48($sp)

	lm${g}	%r6,%r8,6*$SIZE_T($sp)

	br	$rp

.size	private_RC4_set_key,.-private_RC4_set_key

.size	RC4_set_key,.-RC4_set_key



___

}
@@ -203,3 +231,4 @@ RC4_options: 
___



print $code;

close STDOUT;	# force flush

crypto/s390xcap.c

+6 −6
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ 
#include <setjmp.h>

#include <signal.h>



extern unsigned long OPENSSL_s390xcap_P;

extern unsigned long OPENSSL_s390xcap_P[];



static sigjmp_buf ill_jmp;

static void ill_handler (int sig) { siglongjmp(ill_jmp,sig); }
@@ -16,7 +16,9 @@ void OPENSSL_cpuid_setup(void) 
	sigset_t oset;

	struct sigaction ill_act,oact;



	if (OPENSSL_s390xcap_P) return;

	if (OPENSSL_s390xcap_P[0]) return;



	OPENSSL_s390xcap_P[0] = 1UL<<(8*sizeof(unsigned long)-1);



	memset(&ill_act,0,sizeof(ill_act));

	ill_act.sa_handler = ill_handler;
@@ -27,10 +29,8 @@ void OPENSSL_cpuid_setup(void) 
	sigaction (SIGILL,&ill_act,&oact);



	/* protection against missing store-facility-list-extended */

	if (sigsetjmp(ill_jmp,0) == 0)

		OPENSSL_s390xcap_P = OPENSSL_s390x_facilities();

	else

		OPENSSL_s390xcap_P = 1UL<<63;

	if (sigsetjmp(ill_jmp,1) == 0)

		OPENSSL_s390x_facilities();



	sigaction (SIGILL,&oact,NULL);

	sigprocmask(SIG_SETMASK,&oset,NULL);