Loading crypto/modes/asm/ghash-sparcv9.pl +217 −3 Original line number Diff line number Diff line Loading @@ -36,6 +36,13 @@ # references to input data and Z.hi updates to achieve 12 cycles # timing. To anchor to something else, sha1-sparcv9.pl spends 11.6 # cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1. # # October 2012 # # Add VIS3 lookup-table-free implementation using polynomial # multiplication xmulx[hi] and extended addition addxc[cc] # instructions. 3.96/6.26x improvement on T3/T4 or in absolute # terms 9.02/2.61 cycles per byte. $bits=32; for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } Loading Loading @@ -66,6 +73,10 @@ $Htbl="%i1"; $inp="%i2"; $len="%i3"; $code.=<<___ if ($bits==64); .register %g2,#scratch .register %g3,#scratch ___ $code.=<<___; .section ".text",#alloc,#execinstr Loading Loading @@ -321,10 +332,213 @@ gcm_gmult_4bit: restore .type gcm_gmult_4bit,#function .size gcm_gmult_4bit,(.-gcm_gmult_4bit) .asciz "GHASH for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>" ___ {{{ # Straightforward 64-bits-at-a-time approach with pair of 128x64-bit # multiplications followed by 64-bit reductions. While it might be # suboptimal with regard to sheer amount of multiplications, other # methods would require larger amount of 64-bit registers, which we # don't have in 32-bit application. Also, they [alternative methods # such as aggregated reduction] kind of thrive on fast 128-bit SIMD # instructions and these are not option on SPARC... ($Xip,$Htable,$inp,$len)=map("%i$_",(0..3)); ($xE1,$Hhi,$Hlo,$Rhi,$Rlo,$M0hi,$M0lo,$M1hi,$M1lo,$Zhi,$Zlo,$X)= (map("%g$_",(1..5)),map("%o$_",(0..5,7))); ($shl,$shr)=map("%l$_",(0..7)); $code.=<<___; .globl gcm_gmult_vis3 .align 32 gcm_gmult_vis3: save %sp,-$frame,%sp ldx [$Xip+8],$X ! load X.lo ldx [$Htable-8], $Hlo ! load H ldx [$Htable-16],$Hhi mov 0xE1,$xE1 sllx $xE1,57,$xE1 xmulx $X,$Hlo,$M0lo ! HX.lo xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi ldx [$Xip+0],$X ! load X.hi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$Zlo ! Z=((HX.lo)<<1)>>64 addxc $M1hi,$M1hi,$Zhi xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xmulx $X,$Hlo,$M0lo ! HX.hi xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $Zlo, $M0lo,$M0lo xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$M1lo addxc $M1hi,$M1hi,$M1hi xor $M1lo,$Zhi,$Zlo ! Z=(Z^(HX.hi)<<1)>>64 xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi stx $Zlo,[$Xip+8] ! save Xi stx $Zhi,[$Xip+0] ret restore .type gcm_gmult_vis3,#function .size gcm_gmult_vis3,.-gcm_gmult_vis3 .globl gcm_ghash_vis3 .align 32 gcm_ghash_vis3: save %sp,-$frame,%sp ldx [$Xip+0],$Zhi ! load X.hi ldx [$Xip+8],$Zlo ! load X.lo and $inp,7,$shl andn $inp,7,$inp ldx [$Htable-8], $Hlo ! load H ldx [$Htable-16],$Hhi sll $shl,3,$shl prefetch [$inp+63], 20 mov 0xE1,$xE1 sub %g0,$shl,$shr sllx $xE1,57,$xE1 .Loop: ldx [$inp+8],$Rlo ! load *inp brz,pt $shl,1f ldx [$inp+0],$Rhi ldx [$inp+16],$X ! align data srlx $Rlo,$shr,$M0lo sllx $Rlo,$shl,$Rlo sllx $Rhi,$shl,$Rhi srlx $X,$shr,$X or $M0lo,$Rhi,$Rhi or $X,$Rlo,$Rlo 1: add $inp,16,$inp sub $len,16,$len xor $Rlo,$Zlo,$X prefetch [$inp+63], 20 xmulx $X,$Hlo,$M0lo ! HX.lo xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rhi,$Zhi,$X addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$Zlo ! Z=((HX.lo)<<1)>>64 addxc $M1hi,$M1hi,$Zhi xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xmulx $X,$Hlo,$M0lo ! HX.hi xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $Zlo, $M0lo,$M0lo xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$M1lo addxc $M1hi,$M1hi,$M1hi xor $M1lo,$Zhi,$Zlo ! Z=(Z^(HX.hi)<<1)>>64 xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 xor $Rlo,$Zlo,$Zlo ! Z^=res brnz,pt $len,.Loop xor $Rhi,$Zhi,$Zhi stx $Zlo,[$Xip+8] ! save Xi stx $Zhi,[$Xip+0] ret restore .type gcm_ghash_vis3,#function .size gcm_ghash_vis3,.-gcm_ghash_vis3 ___ }}} $code.=<<___; .asciz "GHASH for SPARCv9/VIS3, CRYPTOGAMS by <appro\@openssl.org>" .align 4 ___ $code =~ s/\`([^\`]*)\`/eval $1/gem; print $code; # Purpose of these subroutines is to explicitly encode VIS instructions, # so that one can compile the module without having to specify VIS # extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. # Idea is to reserve for option to produce "universal" binary and let # programmer detect if current CPU is VIS capable at run-time. sub unvis3 { my ($mnemonic,$rs1,$rs2,$rd)=@_; my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); my ($ref,$opf); my %visopf = ( "addxc" => 0x011, "addxccc" => 0x013, "xmulx" => 0x115, "xmulxhi" => 0x116 ); $ref = "$mnemonic\t$rs1,$rs2,$rd"; if ($opf=$visopf{$mnemonic}) { foreach ($rs1,$rs2,$rd) { return $ref if (!/%([goli])([0-9])/); $_=$bias{$1}+$2; } return sprintf ".word\t0x%08x !%s", 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, $ref; } else { return $ref; } } foreach (split("\n",$code)) { s/\`([^\`]*)\`/eval $1/ge; s/\b(xmulx[hi]*|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ &unvis3($1,$2,$3,$4) /ge; print $_,"\n"; } close STDOUT; crypto/modes/gcm128.c +16 −0 Original line number Diff line number Diff line Loading @@ -674,6 +674,13 @@ void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif # elif defined(__sparc__) || defined(__sparc) # include "sparc_arch.h" # define GHASH_ASM_SPARC # define GCM_FUNCREF_4BIT extern unsigned int OPENSSL_sparcv9cap_P[]; void gcm_gmult_vis3(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_vis3(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif #endif Loading Loading @@ -750,6 +757,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; } # elif defined(GHASH_ASM_SPARC) if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { ctx->gmult = gcm_gmult_vis3; ctx->ghash = gcm_ghash_vis3; } else { gcm_init_4bit(ctx->Htable,ctx->H.u); ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; } # else gcm_init_4bit(ctx->Htable,ctx->H.u); # endif Loading Loading
crypto/modes/asm/ghash-sparcv9.pl +217 −3 Original line number Diff line number Diff line Loading @@ -36,6 +36,13 @@ # references to input data and Z.hi updates to achieve 12 cycles # timing. To anchor to something else, sha1-sparcv9.pl spends 11.6 # cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1. # # October 2012 # # Add VIS3 lookup-table-free implementation using polynomial # multiplication xmulx[hi] and extended addition addxc[cc] # instructions. 3.96/6.26x improvement on T3/T4 or in absolute # terms 9.02/2.61 cycles per byte. $bits=32; for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } Loading Loading @@ -66,6 +73,10 @@ $Htbl="%i1"; $inp="%i2"; $len="%i3"; $code.=<<___ if ($bits==64); .register %g2,#scratch .register %g3,#scratch ___ $code.=<<___; .section ".text",#alloc,#execinstr Loading Loading @@ -321,10 +332,213 @@ gcm_gmult_4bit: restore .type gcm_gmult_4bit,#function .size gcm_gmult_4bit,(.-gcm_gmult_4bit) .asciz "GHASH for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>" ___ {{{ # Straightforward 64-bits-at-a-time approach with pair of 128x64-bit # multiplications followed by 64-bit reductions. While it might be # suboptimal with regard to sheer amount of multiplications, other # methods would require larger amount of 64-bit registers, which we # don't have in 32-bit application. Also, they [alternative methods # such as aggregated reduction] kind of thrive on fast 128-bit SIMD # instructions and these are not option on SPARC... ($Xip,$Htable,$inp,$len)=map("%i$_",(0..3)); ($xE1,$Hhi,$Hlo,$Rhi,$Rlo,$M0hi,$M0lo,$M1hi,$M1lo,$Zhi,$Zlo,$X)= (map("%g$_",(1..5)),map("%o$_",(0..5,7))); ($shl,$shr)=map("%l$_",(0..7)); $code.=<<___; .globl gcm_gmult_vis3 .align 32 gcm_gmult_vis3: save %sp,-$frame,%sp ldx [$Xip+8],$X ! load X.lo ldx [$Htable-8], $Hlo ! load H ldx [$Htable-16],$Hhi mov 0xE1,$xE1 sllx $xE1,57,$xE1 xmulx $X,$Hlo,$M0lo ! HX.lo xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi ldx [$Xip+0],$X ! load X.hi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$Zlo ! Z=((HX.lo)<<1)>>64 addxc $M1hi,$M1hi,$Zhi xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xmulx $X,$Hlo,$M0lo ! HX.hi xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $Zlo, $M0lo,$M0lo xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$M1lo addxc $M1hi,$M1hi,$M1hi xor $M1lo,$Zhi,$Zlo ! Z=(Z^(HX.hi)<<1)>>64 xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi stx $Zlo,[$Xip+8] ! save Xi stx $Zhi,[$Xip+0] ret restore .type gcm_gmult_vis3,#function .size gcm_gmult_vis3,.-gcm_gmult_vis3 .globl gcm_ghash_vis3 .align 32 gcm_ghash_vis3: save %sp,-$frame,%sp ldx [$Xip+0],$Zhi ! load X.hi ldx [$Xip+8],$Zlo ! load X.lo and $inp,7,$shl andn $inp,7,$inp ldx [$Htable-8], $Hlo ! load H ldx [$Htable-16],$Hhi sll $shl,3,$shl prefetch [$inp+63], 20 mov 0xE1,$xE1 sub %g0,$shl,$shr sllx $xE1,57,$xE1 .Loop: ldx [$inp+8],$Rlo ! load *inp brz,pt $shl,1f ldx [$inp+0],$Rhi ldx [$inp+16],$X ! align data srlx $Rlo,$shr,$M0lo sllx $Rlo,$shl,$Rlo sllx $Rhi,$shl,$Rhi srlx $X,$shr,$X or $M0lo,$Rhi,$Rhi or $X,$Rlo,$Rlo 1: add $inp,16,$inp sub $len,16,$len xor $Rlo,$Zlo,$X prefetch [$inp+63], 20 xmulx $X,$Hlo,$M0lo ! HX.lo xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rhi,$Zhi,$X addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$Zlo ! Z=((HX.lo)<<1)>>64 addxc $M1hi,$M1hi,$Zhi xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xmulx $X,$Hlo,$M0lo ! HX.hi xmulxhi $X,$Hlo,$M0hi xmulx $X,$Hhi,$M1lo xmulxhi $X,$Hhi,$M1hi xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Rhi,$Zhi,$Zhi addcc $M0lo,$M0lo,$M0lo ! (HX.lo)<<1 xor $Zlo, $M0lo,$M0lo xor $M0hi,$M1lo,$M1lo xmulx $xE1,$M0lo,$Rlo ! res=Z.lo(0xE1<<57) xmulxhi $xE1,$M0lo,$Rhi addxccc $M1lo,$M1lo,$M1lo addxc $M1hi,$M1hi,$M1hi xor $M1lo,$Zhi,$Zlo ! Z=(Z^(HX.hi)<<1)>>64 xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57 xor $Rlo,$Zlo,$Zlo ! Z^=res brnz,pt $len,.Loop xor $Rhi,$Zhi,$Zhi stx $Zlo,[$Xip+8] ! save Xi stx $Zhi,[$Xip+0] ret restore .type gcm_ghash_vis3,#function .size gcm_ghash_vis3,.-gcm_ghash_vis3 ___ }}} $code.=<<___; .asciz "GHASH for SPARCv9/VIS3, CRYPTOGAMS by <appro\@openssl.org>" .align 4 ___ $code =~ s/\`([^\`]*)\`/eval $1/gem; print $code; # Purpose of these subroutines is to explicitly encode VIS instructions, # so that one can compile the module without having to specify VIS # extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a. # Idea is to reserve for option to produce "universal" binary and let # programmer detect if current CPU is VIS capable at run-time. sub unvis3 { my ($mnemonic,$rs1,$rs2,$rd)=@_; my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 ); my ($ref,$opf); my %visopf = ( "addxc" => 0x011, "addxccc" => 0x013, "xmulx" => 0x115, "xmulxhi" => 0x116 ); $ref = "$mnemonic\t$rs1,$rs2,$rd"; if ($opf=$visopf{$mnemonic}) { foreach ($rs1,$rs2,$rd) { return $ref if (!/%([goli])([0-9])/); $_=$bias{$1}+$2; } return sprintf ".word\t0x%08x !%s", 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2, $ref; } else { return $ref; } } foreach (split("\n",$code)) { s/\`([^\`]*)\`/eval $1/ge; s/\b(xmulx[hi]*|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/ &unvis3($1,$2,$3,$4) /ge; print $_,"\n"; } close STDOUT;
crypto/modes/gcm128.c +16 −0 Original line number Diff line number Diff line Loading @@ -674,6 +674,13 @@ void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif # elif defined(__sparc__) || defined(__sparc) # include "sparc_arch.h" # define GHASH_ASM_SPARC # define GCM_FUNCREF_4BIT extern unsigned int OPENSSL_sparcv9cap_P[]; void gcm_gmult_vis3(u64 Xi[2],const u128 Htable[16]); void gcm_ghash_vis3(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); # endif #endif Loading Loading @@ -750,6 +757,15 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block) ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; } # elif defined(GHASH_ASM_SPARC) if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { ctx->gmult = gcm_gmult_vis3; ctx->ghash = gcm_ghash_vis3; } else { gcm_init_4bit(ctx->Htable,ctx->H.u); ctx->gmult = gcm_gmult_4bit; ctx->ghash = gcm_ghash_4bit; } # else gcm_init_4bit(ctx->Htable,ctx->H.u); # endif Loading
