Loading crypto/perlasm/sparcv9_modes.pl +309 −11 Original line number Diff line number Diff line #!/usr/bin/env perl # Specific mode implementations for SPARC T4 modules. # Specific modes implementations for SPARC Architecture 2011. There # is T4 dependency though, an ASI value that is not specified in the # Architecture Manual. But as SPARC universe is rather monocultural, # we imply that processor capable of executing crypto instructions # can handle the ASI in question as well. This means that we ought to # keep eyes open when new processors emerge... # # As for above mentioned ASI. It's so called "block initializing # store" which cancels "read" in "read-update-write" on cache lines. # This is "cooperative" optimization, as it reduces overall pressure # on memory interface. Benefits can't be observed/quantified with # usual benchmarks, on the contrary you can notice that single-thread # performance for parallelizable modes is ~1.5% worse. Special note # about commented 'membar' instructions, otherwise recommended by # manual. Rationale is following. Memory view is consistent from # viewpoint of processor executing the code even when ASI in question # is used. If thread on another processor has to access the result, # its availability would have to be mediated and it can be done only # through a syncronization operation which would requre ... 'membar'. # All this based on suggestions from David Miller. my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); my ($ileft,$iright,$ooff,$omask,$ivoff,$blk_init)=map("%l$_",(0..7)); sub alg_cbc_encrypt_implement { my ($alg,$bits) = @_; Loading @@ -13,6 +32,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_encrypt: save %sp, -$::frame, %sp sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff Loading @@ -33,8 +53,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f3 ___ $::code.=<<___; prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey srlx $len, 4, $len and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -42,9 +63,16 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff alignaddrl $out, %g0, $out cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}cbc_enc_blk ! $inp==$out) srl $omask, $ooff, $omask alignaddrl $out, %g0, $out srlx $len, 4, $len prefetch [$out], 22 .L${bits}_cbc_enc_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -65,6 +93,8 @@ $::code.=<<___; fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 call _${alg}${bits}_encrypt_1x add $inp, 16, $inp Loading Loading @@ -138,6 +168,68 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f8, [$ivec + $omask]0xc0 ___ $::code.=<<___; ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}cbc_enc_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init .L${bits}_cbc_enc_blk_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 5f ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 sllx %o1, $ileft, %o1 or %g1, %o0, %o0 srlx %o2, $iright, %o2 or %o2, %o1, %o1 5: xor %g4, %o0, %o0 ! ^= rk[0] xor %g5, %o1, %o1 movxtod %o0, %f12 movxtod %o1, %f14 fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 prefetch [$inp + 16+63], 20 call _${alg}${bits}_encrypt_1x add $inp, 16, $inp sub $len, 1, $len stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific brnz,pt $len, .L${bits}_cbc_enc_blk_loop add $out, 8, $out !membar 0x0f brnz,pt $blk_init, .L${bits}_cbc_enc_loop mov $blk_init, $len ___ $::code.=<<___ if ($::evp); st %f0, [$ivec + 0] st %f1, [$ivec + 4] st %f2, [$ivec + 8] st %f3, [$ivec + 12] ___ $::code.=<<___ if (!$::evp); brnz,pn $ivoff, 3b nop std %f0, [$ivec + 0] ! write out ivec std %f2, [$ivec + 8] ___ $::code.=<<___; ret restore Loading @@ -154,6 +246,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_decrypt: save %sp, -$::frame, %sp sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff Loading @@ -174,9 +267,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f15 ___ $::code.=<<___; prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_deckey srlx $len, 4, $len andcc $len, 1, %g0 ! is number of blocks even? and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -184,9 +277,17 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}cbc_dec_blk ! $inp==$out) srl $omask, $ooff, $omask andcc $len, 16, %g0 ! is number of blocks even? srlx $len, 4, $len alignaddrl $out, %g0, $out bz %icc, .L${bits}_cbc_dec_loop2x srl $omask, $ooff, $omask prefetch [$out], 22 .L${bits}_cbc_dec_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -205,6 +306,8 @@ $::code.=<<___; movxtod %o2, %f0 movxtod %o3, %f2 prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 call _${alg}${bits}_decrypt_1x add $inp, 16, $inp Loading Loading @@ -304,6 +407,8 @@ $::code.=<<___; movxtod %o4, %f4 movxtod %o5, %f6 prefetch [$out + 63], 22 prefetch [$inp + 32+63], 20 call _${alg}${bits}_decrypt_2x add $inp, 32, $inp Loading Loading @@ -392,6 +497,97 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f4, [$ivec + $omask]0xc0 ___ $::code.=<<___; ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}cbc_dec_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init sub $len, 1, $len add $blk_init, 1, $blk_init .L${bits}_cbc_dec_blk_loop2x: ldx [$inp + 0], %o0 ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 brz,pt $ileft, 5f ldx [$inp + 24], %o3 ldx [$inp + 32], %o4 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 or %g1, %o0, %o0 sllx %o1, $ileft, %o1 srlx %o2, $iright, %g1 or %g1, %o1, %o1 sllx %o2, $ileft, %o2 srlx %o3, $iright, %g1 or %g1, %o2, %o2 sllx %o3, $ileft, %o3 srlx %o4, $iright, %o4 or %o4, %o3, %o3 5: xor %g4, %o0, %o4 ! ^= rk[0] xor %g5, %o1, %o5 movxtod %o4, %f0 movxtod %o5, %f2 xor %g4, %o2, %o4 xor %g5, %o3, %o5 movxtod %o4, %f4 movxtod %o5, %f6 prefetch [$inp + 32+63], 20 call _${alg}${bits}_decrypt_2x add $inp, 32, $inp subcc $len, 2, $len movxtod %o0, %f8 movxtod %o1, %f10 fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 movxtod %o2, %f12 movxtod %o3, %f14 fxor %f8, %f4, %f4 fxor %f10, %f6, %f6 stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific bgu,pt $::size_t_cc, .L${bits}_cbc_dec_blk_loop2x add $out, 8, $out add $blk_init, $len, $len andcc $len, 1, %g0 ! is number of blocks even? !membar 0x0f bnz,pt %icc, .L${bits}_cbc_dec_loop srl $len, 0, $len brnz,pn $len, .L${bits}_cbc_dec_loop2x nop ___ $::code.=<<___ if ($::evp); st %f0, [$ivec + 0] st %f1, [$ivec + 4] st %f2, [$ivec + 8] st %f3, [$ivec + 12] ___ $::code.=<<___ if (!$::evp); brnz,pn $ivoff, 3b nop std %f0, [$ivec + 0] ! write out ivec std %f2, [$ivec + 8] ___ $::code.=<<___; ret restore Loading @@ -409,8 +605,10 @@ $::code.=<<___; ${alg}${bits}_t4_ctr32_encrypt: save %sp, -$::frame, %sp prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey nop sllx $len, 4, $len ld [$ivec + 0], %l4 ! counter ld [$ivec + 4], %l5 Loading @@ -424,7 +622,7 @@ ${alg}${bits}_t4_ctr32_encrypt: xor %g1, %g5, %g5 movxtod %g4, %f14 ! most significant 64 bits andcc $len, 1, %g0 ! is number of blocks even? sub $inp, $out, $blk_init ! $inp!=$out and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -432,9 +630,16 @@ ${alg}${bits}_t4_ctr32_encrypt: mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}_ctr32_blk ! $inp==$out) srl $omask, $ooff, $omask andcc $len, 16, %g0 ! is number of blocks even? alignaddrl $out, %g0, $out bz %icc, .L${bits}_ctr32_loop2x srl $omask, $ooff, $omask srlx $len, 4, $len .L${bits}_ctr32_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -452,6 +657,8 @@ ${alg}${bits}_t4_ctr32_encrypt: add %l7, 1, %l7 movxtod %g1, %f2 srl %l7, 0, %l7 ! clruw prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f4 Loading Loading @@ -531,7 +738,8 @@ $::code.=<<___; add %l7, 1, %l7 movxtod %g1, %f6 srl %l7, 0, %l7 ! clruw prefetch [$out + 63], 22 prefetch [$inp + 32+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f8 Loading Loading @@ -592,6 +800,96 @@ $::code.=<<___; brnz,pt $len, .L${bits}_ctr32_loop2x+4 orn %g0, $omask, $omask ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}_ctr32_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init sub $len, 1, $len add $blk_init, 1, $blk_init .L${bits}_ctr32_blk_loop2x: ldx [$inp + 0], %o0 ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 brz,pt $ileft, 5f ldx [$inp + 24], %o3 ldx [$inp + 32], %o4 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 or %g1, %o0, %o0 sllx %o1, $ileft, %o1 srlx %o2, $iright, %g1 or %g1, %o1, %o1 sllx %o2, $ileft, %o2 srlx %o3, $iright, %g1 or %g1, %o2, %o2 sllx %o3, $ileft, %o3 srlx %o4, $iright, %o4 or %o4, %o3, %o3 5: xor %g5, %l7, %g1 ! ^= rk[0] add %l7, 1, %l7 movxtod %g1, %f2 srl %l7, 0, %l7 ! clruw xor %g5, %l7, %g1 add %l7, 1, %l7 movxtod %g1, %f6 srl %l7, 0, %l7 ! clruw prefetch [$inp + 32+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f8 aes_eround23 %f18, %f14, %f2, %f2 aes_eround01 %f16, %f14, %f6, %f10 aes_eround23 %f18, %f14, %f6, %f6 ___ $::code.=<<___ if ($alg eq "cmll"); camellia_f %f16, %f2, %f14, %f2 camellia_f %f16, %f6, %f14, %f6 camellia_f %f18, %f14, %f2, %f0 camellia_f %f18, %f14, %f6, %f4 ___ $::code.=<<___; call _${alg}${bits}_encrypt_2x+16 add $inp, 32, $inp subcc $len, 2, $len movxtod %o0, %f8 movxtod %o1, %f10 movxtod %o2, %f12 fxor %f8, %f0, %f0 ! ^= inp movxtod %o3, %f8 fxor %f10, %f2, %f2 fxor %f12, %f4, %f4 fxor %f8, %f6, %f6 stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific bgu,pt $::size_t_cc, .L${bits}_ctr32_blk_loop2x add $out, 8, $out add $blk_init, $len, $len andcc $len, 1, %g0 ! is number of blocks even? !membar 0x0f bnz,pt %icc, .L${bits}_ctr32_loop srl $len, 0, $len brnz,pn $len, .L${bits}_ctr32_loop2x nop ret restore .type ${alg}${bits}_t4_ctr32_encrypt,#function Loading Loading
crypto/perlasm/sparcv9_modes.pl +309 −11 Original line number Diff line number Diff line #!/usr/bin/env perl # Specific mode implementations for SPARC T4 modules. # Specific modes implementations for SPARC Architecture 2011. There # is T4 dependency though, an ASI value that is not specified in the # Architecture Manual. But as SPARC universe is rather monocultural, # we imply that processor capable of executing crypto instructions # can handle the ASI in question as well. This means that we ought to # keep eyes open when new processors emerge... # # As for above mentioned ASI. It's so called "block initializing # store" which cancels "read" in "read-update-write" on cache lines. # This is "cooperative" optimization, as it reduces overall pressure # on memory interface. Benefits can't be observed/quantified with # usual benchmarks, on the contrary you can notice that single-thread # performance for parallelizable modes is ~1.5% worse. Special note # about commented 'membar' instructions, otherwise recommended by # manual. Rationale is following. Memory view is consistent from # viewpoint of processor executing the code even when ASI in question # is used. If thread on another processor has to access the result, # its availability would have to be mediated and it can be done only # through a syncronization operation which would requre ... 'membar'. # All this based on suggestions from David Miller. my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); my ($ileft,$iright,$ooff,$omask,$ivoff,$blk_init)=map("%l$_",(0..7)); sub alg_cbc_encrypt_implement { my ($alg,$bits) = @_; Loading @@ -13,6 +32,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_encrypt: save %sp, -$::frame, %sp sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff Loading @@ -33,8 +53,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f3 ___ $::code.=<<___; prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey srlx $len, 4, $len and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -42,9 +63,16 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff alignaddrl $out, %g0, $out cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}cbc_enc_blk ! $inp==$out) srl $omask, $ooff, $omask alignaddrl $out, %g0, $out srlx $len, 4, $len prefetch [$out], 22 .L${bits}_cbc_enc_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -65,6 +93,8 @@ $::code.=<<___; fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 call _${alg}${bits}_encrypt_1x add $inp, 16, $inp Loading Loading @@ -138,6 +168,68 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f8, [$ivec + $omask]0xc0 ___ $::code.=<<___; ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}cbc_enc_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init .L${bits}_cbc_enc_blk_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 5f ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 sllx %o1, $ileft, %o1 or %g1, %o0, %o0 srlx %o2, $iright, %o2 or %o2, %o1, %o1 5: xor %g4, %o0, %o0 ! ^= rk[0] xor %g5, %o1, %o1 movxtod %o0, %f12 movxtod %o1, %f14 fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 prefetch [$inp + 16+63], 20 call _${alg}${bits}_encrypt_1x add $inp, 16, $inp sub $len, 1, $len stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific brnz,pt $len, .L${bits}_cbc_enc_blk_loop add $out, 8, $out !membar 0x0f brnz,pt $blk_init, .L${bits}_cbc_enc_loop mov $blk_init, $len ___ $::code.=<<___ if ($::evp); st %f0, [$ivec + 0] st %f1, [$ivec + 4] st %f2, [$ivec + 8] st %f3, [$ivec + 12] ___ $::code.=<<___ if (!$::evp); brnz,pn $ivoff, 3b nop std %f0, [$ivec + 0] ! write out ivec std %f2, [$ivec + 8] ___ $::code.=<<___; ret restore Loading @@ -154,6 +246,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_decrypt: save %sp, -$::frame, %sp sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff Loading @@ -174,9 +267,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f15 ___ $::code.=<<___; prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_deckey srlx $len, 4, $len andcc $len, 1, %g0 ! is number of blocks even? and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -184,9 +277,17 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}cbc_dec_blk ! $inp==$out) srl $omask, $ooff, $omask andcc $len, 16, %g0 ! is number of blocks even? srlx $len, 4, $len alignaddrl $out, %g0, $out bz %icc, .L${bits}_cbc_dec_loop2x srl $omask, $ooff, $omask prefetch [$out], 22 .L${bits}_cbc_dec_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -205,6 +306,8 @@ $::code.=<<___; movxtod %o2, %f0 movxtod %o3, %f2 prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 call _${alg}${bits}_decrypt_1x add $inp, 16, $inp Loading Loading @@ -304,6 +407,8 @@ $::code.=<<___; movxtod %o4, %f4 movxtod %o5, %f6 prefetch [$out + 63], 22 prefetch [$inp + 32+63], 20 call _${alg}${bits}_decrypt_2x add $inp, 32, $inp Loading Loading @@ -392,6 +497,97 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f4, [$ivec + $omask]0xc0 ___ $::code.=<<___; ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}cbc_dec_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init sub $len, 1, $len add $blk_init, 1, $blk_init .L${bits}_cbc_dec_blk_loop2x: ldx [$inp + 0], %o0 ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 brz,pt $ileft, 5f ldx [$inp + 24], %o3 ldx [$inp + 32], %o4 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 or %g1, %o0, %o0 sllx %o1, $ileft, %o1 srlx %o2, $iright, %g1 or %g1, %o1, %o1 sllx %o2, $ileft, %o2 srlx %o3, $iright, %g1 or %g1, %o2, %o2 sllx %o3, $ileft, %o3 srlx %o4, $iright, %o4 or %o4, %o3, %o3 5: xor %g4, %o0, %o4 ! ^= rk[0] xor %g5, %o1, %o5 movxtod %o4, %f0 movxtod %o5, %f2 xor %g4, %o2, %o4 xor %g5, %o3, %o5 movxtod %o4, %f4 movxtod %o5, %f6 prefetch [$inp + 32+63], 20 call _${alg}${bits}_decrypt_2x add $inp, 32, $inp subcc $len, 2, $len movxtod %o0, %f8 movxtod %o1, %f10 fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 movxtod %o2, %f12 movxtod %o3, %f14 fxor %f8, %f4, %f4 fxor %f10, %f6, %f6 stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific bgu,pt $::size_t_cc, .L${bits}_cbc_dec_blk_loop2x add $out, 8, $out add $blk_init, $len, $len andcc $len, 1, %g0 ! is number of blocks even? !membar 0x0f bnz,pt %icc, .L${bits}_cbc_dec_loop srl $len, 0, $len brnz,pn $len, .L${bits}_cbc_dec_loop2x nop ___ $::code.=<<___ if ($::evp); st %f0, [$ivec + 0] st %f1, [$ivec + 4] st %f2, [$ivec + 8] st %f3, [$ivec + 12] ___ $::code.=<<___ if (!$::evp); brnz,pn $ivoff, 3b nop std %f0, [$ivec + 0] ! write out ivec std %f2, [$ivec + 8] ___ $::code.=<<___; ret restore Loading @@ -409,8 +605,10 @@ $::code.=<<___; ${alg}${bits}_t4_ctr32_encrypt: save %sp, -$::frame, %sp prefetch [$inp], 20 prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey nop sllx $len, 4, $len ld [$ivec + 0], %l4 ! counter ld [$ivec + 4], %l5 Loading @@ -424,7 +622,7 @@ ${alg}${bits}_t4_ctr32_encrypt: xor %g1, %g5, %g5 movxtod %g4, %f14 ! most significant 64 bits andcc $len, 1, %g0 ! is number of blocks even? sub $inp, $out, $blk_init ! $inp!=$out and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft Loading @@ -432,9 +630,16 @@ ${alg}${bits}_t4_ctr32_encrypt: mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff cmp $len, 127 movrnz $ooff, 0, $blk_init ! if ( $out&7 || movleu $::size_t_cc, 0, $blk_init ! $len<128 || brnz,pn $blk_init, .L${bits}_ctr32_blk ! $inp==$out) srl $omask, $ooff, $omask andcc $len, 16, %g0 ! is number of blocks even? alignaddrl $out, %g0, $out bz %icc, .L${bits}_ctr32_loop2x srl $omask, $ooff, $omask srlx $len, 4, $len .L${bits}_ctr32_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f Loading @@ -452,6 +657,8 @@ ${alg}${bits}_t4_ctr32_encrypt: add %l7, 1, %l7 movxtod %g1, %f2 srl %l7, 0, %l7 ! clruw prefetch [$out + 63], 22 prefetch [$inp + 16+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f4 Loading Loading @@ -531,7 +738,8 @@ $::code.=<<___; add %l7, 1, %l7 movxtod %g1, %f6 srl %l7, 0, %l7 ! clruw prefetch [$out + 63], 22 prefetch [$inp + 32+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f8 Loading Loading @@ -592,6 +800,96 @@ $::code.=<<___; brnz,pt $len, .L${bits}_ctr32_loop2x+4 orn %g0, $omask, $omask ret restore !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .align 32 .L${bits}_ctr32_blk: add $out, $len, $blk_init and $blk_init, 63, $blk_init ! tail sub $len, $blk_init, $len add $blk_init, 15, $blk_init ! round up to 16n srlx $len, 4, $len srl $blk_init, 4, $blk_init sub $len, 1, $len add $blk_init, 1, $blk_init .L${bits}_ctr32_blk_loop2x: ldx [$inp + 0], %o0 ldx [$inp + 8], %o1 ldx [$inp + 16], %o2 brz,pt $ileft, 5f ldx [$inp + 24], %o3 ldx [$inp + 32], %o4 sllx %o0, $ileft, %o0 srlx %o1, $iright, %g1 or %g1, %o0, %o0 sllx %o1, $ileft, %o1 srlx %o2, $iright, %g1 or %g1, %o1, %o1 sllx %o2, $ileft, %o2 srlx %o3, $iright, %g1 or %g1, %o2, %o2 sllx %o3, $ileft, %o3 srlx %o4, $iright, %o4 or %o4, %o3, %o3 5: xor %g5, %l7, %g1 ! ^= rk[0] add %l7, 1, %l7 movxtod %g1, %f2 srl %l7, 0, %l7 ! clruw xor %g5, %l7, %g1 add %l7, 1, %l7 movxtod %g1, %f6 srl %l7, 0, %l7 ! clruw prefetch [$inp + 32+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f8 aes_eround23 %f18, %f14, %f2, %f2 aes_eround01 %f16, %f14, %f6, %f10 aes_eround23 %f18, %f14, %f6, %f6 ___ $::code.=<<___ if ($alg eq "cmll"); camellia_f %f16, %f2, %f14, %f2 camellia_f %f16, %f6, %f14, %f6 camellia_f %f18, %f14, %f2, %f0 camellia_f %f18, %f14, %f6, %f4 ___ $::code.=<<___; call _${alg}${bits}_encrypt_2x+16 add $inp, 32, $inp subcc $len, 2, $len movxtod %o0, %f8 movxtod %o1, %f10 movxtod %o2, %f12 fxor %f8, %f0, %f0 ! ^= inp movxtod %o3, %f8 fxor %f10, %f2, %f2 fxor %f12, %f4, %f4 fxor %f8, %f6, %f6 stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific add $out, 8, $out stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific bgu,pt $::size_t_cc, .L${bits}_ctr32_blk_loop2x add $out, 8, $out add $blk_init, $len, $len andcc $len, 1, %g0 ! is number of blocks even? !membar 0x0f bnz,pt %icc, .L${bits}_ctr32_loop srl $len, 0, $len brnz,pn $len, .L${bits}_ctr32_loop2x nop ret restore .type ${alg}${bits}_t4_ctr32_encrypt,#function Loading
