aesni-x86_64.pl

	jmp	.Lxts_dec_done

.align	16
.Lxts_dec_two:
	movups	($inp),$inout0
	movups	16($inp),$inout1
	lea	32($inp),$inp
	xorps	@tweak[0],$inout0
	xorps	@tweak[1],$inout1

	call	_aesni_decrypt3

	xorps	@tweak[0],$inout0
	movdqa	@tweak[2],@tweak[0]
	xorps	@tweak[1],$inout1
	movdqa	@tweak[3],@tweak[1]
	movups	$inout0,($out)
	movups	$inout1,16*1($out)
	lea	16*2($out),$out
	jmp	.Lxts_dec_done

.align	16
.Lxts_dec_three:
	movups	($inp),$inout0
	movups	16*1($inp),$inout1
	movups	16*2($inp),$inout2
	lea	16*3($inp),$inp
	xorps	@tweak[0],$inout0
	xorps	@tweak[1],$inout1
	xorps	@tweak[2],$inout2

	call	_aesni_decrypt3

	xorps	@tweak[0],$inout0
	movdqa	@tweak[3],@tweak[0]
	xorps	@tweak[1],$inout1
	movdqa	@tweak[5],@tweak[1]
	xorps	@tweak[2],$inout2
	movups	$inout0,($out)
	movups	$inout1,16*1($out)
	movups	$inout2,16*2($out)
	lea	16*3($out),$out
	jmp	.Lxts_dec_done

.align	16
.Lxts_dec_four:
	pshufd	\$0x13,$twtmp,$twres
	movdqa	@tweak[5],@tweak[4]
	paddq	@tweak[5],@tweak[5]		# psllq 1,$tweak
	 movups	($inp),$inout0
	pand	$twmask,$twres			# isolate carry and residue
	 movups	16*1($inp),$inout1
	pxor	$twres,@tweak[5]

	movups	16*2($inp),$inout2
	xorps	@tweak[0],$inout0
	movups	16*3($inp),$inout3
	lea	16*4($inp),$inp
	xorps	@tweak[1],$inout1
	xorps	@tweak[2],$inout2
	xorps	@tweak[3],$inout3

	call	_aesni_decrypt4

	xorps	@tweak[0],$inout0
	movdqa	@tweak[4],@tweak[0]
	xorps	@tweak[1],$inout1
	movdqa	@tweak[5],@tweak[1]
	xorps	@tweak[2],$inout2
	movups	$inout0,($out)
	xorps	@tweak[3],$inout3
	movups	$inout1,16*1($out)
	movups	$inout2,16*2($out)
	movups	$inout3,16*3($out)
	lea	16*4($out),$out
	jmp	.Lxts_dec_done

.align	16
.Lxts_dec_done:
	and	\$15,$len_
	jz	.Lxts_dec_ret
.Lxts_dec_done2:
	mov	$len_,$len
	mov	$key_,$key			# restore $key
	mov	$rnds_,$rounds			# restore $rounds

	movups	($inp),$inout0
	xorps	@tweak[1],$inout0
___
	&aesni_generate1("dec",$key,$rounds);
$code.=<<___;
	xorps	@tweak[1],$inout0
	movups	$inout0,($out)

.Lxts_dec_steal:
	movzb	16($inp),%eax			# borrow $rounds ...
	movzb	($out),%ecx			# ... and $key
	lea	1($inp),$inp
	mov	%al,($out)
	mov	%cl,16($out)
	lea	1($out),$out
	sub	\$1,$len
	jnz	.Lxts_dec_steal

	sub	$len_,$out			# rewind $out
	mov	$key_,$key			# restore $key
	mov	$rnds_,$rounds			# restore $rounds

	movups	($out),$inout0
	xorps	@tweak[0],$inout0
___
	&aesni_generate1("dec",$key,$rounds);
$code.=<<___;
	xorps	@tweak[0],$inout0
	movups	$inout0,($out)

.Lxts_dec_ret:
___
$code.=<<___ if ($win64);
	movaps	0x60(%rsp),%xmm6
	movaps	0x70(%rsp),%xmm7
	movaps	0x80(%rsp),%xmm8
	movaps	0x90(%rsp),%xmm9
	movaps	0xa0(%rsp),%xmm10
	movaps	0xb0(%rsp),%xmm11
	movaps	0xc0(%rsp),%xmm12
	movaps	0xd0(%rsp),%xmm13
	movaps	0xe0(%rsp),%xmm14
	movaps	0xf0(%rsp),%xmm15
___
$code.=<<___;
	lea	$frame_size(%rsp),%rsp
.Lxts_dec_epilogue:
	ret
.size	aesni_xts_decrypt,.-aesni_xts_decrypt
___
} }}

########################################################################
# void $PREFIX_cbc_encrypt (const void *inp, void *out,
#			    size_t length, const AES_KEY *key,
#			    unsigned char *ivp,const int enc);
{
my $reserved = $win64?0x40:-0x18;	# used in decrypt
$code.=<<___;
.globl	${PREFIX}_cbc_encrypt
.type	${PREFIX}_cbc_encrypt,\@function,6
.align	16
${PREFIX}_cbc_encrypt:
	test	$len,$len		# check length
	jz	.Lcbc_ret

	mov	240($key),$rnds_	# key->rounds
	mov	$key,$key_		# backup $key
	test	%r9d,%r9d		# 6th argument
	jz	.Lcbc_decrypt
#--------------------------- CBC ENCRYPT ------------------------------#
	movups	($ivp),$inout0		# load iv as initial state
	mov	$rnds_,$rounds
	cmp	\$16,$len
	jb	.Lcbc_enc_tail
	sub	\$16,$len
	jmp	.Lcbc_enc_loop
.align	16
.Lcbc_enc_loop:
	movups	($inp),$inout1		# load input
	lea	16($inp),$inp
	#xorps	$inout1,$inout0
___
	&aesni_generate1("enc",$key,$rounds,$inout0,$inout1);
$code.=<<___;
	mov	$rnds_,$rounds		# restore $rounds
	mov	$key_,$key		# restore $key
	movups	$inout0,0($out)		# store output
	lea	16($out),$out
	sub	\$16,$len
	jnc	.Lcbc_enc_loop
	add	\$16,$len
	jnz	.Lcbc_enc_tail
	movups	$inout0,($ivp)
	jmp	.Lcbc_ret

.Lcbc_enc_tail:
	mov	$len,%rcx	# zaps $key
	xchg	$inp,$out	# $inp is %rsi and $out is %rdi now
	.long	0x9066A4F3	# rep movsb
	mov	\$16,%ecx	# zero tail
	sub	$len,%rcx
	xor	%eax,%eax
	.long	0x9066AAF3	# rep stosb
	lea	-16(%rdi),%rdi	# rewind $out by 1 block
	mov	$rnds_,$rounds	# restore $rounds
	mov	%rdi,%rsi	# $inp and $out are the same
	mov	$key_,$key	# restore $key
	xor	$len,$len	# len=16
	jmp	.Lcbc_enc_loop	# one more spin
#--------------------------- CBC DECRYPT ------------------------------#
.align	16
.Lcbc_decrypt:
___
$code.=<<___ if ($win64);
	lea	-0x58(%rsp),%rsp
	movaps	%xmm6,(%rsp)
	movaps	%xmm7,0x10(%rsp)
	movaps	%xmm8,0x20(%rsp)
	movaps	%xmm9,0x30(%rsp)
.Lcbc_decrypt_body:
___
$code.=<<___;
	movups	($ivp),$iv
	mov	$rnds_,$rounds
	cmp	\$0x70,$len
	jbe	.Lcbc_dec_tail
	shr	\$1,$rnds_
	sub	\$0x70,$len
	mov	$rnds_,$rounds
	movaps	$iv,$reserved(%rsp)
	jmp	.Lcbc_dec_loop8_enter
.align	16
.Lcbc_dec_loop8:
	movaps	$rndkey0,$reserved(%rsp)	# save IV
	movups	$inout7,($out)
	lea	0x10($out),$out
.Lcbc_dec_loop8_enter:
	$movkey		($key),$rndkey0
	movups	($inp),$inout0			# load input
	movups	0x10($inp),$inout1
	$movkey		16($key),$rndkey1

	lea		32($key),$key
	movdqu	0x20($inp),$inout2
	xorps		$rndkey0,$inout0
	movdqu	0x30($inp),$inout3
	xorps		$rndkey0,$inout1
	movdqu	0x40($inp),$inout4
	aesdec		$rndkey1,$inout0
	pxor		$rndkey0,$inout2
	movdqu	0x50($inp),$inout5
	aesdec		$rndkey1,$inout1
	pxor		$rndkey0,$inout3
	movdqu	0x60($inp),$inout6
	aesdec		$rndkey1,$inout2
	pxor		$rndkey0,$inout4
	movdqu	0x70($inp),$inout7
	aesdec		$rndkey1,$inout3
	pxor		$rndkey0,$inout5
	dec		$rounds
	aesdec		$rndkey1,$inout4
	pxor		$rndkey0,$inout6
	aesdec		$rndkey1,$inout5
	pxor		$rndkey0,$inout7
	$movkey		($key),$rndkey0
	aesdec		$rndkey1,$inout6
	aesdec		$rndkey1,$inout7
	$movkey		16($key),$rndkey1

	call		.Ldec_loop8_enter

	movups	($inp),$rndkey1		# re-load input
	movups	0x10($inp),$rndkey0
	xorps	$reserved(%rsp),$inout0	# ^= IV
	xorps	$rndkey1,$inout1
	movups	0x20($inp),$rndkey1
	xorps	$rndkey0,$inout2
	movups	0x30($inp),$rndkey0
	xorps	$rndkey1,$inout3
	movups	0x40($inp),$rndkey1
	xorps	$rndkey0,$inout4
	movups	0x50($inp),$rndkey0
	xorps	$rndkey1,$inout5
	movups	0x60($inp),$rndkey1
	xorps	$rndkey0,$inout6
	movups	0x70($inp),$rndkey0	# IV
	xorps	$rndkey1,$inout7
	movups	$inout0,($out)
	movups	$inout1,0x10($out)
	movups	$inout2,0x20($out)
	movups	$inout3,0x30($out)
	mov	$rnds_,$rounds		# restore $rounds
	movups	$inout4,0x40($out)
	mov	$key_,$key		# restore $key
	movups	$inout5,0x50($out)
	lea	0x80($inp),$inp
	movups	$inout6,0x60($out)
	lea	0x70($out),$out
	sub	\$0x80,$len
	ja	.Lcbc_dec_loop8

	movaps	$inout7,$inout0
	movaps	$rndkey0,$iv
	add	\$0x70,$len
	jle	.Lcbc_dec_tail_collected
	movups	$inout0,($out)
	lea	1($rnds_,$rnds_),$rounds
	lea	0x10($out),$out
.Lcbc_dec_tail:
	movups	($inp),$inout0
	movaps	$inout0,$in0
	cmp	\$0x10,$len
	jbe	.Lcbc_dec_one

	movups	0x10($inp),$inout1
	movaps	$inout1,$in1
	cmp	\$0x20,$len
	jbe	.Lcbc_dec_two

	movups	0x20($inp),$inout2
	movaps	$inout2,$in2
	cmp	\$0x30,$len
	jbe	.Lcbc_dec_three

	movups	0x30($inp),$inout3
	cmp	\$0x40,$len
	jbe	.Lcbc_dec_four

	movups	0x40($inp),$inout4
	cmp	\$0x50,$len
	jbe	.Lcbc_dec_five

	movups	0x50($inp),$inout5
	cmp	\$0x60,$len
	jbe	.Lcbc_dec_six

	movups	0x60($inp),$inout6
	movaps	$iv,$reserved(%rsp)	# save IV
	call	_aesni_decrypt8
	movups	($inp),$rndkey1
	movups	0x10($inp),$rndkey0
	xorps	$reserved(%rsp),$inout0	# ^= IV
	xorps	$rndkey1,$inout1
	movups	0x20($inp),$rndkey1
	xorps	$rndkey0,$inout2
	movups	0x30($inp),$rndkey0
	xorps	$rndkey1,$inout3
	movups	0x40($inp),$rndkey1
	xorps	$rndkey0,$inout4
	movups	0x50($inp),$rndkey0
	xorps	$rndkey1,$inout5
	movups	0x60($inp),$iv		# IV
	xorps	$rndkey0,$inout6
	movups	$inout0,($out)
	movups	$inout1,0x10($out)
	movups	$inout2,0x20($out)
	movups	$inout3,0x30($out)
	movups	$inout4,0x40($out)
	movups	$inout5,0x50($out)
	lea	0x60($out),$out
	movaps	$inout6,$inout0
	sub	\$0x70,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_one:
___
	&aesni_generate1("dec",$key,$rounds);
$code.=<<___;
	xorps	$iv,$inout0
	movaps	$in0,$iv
	sub	\$0x10,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_two:
	xorps	$inout2,$inout2
	call	_aesni_decrypt3
	xorps	$iv,$inout0
	xorps	$in0,$inout1
	movups	$inout0,($out)
	movaps	$in1,$iv
	movaps	$inout1,$inout0
	lea	0x10($out),$out
	sub	\$0x20,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_three:
	call	_aesni_decrypt3
	xorps	$iv,$inout0
	xorps	$in0,$inout1
	movups	$inout0,($out)
	xorps	$in1,$inout2
	movups	$inout1,0x10($out)
	movaps	$in2,$iv
	movaps	$inout2,$inout0
	lea	0x20($out),$out
	sub	\$0x30,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_four:
	call	_aesni_decrypt4
	xorps	$iv,$inout0
	movups	0x30($inp),$iv
	xorps	$in0,$inout1
	movups	$inout0,($out)
	xorps	$in1,$inout2
	movups	$inout1,0x10($out)
	xorps	$in2,$inout3
	movups	$inout2,0x20($out)
	movaps	$inout3,$inout0
	lea	0x30($out),$out
	sub	\$0x40,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_five:
	xorps	$inout5,$inout5
	call	_aesni_decrypt6
	movups	0x10($inp),$rndkey1
	movups	0x20($inp),$rndkey0
	xorps	$iv,$inout0
	xorps	$in0,$inout1
	xorps	$rndkey1,$inout2
	movups	0x30($inp),$rndkey1
	xorps	$rndkey0,$inout3
	movups	0x40($inp),$iv
	xorps	$rndkey1,$inout4
	movups	$inout0,($out)
	movups	$inout1,0x10($out)
	movups	$inout2,0x20($out)
	movups	$inout3,0x30($out)
	lea	0x40($out),$out
	movaps	$inout4,$inout0
	sub	\$0x50,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_six:
	call	_aesni_decrypt6
	movups	0x10($inp),$rndkey1
	movups	0x20($inp),$rndkey0
	xorps	$iv,$inout0
	xorps	$in0,$inout1
	xorps	$rndkey1,$inout2
	movups	0x30($inp),$rndkey1
	xorps	$rndkey0,$inout3
	movups	0x40($inp),$rndkey0
	xorps	$rndkey1,$inout4
	movups	0x50($inp),$iv
	xorps	$rndkey0,$inout5
	movups	$inout0,($out)
	movups	$inout1,0x10($out)
	movups	$inout2,0x20($out)
	movups	$inout3,0x30($out)
	movups	$inout4,0x40($out)
	lea	0x50($out),$out
	movaps	$inout5,$inout0
	sub	\$0x60,$len
	jmp	.Lcbc_dec_tail_collected
.align	16
.Lcbc_dec_tail_collected:
	and	\$15,$len
	movups	$iv,($ivp)
	jnz	.Lcbc_dec_tail_partial
	movups	$inout0,($out)
	jmp	.Lcbc_dec_ret
.align	16
.Lcbc_dec_tail_partial:
	movaps	$inout0,$reserved(%rsp)
	mov	\$16,%rcx
	mov	$out,%rdi
	sub	$len,%rcx
	lea	$reserved(%rsp),%rsi
	.long	0x9066A4F3	# rep movsb

.Lcbc_dec_ret:
___
$code.=<<___ if ($win64);
	movaps	(%rsp),%xmm6
	movaps	0x10(%rsp),%xmm7
	movaps	0x20(%rsp),%xmm8
	movaps	0x30(%rsp),%xmm9
	lea	0x58(%rsp),%rsp
___
$code.=<<___;
.Lcbc_ret:
	ret
.size	${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
___
} 
# int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey,
#				int bits, AES_KEY *key)
{ my ($inp,$bits,$key) = @_4args;
  $bits =~ s/%r/%e/;

$code.=<<___;
.globl	${PREFIX}_set_decrypt_key
.type	${PREFIX}_set_decrypt_key,\@abi-omnipotent
.align	16
${PREFIX}_set_decrypt_key:
	.byte	0x48,0x83,0xEC,0x08	# sub rsp,8
	call	__aesni_set_encrypt_key
	shl	\$4,$bits		# rounds-1 after _aesni_set_encrypt_key
	test	%eax,%eax
	jnz	.Ldec_key_ret
	lea	16($key,$bits),$inp	# points at the end of key schedule

	$movkey	($key),%xmm0		# just swap
	$movkey	($inp),%xmm1
	$movkey	%xmm0,($inp)
	$movkey	%xmm1,($key)
	lea	16($key),$key
	lea	-16($inp),$inp

.Ldec_key_inverse:
	$movkey	($key),%xmm0		# swap and inverse
	$movkey	($inp),%xmm1
	aesimc	%xmm0,%xmm0
	aesimc	%xmm1,%xmm1
	lea	16($key),$key
	lea	-16($inp),$inp
	$movkey	%xmm0,16($inp)
	$movkey	%xmm1,-16($key)
	cmp	$key,$inp
	ja	.Ldec_key_inverse

	$movkey	($key),%xmm0		# inverse middle
	aesimc	%xmm0,%xmm0
	$movkey	%xmm0,($inp)
.Ldec_key_ret:
	add	\$8,%rsp
	ret
.LSEH_end_set_decrypt_key:
.size	${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key
___

# This is based on submission by
#
#	Huang Ying <ying.huang@intel.com>
#	Vinodh Gopal <vinodh.gopal@intel.com>
#	Kahraman Akdemir
#
# Agressively optimized in respect to aeskeygenassist's critical path
# and is contained in %xmm0-5 to meet Win64 ABI requirement.
#
$code.=<<___;
.globl	${PREFIX}_set_encrypt_key
.type	${PREFIX}_set_encrypt_key,\@abi-omnipotent
.align	16
${PREFIX}_set_encrypt_key:
__aesni_set_encrypt_key:
	.byte	0x48,0x83,0xEC,0x08	# sub rsp,8
	mov	\$-1,%rax
	test	$inp,$inp
	jz	.Lenc_key_ret
	test	$key,$key
	jz	.Lenc_key_ret

	movups	($inp),%xmm0		# pull first 128 bits of *userKey
	xorps	%xmm4,%xmm4		# low dword of xmm4 is assumed 0
	lea	16($key),%rax
	cmp	\$256,$bits
	je	.L14rounds
	cmp	\$192,$bits
	je	.L12rounds
	cmp	\$128,$bits
	jne	.Lbad_keybits

.L10rounds:
	mov	\$9,$bits			# 10 rounds for 128-bit key
	$movkey	%xmm0,($key)			# round 0
	aeskeygenassist	\$0x1,%xmm0,%xmm1	# round 1
	call		.Lkey_expansion_128_cold
	aeskeygenassist	\$0x2,%xmm0,%xmm1	# round 2
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x4,%xmm0,%xmm1	# round 3
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x8,%xmm0,%xmm1	# round 4
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x10,%xmm0,%xmm1	# round 5
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x20,%xmm0,%xmm1	# round 6
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x40,%xmm0,%xmm1	# round 7
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x80,%xmm0,%xmm1	# round 8
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x1b,%xmm0,%xmm1	# round 9
	call		.Lkey_expansion_128
	aeskeygenassist	\$0x36,%xmm0,%xmm1	# round 10
	call		.Lkey_expansion_128
	$movkey	%xmm0,(%rax)
	mov	$bits,80(%rax)	# 240(%rdx)
	xor	%eax,%eax
	jmp	.Lenc_key_ret

.align	16
.L12rounds:
	movq	16($inp),%xmm2			# remaining 1/3 of *userKey
	mov	\$11,$bits			# 12 rounds for 192
	$movkey	%xmm0,($key)			# round 0
	aeskeygenassist	\$0x1,%xmm2,%xmm1	# round 1,2
	call		.Lkey_expansion_192a_cold
	aeskeygenassist	\$0x2,%xmm2,%xmm1	# round 2,3
	call		.Lkey_expansion_192b
	aeskeygenassist	\$0x4,%xmm2,%xmm1	# round 4,5
	call		.Lkey_expansion_192a
	aeskeygenassist	\$0x8,%xmm2,%xmm1	# round 5,6
	call		.Lkey_expansion_192b
	aeskeygenassist	\$0x10,%xmm2,%xmm1	# round 7,8
	call		.Lkey_expansion_192a
	aeskeygenassist	\$0x20,%xmm2,%xmm1	# round 8,9
	call		.Lkey_expansion_192b
	aeskeygenassist	\$0x40,%xmm2,%xmm1	# round 10,11
	call		.Lkey_expansion_192a
	aeskeygenassist	\$0x80,%xmm2,%xmm1	# round 11,12
	call		.Lkey_expansion_192b
	$movkey	%xmm0,(%rax)
	mov	$bits,48(%rax)	# 240(%rdx)
	xor	%rax, %rax
	jmp	.Lenc_key_ret

.align	16
.L14rounds:
	movups	16($inp),%xmm2			# remaning half of *userKey
	mov	\$13,$bits			# 14 rounds for 256
	lea	16(%rax),%rax
	$movkey	%xmm0,($key)			# round 0
	$movkey	%xmm2,16($key)			# round 1
	aeskeygenassist	\$0x1,%xmm2,%xmm1	# round 2
	call		.Lkey_expansion_256a_cold
	aeskeygenassist	\$0x1,%xmm0,%xmm1	# round 3
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x2,%xmm2,%xmm1	# round 4
	call		.Lkey_expansion_256a
	aeskeygenassist	\$0x2,%xmm0,%xmm1	# round 5
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x4,%xmm2,%xmm1	# round 6
	call		.Lkey_expansion_256a
	aeskeygenassist	\$0x4,%xmm0,%xmm1	# round 7
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x8,%xmm2,%xmm1	# round 8
	call		.Lkey_expansion_256a
	aeskeygenassist	\$0x8,%xmm0,%xmm1	# round 9
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x10,%xmm2,%xmm1	# round 10
	call		.Lkey_expansion_256a
	aeskeygenassist	\$0x10,%xmm0,%xmm1	# round 11
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x20,%xmm2,%xmm1	# round 12
	call		.Lkey_expansion_256a
	aeskeygenassist	\$0x20,%xmm0,%xmm1	# round 13
	call		.Lkey_expansion_256b
	aeskeygenassist	\$0x40,%xmm2,%xmm1	# round 14
	call		.Lkey_expansion_256a
	$movkey	%xmm0,(%rax)
	mov	$bits,16(%rax)	# 240(%rdx)
	xor	%rax,%rax
	jmp	.Lenc_key_ret

.align	16
.Lbad_keybits:
	mov	\$-2,%rax
.Lenc_key_ret:
	add	\$8,%rsp
	ret
.LSEH_end_set_encrypt_key:

.align	16
.Lkey_expansion_128:
	$movkey	%xmm0,(%rax)
	lea	16(%rax),%rax
.Lkey_expansion_128_cold:
	shufps	\$0b00010000,%xmm0,%xmm4
	xorps	%xmm4, %xmm0
	shufps	\$0b10001100,%xmm0,%xmm4
	xorps	%xmm4, %xmm0
	shufps	\$0b11111111,%xmm1,%xmm1	# critical path
	xorps	%xmm1,%xmm0
	ret

.align 16
.Lkey_expansion_192a:
	$movkey	%xmm0,(%rax)
	lea	16(%rax),%rax
.Lkey_expansion_192a_cold:
	movaps	%xmm2, %xmm5
.Lkey_expansion_192b_warm:
	shufps	\$0b00010000,%xmm0,%xmm4
	movdqa	%xmm2,%xmm3
	xorps	%xmm4,%xmm0
	shufps	\$0b10001100,%xmm0,%xmm4
	pslldq	\$4,%xmm3
	xorps	%xmm4,%xmm0
	pshufd	\$0b01010101,%xmm1,%xmm1	# critical path
	pxor	%xmm3,%xmm2
	pxor	%xmm1,%xmm0
	pshufd	\$0b11111111,%xmm0,%xmm3
	pxor	%xmm3,%xmm2
	ret

.align 16
.Lkey_expansion_192b:
	movaps	%xmm0,%xmm3
	shufps	\$0b01000100,%xmm0,%xmm5
	$movkey	%xmm5,(%rax)
	shufps	\$0b01001110,%xmm2,%xmm3
	$movkey	%xmm3,16(%rax)
	lea	32(%rax),%rax
	jmp	.Lkey_expansion_192b_warm

.align	16
.Lkey_expansion_256a:
	$movkey	%xmm2,(%rax)
	lea	16(%rax),%rax
.Lkey_expansion_256a_cold:
	shufps	\$0b00010000,%xmm0,%xmm4
	xorps	%xmm4,%xmm0
	shufps	\$0b10001100,%xmm0,%xmm4
	xorps	%xmm4,%xmm0
	shufps	\$0b11111111,%xmm1,%xmm1	# critical path
	xorps	%xmm1,%xmm0
	ret

.align 16
.Lkey_expansion_256b:
	$movkey	%xmm0,(%rax)
	lea	16(%rax),%rax

	shufps	\$0b00010000,%xmm2,%xmm4
	xorps	%xmm4,%xmm2
	shufps	\$0b10001100,%xmm2,%xmm4
	xorps	%xmm4,%xmm2
	shufps	\$0b10101010,%xmm1,%xmm1	# critical path
	xorps	%xmm1,%xmm2
	ret
.size	${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
.size	__aesni_set_encrypt_key,.-__aesni_set_encrypt_key
___
}

$code.=<<___;
.align	64
.Lbswap_mask:
	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
.Lincrement32:
	.long	6,6,6,0
.Lincrement64:
	.long	1,0,0,0
.Lxts_magic:
	.long	0x87,0,1,0

.asciz  "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"
.align	64
___

# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
#		CONTEXT *context,DISPATCHER_CONTEXT *disp)
if ($win64) {
$rec="%rcx";
$frame="%rdx";
$context="%r8";
$disp="%r9";

$code.=<<___;
.extern	__imp_RtlVirtualUnwind
___
$code.=<<___ if ($PREFIX eq "aesni");
.type	ecb_ccm64_se_handler,\@abi-omnipotent
.align	16
ecb_ccm64_se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# prologue label
	cmp	%r10,%rbx		# context->Rip<prologue label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lcommon_seh_tail

	lea	0(%rax),%rsi		# %xmm save area
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$8,%ecx		# 4*sizeof(%xmm0)/sizeof(%rax)
	.long	0xa548f3fc		# cld; rep movsq
	lea	0x58(%rax),%rax		# adjust stack pointer

	jmp	.Lcommon_seh_tail
.size	ecb_ccm64_se_handler,.-ecb_ccm64_se_handler

.type	ctr32_se_handler,\@abi-omnipotent
.align	16
ctr32_se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	lea	.Lctr32_body(%rip),%r10
	cmp	%r10,%rbx		# context->Rip<"prologue" label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp

	lea	.Lctr32_ret(%rip),%r10
	cmp	%r10,%rbx
	jae	.Lcommon_seh_tail

	lea	0x20(%rax),%rsi		# %xmm save area
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
	.long	0xa548f3fc		# cld; rep movsq
	lea	0xc8(%rax),%rax		# adjust stack pointer

	jmp	.Lcommon_seh_tail
.size	ctr32_se_handler,.-ctr32_se_handler

.type	xts_se_handler,\@abi-omnipotent
.align	16
xts_se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	120($context),%rax	# pull context->Rax
	mov	248($context),%rbx	# pull context->Rip

	mov	8($disp),%rsi		# disp->ImageBase
	mov	56($disp),%r11		# disp->HandlerData

	mov	0(%r11),%r10d		# HandlerData[0]
	lea	(%rsi,%r10),%r10	# prologue lable
	cmp	%r10,%rbx		# context->Rip<prologue label
	jb	.Lcommon_seh_tail

	mov	152($context),%rax	# pull context->Rsp

	mov	4(%r11),%r10d		# HandlerData[1]
	lea	(%rsi,%r10),%r10	# epilogue label
	cmp	%r10,%rbx		# context->Rip>=epilogue label
	jae	.Lcommon_seh_tail

	lea	0x60(%rax),%rsi		# %xmm save area
	lea	512($context),%rdi	# & context.Xmm6
	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)
	.long	0xa548f3fc		# cld; rep movsq
	lea	0x68+160(%rax),%rax	# adjust stack pointer

	jmp	.Lcommon_seh_tail
.size	xts_se_handler,.-xts_se_handler
___
$code.=<<___;
.type	cbc_se_handler,\@abi-omnipotent
.align	16
cbc_se_handler:
	push	%rsi
	push	%rdi
	push	%rbx
	push	%rbp
	push	%r12
	push	%r13
	push	%r14
	push	%r15
	pushfq
	sub	\$64,%rsp

	mov	152($context),%rax	# pull context->Rsp
	mov	248($context),%rbx	# pull context->Rip

	lea	.Lcbc_decrypt(%rip),%r10
	cmp	%r10,%rbx		# context->Rip<"prologue" label
	jb	.Lcommon_seh_tail

	lea	.Lcbc_decrypt_body(%rip),%r10
	cmp	%r10,%rbx		# context->Rip<cbc_decrypt_body
	jb	.Lrestore_cbc_rax

	lea	.Lcbc_ret(%rip),%r10
	cmp	%r10,%rbx		# context->Rip>="epilogue" label
	jae	.Lcommon_seh_tail

	lea	0(%rax),%rsi		# top of stack
	lea	512($context),%rdi	# &context.Xmm6
	mov	\$8,%ecx		# 4*sizeof(%xmm0)/sizeof(%rax)
	.long	0xa548f3fc		# cld; rep movsq
	lea	0x58(%rax),%rax		# adjust stack pointer
	jmp	.Lcommon_seh_tail

.Lrestore_cbc_rax:
	mov	120($context),%rax

.Lcommon_seh_tail:
	mov	8(%rax),%rdi
	mov	16(%rax),%rsi
	mov	%rax,152($context)	# restore context->Rsp
	mov	%rsi,168($context)	# restore context->Rsi
	mov	%rdi,176($context)	# restore context->Rdi

	mov	40($disp),%rdi		# disp->ContextRecord
	mov	$context,%rsi		# context
	mov	\$154,%ecx		# sizeof(CONTEXT)
	.long	0xa548f3fc		# cld; rep movsq

	mov	$disp,%rsi
	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER
	mov	8(%rsi),%rdx		# arg2, disp->ImageBase
	mov	0(%rsi),%r8		# arg3, disp->ControlPc
	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry
	mov	40(%rsi),%r10		# disp->ContextRecord
	lea	56(%rsi),%r11		# &disp->HandlerData
	lea	24(%rsi),%r12		# &disp->EstablisherFrame
	mov	%r10,32(%rsp)		# arg5
	mov	%r11,40(%rsp)		# arg6
	mov	%r12,48(%rsp)		# arg7
	mov	%rcx,56(%rsp)		# arg8, (NULL)
	call	*__imp_RtlVirtualUnwind(%rip)

	mov	\$1,%eax		# ExceptionContinueSearch
	add	\$64,%rsp
	popfq
	pop	%r15
	pop	%r14
	pop	%r13
	pop	%r12
	pop	%rbp
	pop	%rbx
	pop	%rdi
	pop	%rsi
	ret
.size	cbc_se_handler,.-cbc_se_handler

.section	.pdata
.align	4
___
$code.=<<___ if ($PREFIX eq "aesni");
	.rva	.LSEH_begin_aesni_ecb_encrypt
	.rva	.LSEH_end_aesni_ecb_encrypt
	.rva	.LSEH_info_ecb

	.rva	.LSEH_begin_aesni_ccm64_encrypt_blocks
	.rva	.LSEH_end_aesni_ccm64_encrypt_blocks
	.rva	.LSEH_info_ccm64_enc

	.rva	.LSEH_begin_aesni_ccm64_decrypt_blocks
	.rva	.LSEH_end_aesni_ccm64_decrypt_blocks
	.rva	.LSEH_info_ccm64_dec

	.rva	.LSEH_begin_aesni_ctr32_encrypt_blocks
	.rva	.LSEH_end_aesni_ctr32_encrypt_blocks
	.rva	.LSEH_info_ctr32

	.rva	.LSEH_begin_aesni_xts_encrypt
	.rva	.LSEH_end_aesni_xts_encrypt
	.rva	.LSEH_info_xts_enc

	.rva	.LSEH_begin_aesni_xts_decrypt
	.rva	.LSEH_end_aesni_xts_decrypt
	.rva	.LSEH_info_xts_dec
___
$code.=<<___;
	.rva	.LSEH_begin_${PREFIX}_cbc_encrypt
	.rva	.LSEH_end_${PREFIX}_cbc_encrypt
	.rva	.LSEH_info_cbc

	.rva	${PREFIX}_set_decrypt_key
	.rva	.LSEH_end_set_decrypt_key
	.rva	.LSEH_info_key

	.rva	${PREFIX}_set_encrypt_key
	.rva	.LSEH_end_set_encrypt_key
	.rva	.LSEH_info_key
.section	.xdata
.align	8
___
$code.=<<___ if ($PREFIX eq "aesni");
.LSEH_info_ecb:
	.byte	9,0,0,0
	.rva	ecb_ccm64_se_handler
	.rva	.Lecb_enc_body,.Lecb_enc_ret		# HandlerData[]
.LSEH_info_ccm64_enc:
	.byte	9,0,0,0