Commit 874a3757 authored by Andy Polyakov's avatar Andy Polyakov
Browse files

Rework framework for assembler support for AES counter mode and add 

AES_ctr32_encrypt to aes-s390x.pl.
parent 1cbdca7b

Configure

+16 −13
Original line number Diff line number Diff line
@@ -123,22 +123,22 @@ my $tlib="-lnsl -lsocket"; 
my $bits1="THIRTY_TWO_BIT ";

my $bits2="SIXTY_FOUR_BIT ";



my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o:des-586.o crypt586.o:aes_ctr.o aes-586.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o";

my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o:des-586.o crypt586.o:aes-586.o aesni-x86.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o:cmll-x86.o:ghash-x86.o";



my $x86_elf_asm="$x86_asm:elf";



my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o::aes_ctr.o aes-x86_64.o aesni-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o";

my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o:void";

my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes_ctr.o aes-sparcv9.o:::sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o:::::::ghash-sparcv9.o:void";

my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o::aes-x86_64.o aesni-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o";

my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o:void";

my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes-sparcv9.o:::sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o:::::::ghash-sparcv9.o:void";

my $sparcv8_asm=":sparcv8.o:des_enc-sparc.o fcrypt_b.o::::::::::::void";

my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o:::::sha1-alpha.o:::::::ghash-alpha.o:void";

my $mips3_asm=":bn-mips3.o:::::::::::::void";

my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o::aes_ctr.o aes-s390x.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o::::::void";

my $armv4_asm=":bn_asm.o armv4-mont.o::aes_cbc.o aes_ctr.o aes-armv4.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o:void";

my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:32";

my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:64";

my $ppc32_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-ppc.o:::sha1-ppc.o sha256-ppc.o:::::::";

my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o:::::::";

my $armv4_asm=":bn_asm.o armv4-mont.o::aes_cbc.o aes-armv4.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o:void";

my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:32";

my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o:64";

my $ppc32_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o:::sha1-ppc.o sha256-ppc.o:::::::";

my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o:::::::";

my $no_asm="::::::::::::::void";



# As for $BSDthreads. Idea is to maintain "collective" set of flags,
@@ -187,7 +187,7 @@ my %table=( 
"debug-linux-ppro","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DBN_CTX_DEBUG -DCRYPTO_MDEBUG -DL_ENDIAN -DTERMIO -g -mcpu=pentiumpro -Wall::-D_REENTRANT::-ldl:BN_LLONG ${x86_gcc_des} ${x86_gcc_opts}:${x86_elf_asm}:dlfcn",

"debug-linux-elf","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DBN_CTX_DEBUG -DCRYPTO_MDEBUG -DL_ENDIAN -DTERMIO -g -march=i486 -Wall::-D_REENTRANT::-lefence -ldl:BN_LLONG ${x86_gcc_des} ${x86_gcc_opts}:${x86_elf_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-elf-noefence","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DBN_CTX_DEBUG -DCRYPTO_MDEBUG -DL_ENDIAN -DTERMIO -g -march=i486 -Wall::-D_REENTRANT::-ldl:BN_LLONG ${x86_gcc_des} ${x86_gcc_opts}:${x86_elf_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-ia32-aes", "gcc:-DAES_EXPERIMENTAL -DL_ENDIAN -DTERMIO -O3 -fomit-frame-pointer -Wall::-D_REENTRANT::-ldl:BN_LLONG ${x86_gcc_des} ${x86_gcc_opts}:x86cpuid.o:bn-586.o co-586.o x86-mont.o:des-586.o crypt586.o:aes_x86core.o aes_cbc.o aes_ctr.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o::ghash-x86.o:elf:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-ia32-aes", "gcc:-DAES_EXPERIMENTAL -DL_ENDIAN -DTERMIO -O3 -fomit-frame-pointer -Wall::-D_REENTRANT::-ldl:BN_LLONG ${x86_gcc_des} ${x86_gcc_opts}:x86cpuid.o:bn-586.o co-586.o x86-mont.o:des-586.o crypt586.o:aes_x86core.o aes_cbc.o:bf-586.o:md5-586.o:sha1-586.o sha256-586.o sha512-586.o:cast-586.o:rc4-586.o:rmd-586.o:rc5-586.o:wp_block.o wp-mmx.o::ghash-x86.o:elf:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-generic32","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -DTERMIO -g -Wall::-D_REENTRANT::-ldl:BN_LLONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR:${no_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-generic64","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -DTERMIO -g -Wall::-D_REENTRANT::-ldl:SIXTY_FOUR_BIT_LONG RC4_CHAR RC4_CHUNK DES_INT DES_UNROLL BF_PTR:${no_asm}:dlfcn:linux-shared:-fPIC::.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR)",

"debug-linux-x86_64","gcc:-DBN_DEBUG -DREF_CHECK -DCONF_DEBUG -DCRYPTO_MDEBUG -m64 -DL_ENDIAN -DTERMIO -g -Wall -DMD32_REG_T=int::-D_REENTRANT::-ldl:SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL:${x86_64_asm}:elf:dlfcn:linux-shared:-fPIC:-m64:.so.\$(SHLIB_MAJOR).\$(SHLIB_MINOR):::64",
@@ -493,8 +493,8 @@ my %table=( 
# Visual C targets

#

# Win64 targets, WIN64I denotes IA-64 and WIN64A - AMD64

"VC-WIN64I","cl:-W3 -Gs0 -Gy -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -DUNICODE -D_UNICODE -D_CRT_SECURE_NO_DEPRECATE:::WIN64I::SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN:ia64cpuid.o:ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes_ctr.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o:::::::ghash-ia64.o:ias:win32",

"VC-WIN64A","cl:-W3 -Gs0 -Gy -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -DUNICODE -D_UNICODE -D_CRT_SECURE_NO_DEPRECATE:::WIN64A::SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN:x86_64cpuid.o:bn_asm.o x86_64-mont.o::aes_ctr.o aes-x86_64.o aesni-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:auto:win32",

"VC-WIN64I","cl:-W3 -Gs0 -Gy -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -DUNICODE -D_UNICODE -D_CRT_SECURE_NO_DEPRECATE:::WIN64I::SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN:ia64cpuid.o:ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o:::::::ghash-ia64.o:ias:win32",

"VC-WIN64A","cl:-W3 -Gs0 -Gy -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -DUNICODE -D_UNICODE -D_CRT_SECURE_NO_DEPRECATE:::WIN64A::SIXTY_FOUR_BIT RC4_CHUNK_LL DES_INT EXPORT_VAR_AS_FN:x86_64cpuid.o:bn_asm.o x86_64-mont.o::aes-x86_64.o aesni-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:auto:win32",

# x86 Win32 target defaults to ANSI API, if you want UNICODE, complement

# 'perl Configure VC-WIN32' with '-DUNICODE -D_UNICODE'

"VC-WIN32","cl:-W3 -WX -Gs0 -GF -Gy -nologo -DOPENSSL_SYSNAME_WIN32 -DWIN32_LEAN_AND_MEAN -DL_ENDIAN -D_CRT_SECURE_NO_DEPRECATE:::WIN32::BN_LLONG RC4_INDEX EXPORT_VAR_AS_FN ${x86_gcc_opts}:${x86_asm}:win32n:win32",
@@ -651,7 +651,7 @@ my $rc2 ="crypto/rc2/rc2.h"; 
my $bf	="crypto/bf/bf_locl.h";

my $bn_asm	="bn_asm.o";

my $des_enc="des_enc.o fcrypt_b.o";

my $aes_enc="aes_core.o aes_cbc.o aes_ctr.o";

my $aes_enc="aes_core.o aes_cbc.o";

my $bf_enc	="bf_enc.o";

my $cast_enc="c_enc.o";

my $rc4_enc="rc4_enc.o rc4_skey.o";
@@ -1400,6 +1400,9 @@ if ($rmd160_obj =~ /\.o$/) 
if ($aes_obj =~ /\.o$/)

	{

	$cflags.=" -DAES_ASM";

	# aes_ctr.o is not a real file, only indication that assembler

	# module implements AES_ctr32_encrypt...

	$cflags.=" -DAES_CTR_ASM" if ($aes_obj =~ s/\s*aes_ctr.o//);

	$aes_obj =~ s/\s*aesni\-x86\.o// if ($no_sse2);

	}

else	{

TABLE

+75 −75

File changed.

Preview size limit exceeded, changes collapsed.

crypto/aes/Makefile

+2 −4
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ CFLAG=-g 
MAKEFILE=	Makefile

AR=		ar r



AES_ENC=aes_core.o aes_cbc.o aes_ctr.o

AES_ENC=aes_core.o aes_cbc.o



CFLAGS= $(INCLUDES) $(CFLAG)

ASFLAGS= $(INCLUDES) $(ASFLAG)
@@ -24,7 +24,7 @@ APPS= 


LIB=$(TOP)/libcrypto.a

LIBSRC=aes_core.c aes_misc.c aes_ecb.c aes_cbc.c aes_cfb.c aes_ofb.c \

       aes_ctr.c aes_ige.c aes_wrap.c

       aes_ige.c aes_wrap.c

LIBOBJ=aes_misc.o aes_ecb.o aes_cfb.o aes_ofb.o aes_ige.o aes_wrap.o \

       $(AES_ENC)
@@ -113,8 +113,6 @@ aes_cfb.o: ../../include/openssl/aes.h ../../include/openssl/modes.h 
aes_cfb.o: ../../include/openssl/opensslconf.h aes_cfb.c

aes_core.o: ../../include/openssl/aes.h ../../include/openssl/e_os2.h

aes_core.o: ../../include/openssl/opensslconf.h aes_core.c aes_locl.h

aes_ctr.o: ../../include/openssl/aes.h ../../include/openssl/modes.h

aes_ctr.o: ../../include/openssl/opensslconf.h aes_ctr.c

aes_ecb.o: ../../include/openssl/aes.h ../../include/openssl/e_os2.h

aes_ecb.o: ../../include/openssl/opensslconf.h aes_ecb.c aes_locl.h

aes_ige.o: ../../e_os.h ../../include/openssl/aes.h ../../include/openssl/bio.h

crypto/aes/aes_ctr.c

deleted100644 → 0
+0 −61
Original line number Diff line number Diff line 
/* crypto/aes/aes_ctr.c -*- mode:C; c-file-style: "eay" -*- */

/* ====================================================================

 * Copyright (c) 1998-2002 The OpenSSL Project.  All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 *

 * 1. Redistributions of source code must retain the above copyright

 *    notice, this list of conditions and the following disclaimer. 

 *

 * 2. Redistributions in binary form must reproduce the above copyright

 *    notice, this list of conditions and the following disclaimer in

 *    the documentation and/or other materials provided with the

 *    distribution.

 *

 * 3. All advertising materials mentioning features or use of this

 *    software must display the following acknowledgment:

 *    "This product includes software developed by the OpenSSL Project

 *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"

 *

 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to

 *    endorse or promote products derived from this software without

 *    prior written permission. For written permission, please contact

 *    openssl-core@openssl.org.

 *

 * 5. Products derived from this software may not be called "OpenSSL"

 *    nor may "OpenSSL" appear in their names without prior written

 *    permission of the OpenSSL Project.

 *

 * 6. Redistributions of any form whatsoever must retain the following

 *    acknowledgment:

 *    "This product includes software developed by the OpenSSL Project

 *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"

 *

 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY

 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR

 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT

 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,

 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED

 * OF THE POSSIBILITY OF SUCH DAMAGE.

 * ====================================================================

 *

 */



#include <openssl/aes.h>

#include <openssl/modes.h>



void AES_ctr128_encrypt(const unsigned char *in, unsigned char *out,

			size_t length, const AES_KEY *key,

			unsigned char ivec[AES_BLOCK_SIZE],

			unsigned char ecount_buf[AES_BLOCK_SIZE],

			unsigned int *num) {

	CRYPTO_ctr128_encrypt(in,out,length,key,ivec,ecount_buf,num,(block128_f)AES_encrypt);

}

crypto/aes/asm/aes-s390x.pl

+143 −1
Original line number Diff line number Diff line
@@ -50,6 +50,10 @@ 
# it was measured to be ~6.6x. It's less than previously mentioned 8x,

# because software implementation was optimized.



# May 2010.

#

# Add AES_ctr32_encrypt.



while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}

open STDOUT,">$output";
@@ -1331,10 +1335,148 @@ $code.=<<___; 
4:	ex	$len,0($s1)

	j	.Lcbc_dec_exit

.size	AES_cbc_encrypt,.-AES_cbc_encrypt

.comm  OPENSSL_s390xcap_P,8,8

___

}

#void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,

#                     size_t blocks, const AES_KEY *key,

#                     const unsigned char *ivec)

{

my $inp="%r2";

my $out="%r3";

my $len="%r4";

my $key="%r5";	my $iv0="%r5";

my $ivp="%r6";

my $fp ="%r7";



$code.=<<___;

.globl	AES_ctr32_encrypt

.type	AES_ctr32_encrypt,\@function

.align	16

AES_ctr32_encrypt:

___

$code.=<<___ if (!$softonly);

	l	%r0,240($key)

	lhi	%r1,16

	clr	%r0,%r1

	jl	.Lctr32_software



	stmg	%r6,$s3,48($sp)



	slgr	$out,$inp

	la	%r1,0($key)	# %r1 is permanent copy of $key

	lg	$iv0,0($ivp)	# load ivec

	lg	$ivp,8($ivp)



	# prepare and allocate stack frame

	lghi	$s0,-272	# guarantee at least 256-bytes buffer

	lghi	$s1,-4096

	lgr	$fp,$sp

	algr	$s0,$sp

	ngr	$s0,$s1		# align at page boundary

	la	$sp,0($s0)	# alloca

	stg	$fp,0($s0)	# back-chain



	# calculate resultant buffer size

	la	$s0,16($s0)	# buffer starts at offset of 16

	slgr	$fp,$s0

	srlg	$fp,$fp,4	# $fp is buffer length in blocks, minimum 16

	stg	$fp,8($sp)



	slgr	$len,$fp

	brc	1,.Lctr32_hw_loop	# not zero, no borrow

	algr	$fp,$len

	lghi	$len,0

	stg	$fp,8($sp)



.Lctr32_hw_loop:

	la	$s2,16($sp)

	lgr	$s3,$fp

.Lctr32_hw_prepare:

	stg	$iv0,0($s2)

	stg	$ivp,8($s2)

	la	$s2,16($s2)

	ahi	$ivp,1		# 32-bit increment, preserves upper half

	brct	$s3,.Lctr32_hw_prepare



	la	$s0,16($sp)	# inp

	sllg	$s1,$fp,4	# len

	la	$s2,16($sp)	# out

	.long	0xb92e00a8	# km %r10,%r8

	brc	1,.-4		# pay attention to "partial completion"



	la	$s2,16($sp)

	lgr	$s3,$fp

	slgr	$s2,$inp

.Lctr32_hw_xor:

	lg	$s0,0($inp)

	lg	$s1,8($inp)

	xg	$s0,0($s2,$inp)

	xg	$s1,8($s2,$inp)

	stg	$s0,0($out,$inp)

	stg	$s1,8($out,$inp)

	la	$inp,16($inp)

	brct	$s3,.Lctr32_hw_xor



	slgr	$len,$fp

	brc	1,.Lctr32_hw_loop	# not zero, no borrow

	algr	$fp,$len

	lghi	$len,0

	brc	4+1,.Lctr32_hw_loop	# not zero



	lg	$s0,0($sp)

	lg	$s1,8($sp)

	la	$s2,16($sp)

.Lctr32_hw_zap:

	stg	$s0,0($s2)

	stg	$s0,8($s2)

	la	$s2,16($s2)

	brct	$s1,.Lctr32_hw_zap



	la	$sp,0($s0)

	lmg	%r6,$s3,48($sp)

	br	$ra

.align	16

.Lctr32_software:

___

$code.=<<___;

	stmg	$key,$ra,40($sp)

	slgr	$out,$inp

	larl	$tbl,AES_Te

	llgf	$t1,12($ivp)



.Lctr32_loop:

	stmg	$inp,$len,16($sp)

	llgf	$s0,0($ivp)

	llgf	$s1,4($ivp)

	llgf	$s2,8($ivp)

	lgr	$s3,$t1

	st	$t1,128($sp)

	lgr	%r4,$key



	bras	$ra,_s390x_AES_encrypt



	lmg	$inp,$ivp,16($sp)

	llgf	$t1,128($sp)

	x	$s0,0($inp)

	x	$s1,4($inp)

	x	$s2,8($inp)

	x	$s3,12($inp)

	st	$s0,0($out,$inp)

	st	$s1,4($out,$inp)

	st	$s2,8($out,$inp)

	st	$s3,12($out,$inp)



	la	$inp,16($inp)

	ahi	$t1,1		# 32-bit increment

	brct	$len,.Lctr32_loop



	lmg	%r6,$ra,48($sp)

	br	$ra

.size	AES_ctr32_encrypt,.-AES_ctr32_encrypt

___

}

$code.=<<___;

.comm  OPENSSL_s390xcap_P,8,8

.string	"AES for s390x, CRYPTOGAMS by <appro\@openssl.org>"

___