Commit 5ac7bde7 authored by Andy Polyakov's avatar Andy Polyakov
Browse files

Throw in Montgomery multiplication assembler for x86_64.

parent 9b4eab50

Configure

+3 −1
Original line number Diff line number Diff line
@@ -118,7 +118,7 @@ my $x86_elf_asm="x86cpuid-elf.o:bn86-elf.o co86-elf.o:dx86-elf.o yx86-elf.o:ax86 
my $x86_coff_asm="x86cpuid-cof.o:bn86-cof.o co86-cof.o:dx86-cof.o yx86-cof.o:ax86-cof.o:bx86-cof.o:mx86-cof.o:sx86-cof.o s512sse2-cof.o:cx86-cof.o:rx86-cof.o:rm86-cof.o:r586-cof.o";

my $x86_out_asm="x86cpuid-out.o:bn86-out.o co86-out.o:dx86-out.o yx86-out.o:ax86-out.o:bx86-out.o:mx86-out.o:sx86-out.o s512sse2-out.o:cx86-out.o:rx86-out.o:rm86-out.o:r586-out.o";



my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o::aes-x86_64.o::md5-x86_64.o:sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o::";

my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o::aes-x86_64.o::md5-x86_64.o:sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o::";

my $ia64_asm="ia64cpuid.o:bn-ia64.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o::";



my $no_asm="::::::::::";
@@ -1171,6 +1171,8 @@ $bn_obj = $bn_asm unless $bn_obj ne ""; 
$cflags.=" -DOPENSSL_BN_ASM_PART_WORDS" if ($bn_obj =~ /bn86/);

$cflags.=" -DOPENSSL_IA32_SSE2" if (!$no_sse2 && $bn_obj =~ /bn86/);



$cflags.=" -DOPENSSL_BN_ASM_MONT" if ($bn_obj =~ /\-mont/);



$des_obj=$des_enc	unless ($des_obj =~ /\.o$/);

$bf_obj=$bf_enc		unless ($bf_obj =~ /\.o$/);

$cast_obj=$cast_enc	unless ($cast_obj =~ /\.o$/);

TABLE

+3 −3
Original line number Diff line number Diff line
@@ -252,7 +252,7 @@ $sys_id = 
$lflags       = 

$bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK DES_INT DES_UNROLL

$cpuid_obj    = x86_64cpuid.o

$bn_obj       = x86_64-gcc.o

$bn_obj       = x86_64-gcc.o x86_64-mont.o

$des_obj      = 

$aes_obj      = aes-x86_64.o

$bf_obj       =
@@ -2871,7 +2871,7 @@ $sys_id = 
$lflags       = -ldl

$bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK BF_PTR2 DES_INT DES_UNROLL

$cpuid_obj    = x86_64cpuid.o

$bn_obj       = x86_64-gcc.o

$bn_obj       = x86_64-gcc.o x86_64-mont.o

$des_obj      = 

$aes_obj      = aes-x86_64.o

$bf_obj       =
@@ -3681,7 +3681,7 @@ $sys_id = 
$lflags       = -lsocket -lnsl -ldl

$bn_ops       = SIXTY_FOUR_BIT_LONG RC4_CHUNK BF_PTR2 DES_INT DES_UNROLL

$cpuid_obj    = x86_64cpuid.o

$bn_obj       = x86_64-gcc.o

$bn_obj       = x86_64-gcc.o x86_64-mont.o

$des_obj      = 

$aes_obj      = aes-x86_64.o

$bf_obj       =

crypto/bn/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -91,6 +91,8 @@ bn-mips3.o: asm/mips3.s 


x86_64-gcc.o:	asm/x86_64-gcc.c

	$(CC) $(CFLAGS) -c -o $@ asm/x86_64-gcc.c

x86_64-mont.s:	asm/x86_64-mont.pl

	$(PERL) asm/x86_64-mont.pl $@



bn-ia64.s:	asm/ia64.S

	$(CC) $(CFLAGS) -E asm/ia64.S > $@

crypto/bn/asm/x86_64-mont.pl

0 → 100755
+207 −0
Original line number Diff line number Diff line 
#!/usr/bin/env perl



# ====================================================================

# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL

# project. Rights for redistribution and usage in source and binary

# forms are granted according to the OpenSSL license.

# ====================================================================



# October 2005.

#

# Montgomery multiplication routine for x86_64. While it gives modest

# 9% improvement of rsa4096 sign on Opteron, rsa512 sign runs more

# than twice, >2x, as fast. Most common rsa1024 sign is improved by

# respectful 50%. It remains to be seen if loop unrolling and

# dedicated squaring routine can provide further improvement...



$output=shift;

open STDOUT,"| $^X ../perlasm/x86_64-xlate.pl $output";



# int bn_mul_mont(

$rp="%rdi";	# BN_ULONG *rp,

$ap="%rsi";	# const BN_ULONG *ap,

$bp="%rdx";	# const BN_ULONG *bp,

$np="%rcx";	# const BN_ULONG *np,

$n0="%r8";	# BN_ULONG n0,

$num="%r9";	# int num);

$lo0="%r10";

$hi0="%r11";

$bp="%r12";	# reassign $bp

$hi1="%r13";

$i="%r14";

$j="%r15";

$m0="%rbx";

$m1="%rbp";



$code=<<___;

.text



.globl	bn_mul_mont

.type	bn_mul_mont,\@function,6

.align	16

bn_mul_mont:

	push	%rbx

	push	%rbp

	push	%r12

	push	%r13

	push	%r14

	push	%r15



	lea	2($num),%rax

	mov	%rsp,%rbp

	neg	%rax

	lea	(%rsp,%rax,8),%rsp	# tp=alloca(8*(num+2))

	and	\$-1024,%rsp		# minimize TLB usage

	mov	%rbp,8(%rsp,$num,8)	# tp[num+1]=%rsp

	mov	%rdx,$bp		# $bp reassigned, remember?



	xor	$i,$i			# i=0

	xor	$j,$j			# j=0



	mov	($bp),$m0		# m0=bp[0]

	mov	($ap),%rax

	mulq	$m0			# ap[0]*bp[0]

	mov	%rax,$lo0

	mov	%rdx,$hi0



	imulq	$n0,%rax		# "tp[0]"*n0

	mov	%rax,$m1



	mulq	($np)			# np[0]*m1

	add	$lo0,%rax		# discarded

	adc	\$0,%rdx

	mov	%rdx,$hi1



	lea	1($j),$j		# j++

.L1st:

	mov	($ap,$j,8),%rax

	mulq	$m0			# ap[j]*bp[0]

	add	$hi0,%rax

	adc	\$0,%rdx

	mov	%rax,$lo0

	mov	%rdx,$hi0



	mov	($np,$j,8),%rax

	mulq	$m1			# np[j]*m1

	add	$hi1,%rax

	adc	\$0,%rdx

	add	$lo0,%rax		# np[j]*m1+ap[j]*bp[0]

	adc	\$0,%rdx

	mov	%rax,-8(%rsp,$j,8)	# tp[j-1]

	mov	%rdx,$hi1



	lea	1($j),$j		# j++

	cmp	$num,$j

	jl	.L1st



	xor	%rdx,%rdx

	add	$hi0,$hi1

	adc	\$0,%rdx

	mov	$hi1,-8(%rsp,$j,8)

	mov	%rdx,(%rsp,$j,8)



	lea	1($i),$i		# i++

.align	4

.Louter:

	xor	$j,$j			# j=0



	mov	($bp,$i,8),$m0		# m0=bp[i]

	mov	($ap),%rax		# ap[0]

	mulq	$m0			# ap[0]*bp[i]

	add	(%rsp),%rax		# ap[0]*bp[i]+tp[0]

	adc	\$0,%rdx

	mov	%rax,$lo0

	mov	%rdx,$hi0



	imulq	$n0,%rax		# tp[0]*n0

	mov	%rax,$m1



	mulq	($np,$j,8)		# np[0]*m1

	add	$lo0,%rax		# discarded

	adc	\$0,%rdx

	mov	%rdx,$hi1



	lea	1($j),$j		# j++

.align	4

.Linner:

	mov	($ap,$j,8),%rax

	mulq	$m0			# ap[j]*bp[i]

	add	$hi0,%rax

	adc	\$0,%rdx

	add	(%rsp,$j,8),%rax	# ap[j]*bp[i]+tp[j]

	adc	\$0,%rdx

	mov	%rax,$lo0

	mov	%rdx,$hi0



	mov	($np,$j,8),%rax

	mulq	$m1			# np[j]*m1

	add	$hi1,%rax

	adc	\$0,%rdx

	add	$lo0,%rax		# np[j]*m1+ap[j]*bp[i]+tp[j]

	adc	\$0,%rdx

	mov	%rax,-8(%rsp,$j,8)	# tp[j-1]

	mov	%rdx,$hi1



	lea	1($j),$j		# j++

	cmp	$num,$j

	jl	.Linner



	xor	%rdx,%rdx		# $j equals to num here...

	add	$hi0,$hi1

	adc	\$0,%rdx

	add	(%rsp,$j,8),$hi1	# pull upmost overflow bit

	adc	\$0,%rdx

	mov	$hi1,-8(%rsp,$j,8)

	mov	%rdx,(%rsp,$j,8)	# store upmost overflow bit



	lea	1($i),$i		# i++

	cmp	$num,$i

	jl	.Louter



	sub	$i,$i			# clear CF at once

	cmp	\$0,%rdx		# %rdx still holds upmost overflow bit

	jnz	.Lsub			# ... and $j still equals to num

	mov	-8(%rsp,$num,8),%rax

	cmp	-8($np,$num,8),%rax	# tp[num-1]-np[num-1]

	jae	.Lsub



	lea	-1($num),$j		# j=num-1

.align	4

.Lcopy:

	mov	(%rsp,$j,8),%rax

	mov	%rax,($rp,$j,8)		# rp[i]=tp[i]

	mov	$i,(%rsp,$j,8)		# zap temporary vector

	dec	$j

	jge	.Lcopy

.align	4

.Lexit:

	mov	8(%rsp,$num,8),%rsp	# restore %rsp

	mov	\$1,%rax

	pop	%r15

	pop	%r14

	pop	%r13

	pop	%r12

	pop	%rbp

	pop	%rbx

	ret



.align	16

.Lsub:	mov	(%rsp,$i,8),%rax

	sbb	($np,$i,8),%rax

	mov	%rax,($rp,$i,8)		# rp[i]=tp[i]-np[j]

	lea	1($i),$i		# i++

	dec	$j			# doesn't affect cf!

	jg	.Lsub

	lea	-1($num),$j		# j=num-1

	sbb	\$0,%rdx

	jc	.Lcopy			# tp was less than np

.align	4

.Lzap:	mov	$i,(%rsp,$j,8)		# zap temporary vector

	dec	$j

	jge	.Lzap

	jmp	.Lexit

.size	bn_mul_mont,.-bn_mul_mont

___



print $code;

close STDOUT;