Newer
Older
#!/usr/bin/env perl
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# project. Rights for redistribution and usage in source and binary
# forms are granted according to the OpenSSL license.
# ====================================================================
#
#
# You might fail to appreciate this module performance from the first
# try. If compared to "vanilla" linux-ia32-icc target, i.e. considered
# to be *the* best Intel C compiler without -KPIC, performance appears
# to be virtually identical... But try to re-configure with shared
# library support... Aha! Intel compiler "suddenly" lags behind by 30%
# [on P4, more on others]:-) And if compared to position-independent
# code generated by GNU C, this code performs *more* than *twice* as
# fast! Yes, all this buzz about PIC means that unlike other hand-
# coded implementations, this one was explicitly designed to be safe
# to use even in shared library context... This also means that this
# code isn't necessarily absolutely fastest "ever," because in order
# to achieve position independence an extra register has to be
# off-loaded to stack, which affects the benchmark result.
#
# Special note about instruction choice. Do you recall RC4_INT code
# performing poorly on P4? It might be the time to figure out why.
# RC4_INT code implies effective address calculations in base+offset*4
# form. Trouble is that it seems that offset scaling turned to be
# critical path... At least eliminating scaling resulted in 2.8x RC4
# performance improvement [as you might recall]. As AES code is hungry
# for scaling too, I [try to] avoid the latter by favoring off-by-2
# shifts and masking the result with 0xFF<<2 instead of "boring" 0xFF.
#
# As was shown by Dean Gaudet <dean@arctic.org>, the above note turned
# void. Performance improvement with off-by-2 shifts was observed on
# intermediate implementation, which was spilling yet another register
# to stack... Final offset*4 code below runs just a tad faster on P4,
# but exhibits up to 10% improvement on other cores.
#
# Second version is "monolithic" replacement for aes_core.c, which in
# addition to AES_[de|en]crypt implements AES_set_[de|en]cryption_key.
# This made it possible to implement little-endian variant of the
# algorithm without modifying the base C code. Motivating factor for
# the undertaken effort was that it appeared that in tight IA-32
# register window little-endian flavor could achieve slightly higher
# Instruction Level Parallelism, and it indeed resulted in up to 15%
# better performance on most recent µ-archs...
#
# Third version adds AES_cbc_encrypt implementation, which resulted in
# up to 40% performance imrovement of CBC benchmark results. 40% was
# observed on P4 core, where "overall" imrovement coefficient, i.e. if
# compared to PIC generated by GCC and in CBC mode, was observed to be
# as large as 4x:-) CBC performance is virtually identical to ECB now
# and on some platforms even better, e.g. 17.6 "small" cycles/byte on
# Opteron, because certain function prologues and epilogues are
# effectively taken out of the loop...
#
# Version 3.2 implements compressed tables and prefetch of these tables
# in CBC[!] mode. Former means that 3/4 of table references are now
# misaligned, which unfortunately has negative impact on elder IA-32
# implementations, Pentium suffered 30% penalty, PIII - 10%.
#
# Version 3.3 avoids L1 cache aliasing between stack frame and
# S-boxes, and 3.4 - L1 cache aliasing even between key schedule. The
# latter is achieved by copying the key schedule to controlled place in
# stack. This unfortunately has rather strong impact on small block CBC
# performance, ~2x deterioration on 16-byte block if compared to 3.3.
#
# Current ECB performance numbers for 128-bit key in CPU cycles per
# processed byte [measure commonly used by AES benchmarkers] are:
#
# small footprint fully unrolled
# P4 24 22
# AMD K8 20 19
# PIII 25 23
# Pentium 81 78
push(@INC,"perlasm","../../perlasm");
require "x86asm.pl";
&asm_init($ARGV[0],"aes-586.pl",$ARGV[$#ARGV] eq "386");
$s0="eax";
$s1="ebx";
$s2="ecx";
$s3="edx";
$key="edi";
$acc="esi";
$compromise=0; # $compromise=128 abstains from copying key
# schedule to stack when encrypting inputs
# shorter than 128 bytes at the cost of
# risksing aliasing with S-boxes. In return
# you get way better, up to +70%, small block
# performance.
$small_footprint=1; # $small_footprint=1 code is ~5% slower [on
# recent µ-archs], but ~5 times smaller!
# I favor compact code to minimize cache
# contention and in hope to "collect" 5% back
# in real-life applications...
$vertical_spin=0; # shift "verticaly" defaults to 0, because of
# its proof-of-concept status...
# Note that there is no decvert(), as well as last encryption round is
# performed with "horizontal" shifts. This is because this "vertical"
# implementation [one which groups shifts on a given $s[i] to form a
# "column," unlike "horizontal" one, which groups shifts on different
# $s[i] to form a "row"] is work in progress. It was observed to run
# few percents faster on Intel cores, but not AMD. On AMD K8 core it's
# whole 12% slower:-( So we face a trade-off... Shall it be resolved
# some day? Till then the code is considered experimental and by
# default remains dormant...
sub encvert()
{ my ($te,@s) = @_;
my $v0 = $acc, $v1 = $key;
&mov ($v0,$s[3]); # copy s3
&mov (&DWP(4,"esp"),$s[2]); # save s2
&mov ($v1,$s[0]); # copy s0
&mov (&DWP(8,"esp"),$s[1]); # save s1
&movz ($s[2],&HB($s[0]));
&and ($s[0],0xFF);
&mov ($s[0],&DWP(0,$te,$s[0],8)); # s0>>0
&shr ($v1,16);
&mov ($s[3],&DWP(3,$te,$s[2],8)); # s0>>8
&movz ($s[1],&HB($v1));
&and ($v1,0xFF);
&mov ($s[2],&DWP(2,$te,$v1,8)); # s0>>16
&mov ($v1,$v0);
&mov ($s[1],&DWP(1,$te,$s[1],8)); # s0>>24
&and ($v0,0xFF);
&xor ($s[3],&DWP(0,$te,$v0,8)); # s3>>0
&movz ($v0,&HB($v1));
&shr ($v1,16);
&xor ($s[2],&DWP(3,$te,$v0,8)); # s3>>8
&movz ($v0,&HB($v1));
&and ($v1,0xFF);
&xor ($s[1],&DWP(2,$te,$v1,8)); # s3>>16
&mov ($v1,&DWP(4,"esp")); # restore s2
&xor ($s[0],&DWP(1,$te,$v0,8)); # s3>>24
&mov ($v0,$v1);
&and ($v1,0xFF);
&xor ($s[2],&DWP(0,$te,$v1,8)); # s2>>0
&movz ($v1,&HB($v0));
&shr ($v0,16);
&xor ($s[1],&DWP(3,$te,$v1,8)); # s2>>8
&movz ($v1,&HB($v0));
&and ($v0,0xFF);
&xor ($s[0],&DWP(2,$te,$v0,8)); # s2>>16
&mov ($v0,&DWP(8,"esp")); # restore s1
&xor ($s[3],&DWP(1,$te,$v1,8)); # s2>>24
&mov ($v1,$v0);
&and ($v0,0xFF);
&xor ($s[1],&DWP(0,$te,$v0,8)); # s1>>0
&movz ($v0,&HB($v1));
&shr ($v1,16);
&xor ($s[0],&DWP(3,$te,$v0,8)); # s1>>8
&movz ($v0,&HB($v1));
&and ($v1,0xFF);
&xor ($s[3],&DWP(2,$te,$v1,8)); # s1>>16
&mov ($key,&DWP(12,"esp")); # reincarnate v1 as key
&xor ($s[2],&DWP(1,$te,$v0,8)); # s1>>24
}
sub encstep()
{ my ($i,$te,@s) = @_;
my $tmp = $key;
my $out = $i==3?$s[0]:$acc;
# lines marked with #%e?x[i] denote "reordered" instructions...
if ($i==3) { &mov ($key,&DWP(12,"esp")); }##%edx
else { &mov ($out,$s[0]);
&and ($out,0xFF); }
if ($i==1) { &shr ($s[0],16); }#%ebx[1]
if ($i==2) { &shr ($s[0],24); }#%ecx[2]
&mov ($out,&DWP(0,$te,$out,8));
if ($i==3) { $tmp=$s[1]; }##%eax
&movz ($tmp,&HB($s[1]));
&xor ($out,&DWP(3,$te,$tmp,8));
if ($i==3) { $tmp=$s[2]; &mov ($s[1],&DWP(4,"esp")); }##%ebx
else { &mov ($tmp,$s[2]);
&shr ($tmp,16); }
if ($i==2) { &and ($s[1],0xFF); }#%edx[2]
&and ($tmp,0xFF);
&xor ($out,&DWP(2,$te,$tmp,8));
if ($i==3) { $tmp=$s[3]; &mov ($s[2],&DWP(8,"esp")); }##%ecx
elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
else { &mov ($tmp,$s[3]);
&shr ($tmp,24) }
&xor ($out,&DWP(1,$te,$tmp,8));
if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
if ($i==3) { &mov ($s[3],$acc); }
&comment();
}
sub enclast()
{ my ($i,$te,@s)=@_;
my $tmp = $key;
my $out = $i==3?$s[0]:$acc;
if ($i==3) { &mov ($key,&DWP(12,"esp")); }##%edx
else { &mov ($out,$s[0]); }
&and ($out,0xFF);
if ($i==1) { &shr ($s[0],16); }#%ebx[1]
if ($i==2) { &shr ($s[0],24); }#%ecx[2]
&mov ($out,&DWP(2,$te,$out,8));
&and ($out,0x000000ff);
if ($i==3) { $tmp=$s[1]; }##%eax
&movz ($tmp,&HB($s[1]));
&mov ($tmp,&DWP(0,$te,$tmp,8));
&and ($tmp,0x0000ff00);
if ($i==3) { $tmp=$s[2]; &mov ($s[1],&DWP(4,"esp")); }##%ebx
else { mov ($tmp,$s[2]);
&shr ($tmp,16); }
if ($i==2) { &and ($s[1],0xFF); }#%edx[2]
&and ($tmp,0xFF);
&mov ($tmp,&DWP(0,$te,$tmp,8));
&and ($tmp,0x00ff0000);
if ($i==3) { $tmp=$s[3]; &mov ($s[2],&DWP(8,"esp")); }##%ecx
elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
else { &mov ($tmp,$s[3]);
&shr ($tmp,24); }
&mov ($tmp,&DWP(2,$te,$tmp,8));
&and ($tmp,0xff000000);
if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
if ($i==3) { &mov ($s[3],$acc); }
sub _data_word() { my $i; while(defined($i=shift)) { &data_word($i,$i); } }
&function_begin_B("_x86_AES_encrypt");
if ($vertical_spin) {
# I need high parts of volatile registers to be accessible...
&exch ($s1="edi",$key="ebx");
&mov ($s2="esi",$acc="ecx");
}
# note that caller is expected to allocate stack frame for me!
&mov (&DWP(12,"esp"),$key); # save key
&xor ($s0,&DWP(0,$key)); # xor with key
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&mov ($acc,&DWP(240,$key)); # load key->rounds
&lea ($acc,&DWP(-2,$acc,$acc));
&lea ($acc,&DWP(0,$key,$acc,8));
&mov (&DWP(16,"esp"),$acc); # end of key schedule
if ($vertical_spin) {
&encvert("ebp",$s0,$s1,$s2,$s3);
} else {
&encstep(0,"ebp",$s0,$s1,$s2,$s3);
&encstep(1,"ebp",$s1,$s2,$s3,$s0);
&encstep(2,"ebp",$s2,$s3,$s0,$s1);
&encstep(3,"ebp",$s3,$s0,$s1,$s2);
}
&add ($key,16); # advance rd_key
&xor ($s0,&DWP(0,$key));
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&cmp ($key,&DWP(16,"esp"));
&mov (&DWP(12,"esp"),$key);
&jb (&label("loop"));
}
else {
&cmp ($acc,10);
&cmp ($acc,12);
&jle (&label("12rounds"));
&set_label("14rounds");
for ($i=1;$i<3;$i++) {
if ($vertical_spin) {
&encvert("ebp",$s0,$s1,$s2,$s3);
} else {
&encstep(0,"ebp",$s0,$s1,$s2,$s3);
&encstep(1,"ebp",$s1,$s2,$s3,$s0);
&encstep(2,"ebp",$s2,$s3,$s0,$s1);
&encstep(3,"ebp",$s3,$s0,$s1,$s2);
}
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
&add ($key,32);
&mov (&DWP(12,"esp"),$key); # advance rd_key
&set_label("12rounds");
for ($i=1;$i<3;$i++) {
if ($vertical_spin) {
&encvert("ebp",$s0,$s1,$s2,$s3);
} else {
&encstep(0,"ebp",$s0,$s1,$s2,$s3);
&encstep(1,"ebp",$s1,$s2,$s3,$s0);
&encstep(2,"ebp",$s2,$s3,$s0,$s1);
&encstep(3,"ebp",$s3,$s0,$s1,$s2);
}
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
&add ($key,32);
&mov (&DWP(12,"esp"),$key); # advance rd_key
&set_label("10rounds");
for ($i=1;$i<10;$i++) {
if ($vertical_spin) {
&encvert("ebp",$s0,$s1,$s2,$s3);
} else {
&encstep(0,"ebp",$s0,$s1,$s2,$s3);
&encstep(1,"ebp",$s1,$s2,$s3,$s0);
&encstep(2,"ebp",$s2,$s3,$s0,$s1);
&encstep(3,"ebp",$s3,$s0,$s1,$s2);
}
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
if ($vertical_spin) {
# "reincarnate" some registers for "horizontal" spin...
&mov ($s1="ebx",$key="edi");
&mov ($s2="ecx",$acc="esi");
}
&enclast(0,"ebp",$s0,$s1,$s2,$s3);
&enclast(1,"ebp",$s1,$s2,$s3,$s0);
&enclast(2,"ebp",$s2,$s3,$s0,$s1);
&enclast(3,"ebp",$s3,$s0,$s1,$s2);
&add ($key,$small_footprint?16:160);
&xor ($s0,&DWP(0,$key));
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&ret ();
&set_label("AES_Te",64); # Yes! I keep it in the code segment!
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
&_data_word(0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6);
&_data_word(0x0df2f2ff, 0xbd6b6bd6, 0xb16f6fde, 0x54c5c591);
&_data_word(0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56);
&_data_word(0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec);
&_data_word(0x45caca8f, 0x9d82821f, 0x40c9c989, 0x877d7dfa);
&_data_word(0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb);
&_data_word(0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45);
&_data_word(0xbf9c9c23, 0xf7a4a453, 0x967272e4, 0x5bc0c09b);
&_data_word(0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c);
&_data_word(0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83);
&_data_word(0x5c343468, 0xf4a5a551, 0x34e5e5d1, 0x08f1f1f9);
&_data_word(0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a);
&_data_word(0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d);
&_data_word(0x28181830, 0xa1969637, 0x0f05050a, 0xb59a9a2f);
&_data_word(0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df);
&_data_word(0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea);
&_data_word(0x1b090912, 0x9e83831d, 0x742c2c58, 0x2e1a1a34);
&_data_word(0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b);
&_data_word(0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d);
&_data_word(0x7b292952, 0x3ee3e3dd, 0x712f2f5e, 0x97848413);
&_data_word(0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1);
&_data_word(0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6);
&_data_word(0xbe6a6ad4, 0x46cbcb8d, 0xd9bebe67, 0x4b393972);
&_data_word(0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85);
&_data_word(0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed);
&_data_word(0xc5434386, 0xd74d4d9a, 0x55333366, 0x94858511);
&_data_word(0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe);
&_data_word(0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b);
&_data_word(0xf35151a2, 0xfea3a35d, 0xc0404080, 0x8a8f8f05);
&_data_word(0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1);
&_data_word(0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142);
&_data_word(0x30101020, 0x1affffe5, 0x0ef3f3fd, 0x6dd2d2bf);
&_data_word(0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3);
&_data_word(0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e);
&_data_word(0x57c4c493, 0xf2a7a755, 0x827e7efc, 0x473d3d7a);
&_data_word(0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6);
&_data_word(0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3);
&_data_word(0x66222244, 0x7e2a2a54, 0xab90903b, 0x8388880b);
&_data_word(0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428);
&_data_word(0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad);
&_data_word(0x3be0e0db, 0x56323264, 0x4e3a3a74, 0x1e0a0a14);
&_data_word(0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8);
&_data_word(0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4);
&_data_word(0xa8919139, 0xa4959531, 0x37e4e4d3, 0x8b7979f2);
&_data_word(0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda);
&_data_word(0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949);
&_data_word(0xb46c6cd8, 0xfa5656ac, 0x07f4f4f3, 0x25eaeacf);
&_data_word(0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810);
&_data_word(0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c);
&_data_word(0x241c1c38, 0xf1a6a657, 0xc7b4b473, 0x51c6c697);
&_data_word(0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e);
&_data_word(0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f);
&_data_word(0x907070e0, 0x423e3e7c, 0xc4b5b571, 0xaa6666cc);
&_data_word(0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c);
&_data_word(0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969);
&_data_word(0x91868617, 0x58c1c199, 0x271d1d3a, 0xb99e9e27);
&_data_word(0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122);
&_data_word(0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433);
&_data_word(0xb69b9b2d, 0x221e1e3c, 0x92878715, 0x20e9e9c9);
&_data_word(0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5);
&_data_word(0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a);
&_data_word(0xdabfbf65, 0x31e6e6d7, 0xc6424284, 0xb86868d0);
&_data_word(0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e);
&_data_word(0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c);
#rcon:
&data_word(0x00000001, 0x00000002, 0x00000004, 0x00000008);
&data_word(0x00000010, 0x00000020, 0x00000040, 0x00000080);
Andy Polyakov
committed
&data_word(0x0000001b, 0x00000036, 0, 0, 0, 0, 0, 0);
&function_end_B("_x86_AES_encrypt");
# void AES_encrypt (const void *inp,void *out,const AES_KEY *key);
&public_label("AES_Te");
&function_begin("AES_encrypt");
&mov ($acc,&wparam(0)); # load inp
&mov ($key,&wparam(2)); # load key
&mov ($s0,"esp");
&sub ("esp",24);
&and ("esp",-64);
&add ("esp",4);
&mov (&DWP(16,"esp"),$s0);
&call (&label("pic_point")); # make it PIC!
&set_label("pic_point");
&blindpop("ebp");
&lea ("ebp",&DWP(&label("AES_Te")."-".&label("pic_point"),"ebp"));
&mov ($s0,&DWP(0,$acc)); # load input data
&mov ($s1,&DWP(4,$acc));
&mov ($s2,&DWP(8,$acc));
&mov ($s3,&DWP(12,$acc));
&call ("_x86_AES_encrypt");
&mov ("esp",&DWP(16,"esp"));
&mov ($acc,&wparam(1)); # load out
&mov (&DWP(0,$acc),$s0); # write output data
&mov (&DWP(4,$acc),$s1);
&mov (&DWP(8,$acc),$s2);
&mov (&DWP(12,$acc),$s3);
&function_end("AES_encrypt");
#------------------------------------------------------------------#
sub decstep()
{ my ($i,$td,@s) = @_;
my $tmp = $key;
my $out = $i==3?$s[0]:$acc;
# no instructions are reordered, as performance appears
# optimal... or rather that all attempts to reorder didn't
# result in better performance [which by the way is not a
# bit lower than ecryption].
if($i==3) { &mov ($key,&DWP(12,"esp")); }
else { &mov ($out,$s[0]); }
&and ($out,0xFF);
&mov ($out,&DWP(0,$td,$out,8));
if ($i==3) { $tmp=$s[1]; }
&movz ($tmp,&HB($s[1]));
&xor ($out,&DWP(3,$td,$tmp,8));
if ($i==3) { $tmp=$s[2]; &mov ($s[1],$acc); }
else { &mov ($tmp,$s[2]); }
&shr ($tmp,16);
&and ($tmp,0xFF);
&xor ($out,&DWP(2,$td,$tmp,8));
if ($i==3) { $tmp=$s[3]; &mov ($s[2],&DWP(8,"esp")); }
else { &mov ($tmp,$s[3]); }
&shr ($tmp,24);
&xor ($out,&DWP(1,$td,$tmp,8));
if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
if ($i==3) { &mov ($s[3],&DWP(4,"esp")); }
&comment();
}
sub declast()
{ my ($i,$td,@s)=@_;
my $tmp = $key;
my $out = $i==3?$s[0]:$acc;
if($i==3) { &mov ($key,&DWP(12,"esp")); }
else { &mov ($out,$s[0]); }
&and ($out,0xFF);
&mov ($out,&DWP(2048,$td,$out,4));
&and ($out,0x000000ff);
if ($i==3) { $tmp=$s[1]; }
&movz ($tmp,&HB($s[1]));
&mov ($tmp,&DWP(2048,$td,$tmp,4));
&and ($tmp,0x0000ff00);
if ($i==3) { $tmp=$s[2]; &mov ($s[1],$acc); }
else { mov ($tmp,$s[2]); }
&shr ($tmp,16);
&and ($tmp,0xFF);
&mov ($tmp,&DWP(2048,$td,$tmp,4));
&and ($tmp,0x00ff0000);
if ($i==3) { $tmp=$s[3]; &mov ($s[2],&DWP(8,"esp")); }
else { &mov ($tmp,$s[3]); }
&shr ($tmp,24);
&mov ($tmp,&DWP(2048,$td,$tmp,4));
&and ($tmp,0xff000000);
if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
if ($i==3) { &mov ($s[3],&DWP(4,"esp")); }
&function_begin_B("_x86_AES_decrypt");
# note that caller is expected to allocate stack frame for me!
&mov (&DWP(12,"esp"),$key); # save key
&xor ($s0,&DWP(0,$key)); # xor with key
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&mov ($acc,&DWP(240,$key)); # load key->rounds
&lea ($acc,&DWP(-2,$acc,$acc));
&lea ($acc,&DWP(0,$key,$acc,8));
&mov (&DWP(16,"esp"),$acc); # end of key schedule
&align (4);
&set_label("loop");
&decstep(0,"ebp",$s0,$s3,$s2,$s1);
&decstep(1,"ebp",$s1,$s0,$s3,$s2);
&decstep(2,"ebp",$s2,$s1,$s0,$s3);
&decstep(3,"ebp",$s3,$s2,$s1,$s0);
&add ($key,16); # advance rd_key
&xor ($s0,&DWP(0,$key));
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&cmp ($key,&DWP(16,"esp"));
&mov (&DWP(12,"esp"),$key);
&jb (&label("loop"));
}
else {
&cmp ($acc,10);
&cmp ($acc,12);
&jle (&label("12rounds"));
&set_label("14rounds");
for ($i=1;$i<3;$i++) {
&decstep(0,"ebp",$s0,$s3,$s2,$s1);
&decstep(1,"ebp",$s1,$s0,$s3,$s2);
&decstep(2,"ebp",$s2,$s1,$s0,$s3);
&decstep(3,"ebp",$s3,$s2,$s1,$s0);
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
&add ($key,32);
&mov (&DWP(12,"esp"),$key); # advance rd_key
&set_label("12rounds");
for ($i=1;$i<3;$i++) {
&decstep(0,"ebp",$s0,$s3,$s2,$s1);
&decstep(1,"ebp",$s1,$s0,$s3,$s2);
&decstep(2,"ebp",$s2,$s1,$s0,$s3);
&decstep(3,"ebp",$s3,$s2,$s1,$s0);
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
&add ($key,32);
&mov (&DWP(12,"esp"),$key); # advance rd_key
&set_label("10rounds");
for ($i=1;$i<10;$i++) {
&decstep(0,"ebp",$s0,$s3,$s2,$s1);
&decstep(1,"ebp",$s1,$s0,$s3,$s2);
&decstep(2,"ebp",$s2,$s1,$s0,$s3);
&decstep(3,"ebp",$s3,$s2,$s1,$s0);
&xor ($s0,&DWP(16*$i+0,$key));
&xor ($s1,&DWP(16*$i+4,$key));
&xor ($s2,&DWP(16*$i+8,$key));
&xor ($s3,&DWP(16*$i+12,$key));
}
}
&declast(0,"ebp",$s0,$s3,$s2,$s1);
&declast(1,"ebp",$s1,$s0,$s3,$s2);
&declast(2,"ebp",$s2,$s1,$s0,$s3);
&declast(3,"ebp",$s3,$s2,$s1,$s0);
&add ($key,$small_footprint?16:160);
&xor ($s0,&DWP(0,$key));
&xor ($s1,&DWP(4,$key));
&xor ($s2,&DWP(8,$key));
&xor ($s3,&DWP(12,$key));
&ret ();
&set_label("AES_Td",64); # Yes! I keep it in the code segment!
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
&_data_word(0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a);
&_data_word(0xcb6bab3b, 0xf1459d1f, 0xab58faac, 0x9303e34b);
&_data_word(0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5);
&_data_word(0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5);
&_data_word(0x495ab1de, 0x671bba25, 0x980eea45, 0xe1c0fe5d);
&_data_word(0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b);
&_data_word(0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295);
&_data_word(0x2d83bed4, 0xd3217458, 0x2969e049, 0x44c8c98e);
&_data_word(0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927);
&_data_word(0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d);
&_data_word(0x184adf63, 0x82311ae5, 0x60335197, 0x457f5362);
&_data_word(0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9);
&_data_word(0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52);
&_data_word(0x23d373ab, 0xe2024b72, 0x578f1fe3, 0x2aab5566);
&_data_word(0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3);
&_data_word(0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed);
&_data_word(0x2b1ccf8a, 0x92b479a7, 0xf0f207f3, 0xa1e2694e);
&_data_word(0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4);
&_data_word(0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4);
&_data_word(0x39ec830b, 0xaaef6040, 0x069f715e, 0x51106ebd);
&_data_word(0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d);
&_data_word(0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060);
&_data_word(0x24fb9819, 0x97e9bdd6, 0xcc434089, 0x779ed967);
&_data_word(0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879);
&_data_word(0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000);
&_data_word(0x83868009, 0x48ed2b32, 0xac70111e, 0x4e725a6c);
&_data_word(0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36);
&_data_word(0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624);
&_data_word(0xb1670a0c, 0x0fe75793, 0xd296eeb4, 0x9e919b1b);
&_data_word(0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c);
&_data_word(0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12);
&_data_word(0x0b0d090e, 0xadc78bf2, 0xb9a8b62d, 0xc8a91e14);
&_data_word(0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3);
&_data_word(0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b);
&_data_word(0x7629438b, 0xdcc623cb, 0x68fcedb6, 0x63f1e4b8);
&_data_word(0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684);
&_data_word(0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7);
&_data_word(0x4b2f9e1d, 0xf330b2dc, 0xec52860d, 0xd0e3c177);
&_data_word(0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947);
&_data_word(0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322);
&_data_word(0xc74e4987, 0xc1d138d9, 0xfea2ca8c, 0x360bd498);
&_data_word(0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f);
&_data_word(0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54);
&_data_word(0xc2138df6, 0xe8b8d890, 0x5ef7392e, 0xf5afc382);
&_data_word(0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf);
&_data_word(0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb);
&_data_word(0x097826cd, 0xf418596e, 0x01b79aec, 0xa89a4f83);
&_data_word(0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef);
&_data_word(0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029);
&_data_word(0xafb2a431, 0x31233f2a, 0x3094a5c6, 0xc066a235);
&_data_word(0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733);
&_data_word(0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117);
&_data_word(0x8dd64d76, 0x4db0ef43, 0x544daacc, 0xdf0496e4);
&_data_word(0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546);
&_data_word(0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb);
&_data_word(0x5a1d67b3, 0x52d2db92, 0x335610e9, 0x1347d66d);
&_data_word(0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb);
&_data_word(0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a);
&_data_word(0x59dfd29c, 0x3f73f255, 0x79ce1418, 0xbf37c773);
&_data_word(0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478);
&_data_word(0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2);
&_data_word(0x72c31d16, 0x0c25e2bc, 0x8b493c28, 0x41950dff);
&_data_word(0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664);
&_data_word(0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0);
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
#Td4:
&data_word(0x52525252, 0x09090909, 0x6a6a6a6a, 0xd5d5d5d5);
&data_word(0x30303030, 0x36363636, 0xa5a5a5a5, 0x38383838);
&data_word(0xbfbfbfbf, 0x40404040, 0xa3a3a3a3, 0x9e9e9e9e);
&data_word(0x81818181, 0xf3f3f3f3, 0xd7d7d7d7, 0xfbfbfbfb);
&data_word(0x7c7c7c7c, 0xe3e3e3e3, 0x39393939, 0x82828282);
&data_word(0x9b9b9b9b, 0x2f2f2f2f, 0xffffffff, 0x87878787);
&data_word(0x34343434, 0x8e8e8e8e, 0x43434343, 0x44444444);
&data_word(0xc4c4c4c4, 0xdededede, 0xe9e9e9e9, 0xcbcbcbcb);
&data_word(0x54545454, 0x7b7b7b7b, 0x94949494, 0x32323232);
&data_word(0xa6a6a6a6, 0xc2c2c2c2, 0x23232323, 0x3d3d3d3d);
&data_word(0xeeeeeeee, 0x4c4c4c4c, 0x95959595, 0x0b0b0b0b);
&data_word(0x42424242, 0xfafafafa, 0xc3c3c3c3, 0x4e4e4e4e);
&data_word(0x08080808, 0x2e2e2e2e, 0xa1a1a1a1, 0x66666666);
&data_word(0x28282828, 0xd9d9d9d9, 0x24242424, 0xb2b2b2b2);
&data_word(0x76767676, 0x5b5b5b5b, 0xa2a2a2a2, 0x49494949);
&data_word(0x6d6d6d6d, 0x8b8b8b8b, 0xd1d1d1d1, 0x25252525);
&data_word(0x72727272, 0xf8f8f8f8, 0xf6f6f6f6, 0x64646464);
&data_word(0x86868686, 0x68686868, 0x98989898, 0x16161616);
&data_word(0xd4d4d4d4, 0xa4a4a4a4, 0x5c5c5c5c, 0xcccccccc);
&data_word(0x5d5d5d5d, 0x65656565, 0xb6b6b6b6, 0x92929292);
&data_word(0x6c6c6c6c, 0x70707070, 0x48484848, 0x50505050);
&data_word(0xfdfdfdfd, 0xedededed, 0xb9b9b9b9, 0xdadadada);
&data_word(0x5e5e5e5e, 0x15151515, 0x46464646, 0x57575757);
&data_word(0xa7a7a7a7, 0x8d8d8d8d, 0x9d9d9d9d, 0x84848484);
&data_word(0x90909090, 0xd8d8d8d8, 0xabababab, 0x00000000);
&data_word(0x8c8c8c8c, 0xbcbcbcbc, 0xd3d3d3d3, 0x0a0a0a0a);
&data_word(0xf7f7f7f7, 0xe4e4e4e4, 0x58585858, 0x05050505);
&data_word(0xb8b8b8b8, 0xb3b3b3b3, 0x45454545, 0x06060606);
&data_word(0xd0d0d0d0, 0x2c2c2c2c, 0x1e1e1e1e, 0x8f8f8f8f);
&data_word(0xcacacaca, 0x3f3f3f3f, 0x0f0f0f0f, 0x02020202);
&data_word(0xc1c1c1c1, 0xafafafaf, 0xbdbdbdbd, 0x03030303);
&data_word(0x01010101, 0x13131313, 0x8a8a8a8a, 0x6b6b6b6b);
&data_word(0x3a3a3a3a, 0x91919191, 0x11111111, 0x41414141);
&data_word(0x4f4f4f4f, 0x67676767, 0xdcdcdcdc, 0xeaeaeaea);
&data_word(0x97979797, 0xf2f2f2f2, 0xcfcfcfcf, 0xcececece);
&data_word(0xf0f0f0f0, 0xb4b4b4b4, 0xe6e6e6e6, 0x73737373);
&data_word(0x96969696, 0xacacacac, 0x74747474, 0x22222222);
&data_word(0xe7e7e7e7, 0xadadadad, 0x35353535, 0x85858585);
&data_word(0xe2e2e2e2, 0xf9f9f9f9, 0x37373737, 0xe8e8e8e8);
&data_word(0x1c1c1c1c, 0x75757575, 0xdfdfdfdf, 0x6e6e6e6e);
&data_word(0x47474747, 0xf1f1f1f1, 0x1a1a1a1a, 0x71717171);
&data_word(0x1d1d1d1d, 0x29292929, 0xc5c5c5c5, 0x89898989);
&data_word(0x6f6f6f6f, 0xb7b7b7b7, 0x62626262, 0x0e0e0e0e);
&data_word(0xaaaaaaaa, 0x18181818, 0xbebebebe, 0x1b1b1b1b);
&data_word(0xfcfcfcfc, 0x56565656, 0x3e3e3e3e, 0x4b4b4b4b);
&data_word(0xc6c6c6c6, 0xd2d2d2d2, 0x79797979, 0x20202020);
&data_word(0x9a9a9a9a, 0xdbdbdbdb, 0xc0c0c0c0, 0xfefefefe);
&data_word(0x78787878, 0xcdcdcdcd, 0x5a5a5a5a, 0xf4f4f4f4);
&data_word(0x1f1f1f1f, 0xdddddddd, 0xa8a8a8a8, 0x33333333);
&data_word(0x88888888, 0x07070707, 0xc7c7c7c7, 0x31313131);
&data_word(0xb1b1b1b1, 0x12121212, 0x10101010, 0x59595959);
&data_word(0x27272727, 0x80808080, 0xecececec, 0x5f5f5f5f);
&data_word(0x60606060, 0x51515151, 0x7f7f7f7f, 0xa9a9a9a9);
&data_word(0x19191919, 0xb5b5b5b5, 0x4a4a4a4a, 0x0d0d0d0d);
&data_word(0x2d2d2d2d, 0xe5e5e5e5, 0x7a7a7a7a, 0x9f9f9f9f);
&data_word(0x93939393, 0xc9c9c9c9, 0x9c9c9c9c, 0xefefefef);
&data_word(0xa0a0a0a0, 0xe0e0e0e0, 0x3b3b3b3b, 0x4d4d4d4d);
&data_word(0xaeaeaeae, 0x2a2a2a2a, 0xf5f5f5f5, 0xb0b0b0b0);
&data_word(0xc8c8c8c8, 0xebebebeb, 0xbbbbbbbb, 0x3c3c3c3c);
&data_word(0x83838383, 0x53535353, 0x99999999, 0x61616161);
&data_word(0x17171717, 0x2b2b2b2b, 0x04040404, 0x7e7e7e7e);
&data_word(0xbabababa, 0x77777777, 0xd6d6d6d6, 0x26262626);
&data_word(0xe1e1e1e1, 0x69696969, 0x14141414, 0x63636363);
&data_word(0x55555555, 0x21212121, 0x0c0c0c0c, 0x7d7d7d7d);
&function_end_B("_x86_AES_decrypt");
# void AES_decrypt (const void *inp,void *out,const AES_KEY *key);
&public_label("AES_Td");
&function_begin("AES_decrypt");
&mov ($acc,&wparam(0)); # load inp
&mov ($key,&wparam(2)); # load key
&mov ($s0,"esp");
&sub ("esp",24);
&and ("esp",-64);
&add ("esp",4);
&mov (&DWP(16,"esp"),$s0);
&call (&label("pic_point")); # make it PIC!
&set_label("pic_point");
&blindpop("ebp");
&lea ("ebp",&DWP(&label("AES_Td")."-".&label("pic_point"),"ebp"));
&mov ($s0,&DWP(0,$acc)); # load input data
&mov ($s1,&DWP(4,$acc));
&mov ($s2,&DWP(8,$acc));
&mov ($s3,&DWP(12,$acc));
&call ("_x86_AES_decrypt");
&mov ("esp",&DWP(16,"esp"));
&mov ($acc,&wparam(1)); # load out
&mov (&DWP(0,$acc),$s0); # write output data
&mov (&DWP(4,$acc),$s1);
&mov (&DWP(8,$acc),$s2);
&mov (&DWP(12,$acc),$s3);
&function_end("AES_decrypt");
# void AES_cbc_encrypt (const void char *inp, unsigned char *out,
# size_t length, const AES_KEY *key,
# unsigned char *ivp,const int enc);
{
# stack frame layout
# -4(%esp) 0(%esp) return address
# 0(%esp) 4(%esp) tmp1
# 4(%esp) 8(%esp) tmp2
# 8(%esp) 12(%esp) key
# 12(%esp) 16(%esp) end of key schedule
my $_esp=&DWP(16,"esp"); #saved %esp
my $_inp=&DWP(20,"esp"); #copy of wparam(0)
my $_out=&DWP(24,"esp"); #copy of wparam(1)
my $_len=&DWP(28,"esp"); #copy of wparam(2)
my $_key=&DWP(32,"esp"); #copy of wparam(3)
my $_ivp=&DWP(36,"esp"); #copy of wparam(4)
my $_tmp=&DWP(40,"esp"); #volatile variable
my $ivec=&DWP(44,"esp"); #ivec[16]
my $aes_key=&DWP(60,"esp"); #copy of aes_key
&public_label("AES_Te");
&public_label("AES_Td");
&function_begin("AES_cbc_encrypt");
&mov ($s2 eq "ecx"? $s2 : "",&wparam(2)); # load len
&cmp ($s2,0);
&je (&label("enc_out"));
&call (&label("pic_point")); # make it PIC!
&set_label("pic_point");
&blindpop("ebp");
&pushf ();
&cld ();
&cmp (&wparam(5),0);
&je (&label("DECRYPT"));
&lea ("ebp",&DWP(&label("AES_Te")."-".&label("pic_point"),"ebp"));
# allocate aligned stack frame...
&and ($key,-64);
# ... and make sure it doesn't alias with AES_Te modulo 4096
&mov ($s0,"ebp");
&lea ($s1,&DWP(2048,"ebp"));
&mov ($s3,$key);
&and ($s0,0xfff); # s = %ebp&0xfff
&and ($s1,0xfff); # e = (%ebp+2048)&0xfff
&and ($s3,0xfff); # p = %esp&0xfff
&cmp ($s3,$s1); # if (p>=e) %esp =- (p-e);
&jb (&label("te_break_out"));
&sub ($s3,$s1);
&sub ($key,$s3);
&jmp (&label("te_ok"));
&set_label("te_break_out"); # else %esp -= (p-s)&0xfff + framesz;
&sub ($s3,$s0);
&and ($s3,0xfff);
&sub ($key,$s3);
&align (4);
&set_label("te_ok");
&mov ($s0,&wparam(0)); # load inp
&mov ($s1,&wparam(1)); # load out
&mov ($s3,&wparam(3)); # load key
&mov ($acc,&wparam(4)); # load ivp
&exch ("esp",$key);
&add ("esp",4); # reserve for return address!
&mov ($_esp,$key); # save %esp
&mov ($_inp,$s0); # save copy of inp
&mov ($_out,$s1); # save copy of out
&mov ($_len,$s2); # save copy of len
&mov ($_key,$s3); # save copy of key
&mov ($_ivp,$acc); # save copy of ivp
if ($compromise) {
&cmp ($s2,$compromise);
&jb (&label("skip_ecopy"));
}
# copy key schedule to stack
&mov ("esi",$s3);
&lea ("edi",$aes_key);
&mov ($_key,"edi");
&align (4);
&data_word(0xF689A5F3); # rep movsd
&set_label("skip_ecopy") if ($compromise);
&mov ($acc,$s0);
&mov ($key,16);
&align (4);
&set_label("prefetch_te");
&mov ($s0,&DWP(0,"ebp"));
&mov ($s1,&DWP(32,"ebp"));
&mov ($s2,&DWP(64,"ebp"));
&mov ($s3,&DWP(96,"ebp"));
&lea ("ebp",&DWP(128,"ebp"));
&dec ($key);
&jnz (&label("prefetch_te"));
&sub ("ebp",2048);
&mov ($s2,$_len);
&mov ($key,$_ivp);
&test ($s2,0xFFFFFFF0);
&jz (&label("enc_tail")); # short input...
&mov ($s0,&DWP(0,$key)); # load iv
&mov ($s1,&DWP(4,$key));
&align (4);
&set_label("enc_loop");
&mov ($s2,&DWP(8,$key));
&mov ($s3,&DWP(12,$key));
&xor ($s0,&DWP(0,$acc)); # xor input data
&xor ($s1,&DWP(4,$acc));
&xor ($s2,&DWP(8,$acc));
&xor ($s3,&DWP(12,$acc));
&mov ($key,$_key); # load key
&call ("_x86_AES_encrypt");
&mov ($acc,$_inp); # load inp
&mov ($key,$_out); # load out
&mov (&DWP(0,$key),$s0); # save output data
&mov (&DWP(4,$key),$s1);
&mov (&DWP(8,$key),$s2);
&mov (&DWP(12,$key),$s3);
&mov ($s2,$_len); # load len
&lea ($acc,&DWP(16,$acc));
&mov ($_inp,$acc); # save inp
&lea ($s3,&DWP(16,$key));
&mov ($_out,$s3); # save out
&sub ($s2,16);
&test ($s2,0xFFFFFFF0);
&mov ($_len,$s2); # save len
&jnz (&label("enc_loop"));
&test ($s2,15);
&jnz (&label("enc_tail"));
&mov ($acc,$_ivp); # load ivp
&mov ($s2,&DWP(8,$key)); # restore last dwords
&mov ($s3,&DWP(12,$key));
&mov (&DWP(0,$acc),$s0); # save ivec
&mov (&DWP(4,$acc),$s1);
&mov (&DWP(8,$acc),$s2);
&mov (&DWP(12,$acc),$s3);
&mov ("edi",$_key);
&mov ("esp",$_esp);
if ($compromise) {
&cmp (&wparam(2),$compromise);
&jb (&label("skip_ezero"));
}
# zero copy of key schedule
&xor ("eax","eax");
&align (4);
&data_word(0xF689ABF3); # rep stosd
&set_label("skip_ezero") if ($compromise);
&popf ();
&set_label("enc_out");
&function_end_A();
&pushf (); # kludge, never executed
&align (4);
&set_label("enc_tail");
&push ($key eq "edi" ? $key : ""); # push ivp
&mov ($key,$_out); # load out
&mov ($s1,16);
&sub ($s1,$s2);
&cmp ($key,$acc); # compare with inp
&je (&label("enc_in_place"));
&align (4);
&data_word(0xF689A4F3); # rep movsb # copy input
&jmp (&label("enc_skip_in_place"));
&set_label("enc_in_place");
&lea ($key,&DWP(0,$key,$s2));
&set_label("enc_skip_in_place");
&mov ($s2,$s1);
&xor ($s0,$s0);
&align (4);
&data_word(0xF689AAF3); # rep stosb # zero tail
&pop ($key); # pop ivp
&mov ($acc,$_out); # output as input
&mov ($s0,&DWP(0,$key));
&mov ($s1,&DWP(4,$key));
&mov ($_len,16); # len=16
&jmp (&label("enc_loop")); # one more spin...
#----------------------------- DECRYPT -----------------------------#
&align (4);
&set_label("DECRYPT");
&lea ("ebp",&DWP(&label("AES_Td")."-".&label("pic_point"),"ebp"));
# allocate aligned stack frame...
&and ($key,-64);
# ... and make sure it doesn't alias with AES_Td modulo 4096
&mov ($s0,"ebp");
&lea ($s1,&DWP(3072,"ebp"));
&mov ($s3,$key);