Commit 1fab06a6 authored by Andy Polyakov's avatar Andy Polyakov
Browse files

crypto/blake2: make lowest-level function handle multiple blocks.. 



This minimizes inter-block overhead. Performance gain naturally
varies from case to case, up to 10% was spotted so far. There is
one thing to recognize, given same circumstances gain would be
higher faster computational part is. Or in other words biggest
improvement coefficient would have been observed with assembly.

Reviewed-by: default avatarEmilia Käsper <emilia@openssl.org>
Reviewed-by: default avatarRich Salz <rsalz@openssl.org>
parent 349232d1

crypto/blake2/blake2_impl.h

+24 −29
Original line number Diff line number Diff line
@@ -30,10 +30,10 @@ static ossl_inline uint32_t load32(const uint8_t *src) 
        memcpy(&w, src, sizeof(w));

        return w;

    } else {

        uint32_t w = *src++;

        w |= (uint32_t)(*src++) <<  8;

        w |= (uint32_t)(*src++) << 16;

        w |= (uint32_t)(*src++) << 24;

        uint32_t w = ((uint32_t)src[0])

                   | ((uint32_t)src[1] <<  8)

                   | ((uint32_t)src[2] << 16)

                   | ((uint32_t)src[3] << 24);

        return w;

    }

}
@@ -50,14 +50,14 @@ static ossl_inline uint64_t load64(const uint8_t *src) 
        memcpy(&w, src, sizeof(w));

        return w;

    } else {

        uint64_t w = *src++;

        w |= (uint64_t)(*src++) <<  8;

        w |= (uint64_t)(*src++) << 16;

        w |= (uint64_t)(*src++) << 24;

        w |= (uint64_t)(*src++) << 32;

        w |= (uint64_t)(*src++) << 40;

        w |= (uint64_t)(*src++) << 48;

        w |= (uint64_t)(*src++) << 56;

        uint64_t w = ((uint64_t)src[0])

                   | ((uint64_t)src[1] <<  8)

                   | ((uint64_t)src[2] << 16)

                   | ((uint64_t)src[3] << 24)

                   | ((uint64_t)src[4] << 32)

                   | ((uint64_t)src[5] << 40)

                   | ((uint64_t)src[6] << 48)

                   | ((uint64_t)src[7] << 56);

        return w;

    }

}
@@ -100,29 +100,24 @@ static ossl_inline void store64(uint8_t *dst, uint64_t w) 


static ossl_inline uint64_t load48(const uint8_t *src)

{

    uint64_t w = *src++;

    w |= (uint64_t)(*src++) <<  8;

    w |= (uint64_t)(*src++) << 16;

    w |= (uint64_t)(*src++) << 24;

    w |= (uint64_t)(*src++) << 32;

    w |= (uint64_t)(*src++) << 40;

    uint64_t w = ((uint64_t)src[0])

               | ((uint64_t)src[1] <<  8)

               | ((uint64_t)src[2] << 16)

               | ((uint64_t)src[3] << 24)

               | ((uint64_t)src[4] << 32)

               | ((uint64_t)src[5] << 40);

    return w;

}



static ossl_inline void store48(uint8_t *dst, uint64_t w)

{

    uint8_t *p = (uint8_t *)dst;

    *p++ = (uint8_t)w;

    w >>= 8;

    *p++ = (uint8_t)w;

    w >>= 8;

    *p++ = (uint8_t)w;

    w >>= 8;

    *p++ = (uint8_t)w;

    w >>= 8;

    *p++ = (uint8_t)w;

    w >>= 8;

    *p++ = (uint8_t)w;

    p[0] = (uint8_t)w;

    p[1] = (uint8_t)(w>>8);

    p[2] = (uint8_t)(w>>16);

    p[3] = (uint8_t)(w>>24);

    p[4] = (uint8_t)(w>>32);

    p[5] = (uint8_t)(w>>40);

}



static ossl_inline uint32_t rotr32(const uint32_t w, const unsigned int c)

crypto/blake2/blake2b.c

+125 −72
Original line number Diff line number Diff line
@@ -15,6 +15,12 @@ 
 * can be found at https://blake2.net.

 */



#ifndef BLAKE_DEBUG

# undef NDEBUG                  /* avoid conflicting definitions */

# define NDEBUG

#endif



#include <assert.h>

#include <string.h>

#include <openssl/crypto.h>

#include "e_os.h"
@@ -52,14 +58,6 @@ static ossl_inline void blake2b_set_lastblock(BLAKE2B_CTX *S) 
    S->f[0] = -1;

}



/* Increment the data hashed counter. */

static ossl_inline void blake2b_increment_counter(BLAKE2B_CTX *S,

                                                  const uint64_t inc)

{

    S->t[0] += inc;

    S->t[1] += (S->t[0] < inc);

}



/* Initialize the hashing state. */

static ossl_inline void blake2b_init0(BLAKE2B_CTX *S)

{
@@ -108,20 +106,48 @@ int BLAKE2b_Init(BLAKE2B_CTX *c) 


/* Permute the state while xoring in the block of data. */

static void blake2b_compress(BLAKE2B_CTX *S,

                            const uint8_t block[BLAKE2B_BLOCKBYTES])

                            const uint8_t *blocks,

                            size_t len)

{

    uint64_t m[16];

    uint64_t v[16];

    int i;

    size_t increment;



    for(i = 0; i < 16; ++i) {

        m[i] = load64(block + i * sizeof(m[i]));

    }

    /*

     * There are two distinct usage vectors for this function:

     *

     * a) BLAKE2b_Update uses it to process complete blocks,

     *    possibly more than one at a time;

     *

     * b) BLAK2b_Final uses it to process last block, always

     *    single but possibly incomplete, in which case caller

     *    pads input with zeros.

     */

    assert(len < BLAKE2B_BLOCKBYTES || len % BLAKE2B_BLOCKBYTES == 0);



    /*

     * Since last block is always processed with separate call,

     * |len| not being multiple of complete blocks can be observed

     * only with |len| being less than BLAKE2B_BLOCKBYTES ("less"

     * including even zero), which is why following assignment doesn't

     * have to reside inside the main loop below.

     */

    increment = len < BLAKE2B_BLOCKBYTES ? len : BLAKE2B_BLOCKBYTES;



    for (i = 0; i < 8; ++i) {

        v[i] = S->h[i];

    }



    do {

        for (i = 0; i < 16; ++i) {

            m[i] = load64(blocks + i * sizeof(m[i]));

        }



        /* blake2b_increment_counter */

        S->t[0] += increment;

        S->t[1] += (S->t[0] < increment);



        v[8]  = blake2b_IV[0];

        v[9]  = blake2b_IV[1];

        v[10] = blake2b_IV[2];
@@ -152,6 +178,12 @@ static void blake2b_compress(BLAKE2B_CTX *S, 
            G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \

            G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \

        } while (0)

#if defined(OPENSSL_SMALL_FOOTPRINT)

        /* 3x size reduction on x86_64, almost 7x on ARMv8, 9x on ARMv4 */

        for (i = 0; i < 12; i++) {

            ROUND(i);

        }

#else

        ROUND(0);

        ROUND(1);

        ROUND(2);
@@ -164,13 +196,16 @@ static void blake2b_compress(BLAKE2B_CTX *S, 
        ROUND(9);

        ROUND(10);

        ROUND(11);

#endif



        for (i = 0; i < 8; ++i) {

        S->h[i] = S->h[i] ^ v[i] ^ v[i + 8];

            S->h[i] = v[i] ^= v[i + 8] ^ S->h[i];

        }



#undef G

#undef ROUND

        blocks += increment;

        len -= increment;

    } while (len);

}



/* Absorb the input data into the hash state.  Always returns 1. */
@@ -179,22 +214,41 @@ int BLAKE2b_Update(BLAKE2B_CTX *c, const void *data, size_t datalen) 
    const uint8_t *in = data;

    size_t fill;



    while(datalen > 0) {

    /*

     * Intuitively one would expect intermediate buffer, c->buf, to

     * store incomplete blocks. But in this case we are interested to

     * temporarily stash even complete blocks, because last one in the

     * stream has to be treated in special way, and at this point we

     * don't know if last block in *this* call is last one "ever". This

     * is the reason for why |datalen| is compared as >, and not >=.

     */

    fill = sizeof(c->buf) - c->buflen;

        /* Must be >, not >=, so that last block can be hashed differently */

    if (datalen > fill) {

        if (c->buflen) {

            memcpy(c->buf + c->buflen, in, fill); /* Fill buffer */

            blake2b_increment_counter(c, BLAKE2B_BLOCKBYTES);

            blake2b_compress(c, c->buf); /* Compress */

            blake2b_compress(c, c->buf, BLAKE2B_BLOCKBYTES);

            c->buflen = 0;

            in += fill;

            datalen -= fill;

        } else { /* datalen <= fill */

            memcpy(c->buf + c->buflen, in, datalen);

            c->buflen += datalen; /* Be lazy, do not compress */

            return 1;

        }

        if (datalen > BLAKE2B_BLOCKBYTES) {

            size_t stashlen = datalen % BLAKE2B_BLOCKBYTES;

            /*

	     * If |datalen| is a multiple of the blocksize, stash

	     * last complete block, it can be final one...

             */

            stashlen = stashlen ? stashlen : BLAKE2B_BLOCKBYTES;

            datalen -= stashlen;

            blake2b_compress(c, in, datalen);

            in += datalen;

            datalen = stashlen;

        }

    }



    assert(datalen <= BLAKE2B_BLOCKBYTES);



    memcpy(c->buf + c->buflen, in, datalen);

    c->buflen += datalen; /* Be lazy, do not compress */



    return 1;

}
@@ -207,11 +261,10 @@ int BLAKE2b_Final(unsigned char *md, BLAKE2B_CTX *c) 
{

    int i;



    blake2b_increment_counter(c, c->buflen);

    blake2b_set_lastblock(c);

    /* Padding */

    memset(c->buf + c->buflen, 0, sizeof(c->buf) - c->buflen);

    blake2b_compress(c, c->buf);

    blake2b_compress(c, c->buf, c->buflen);



    /* Output full hash to message digest */

    for (i = 0; i < 8; ++i) {

crypto/blake2/blake2s.c

+123 −70
Original line number Diff line number Diff line
@@ -15,6 +15,12 @@ 
 * can be found at https://blake2.net.

 */



#ifndef BLAKE_DEBUG

# undef NDEBUG                  /* avoid conflicting definitions */

# define NDEBUG

#endif



#include <assert.h>

#include <string.h>

#include <openssl/crypto.h>

#include "e_os.h"
@@ -48,14 +54,6 @@ static ossl_inline void blake2s_set_lastblock(BLAKE2S_CTX *S) 
    S->f[0] = -1;

}



/* Increment the data hashed counter. */

static ossl_inline void blake2s_increment_counter(BLAKE2S_CTX *S,

                                                  const uint32_t inc)

{

    S->t[0] += inc;

    S->t[1] += (S->t[0] < inc);

}



/* Initialize the hashing state. */

static ossl_inline void blake2s_init0(BLAKE2S_CTX *S)

{
@@ -104,20 +102,48 @@ int BLAKE2s_Init(BLAKE2S_CTX *c) 


/* Permute the state while xoring in the block of data. */

static void blake2s_compress(BLAKE2S_CTX *S,

                            const uint8_t block[BLAKE2S_BLOCKBYTES])

                            const uint8_t *blocks,

                            size_t len)

{

    uint32_t m[16];

    uint32_t v[16];

    size_t i;

    size_t increment;



    for(i = 0; i < 16; ++i) {

        m[i] = load32(block + i * sizeof(m[i]));

    }

    /*

     * There are two distinct usage vectors for this function:

     *

     * a) BLAKE2s_Update uses it to process complete blocks,

     *    possibly more than one at a time;

     *

     * b) BLAK2s_Final uses it to process last block, always

     *    single but possibly incomplete, in which case caller

     *    pads input with zeros.

     */

    assert(len < BLAKE2S_BLOCKBYTES || len % BLAKE2S_BLOCKBYTES == 0);



    /*

     * Since last block is always processed with separate call,

     * |len| not being multiple of complete blocks can be observed

     * only with |len| being less than BLAKE2S_BLOCKBYTES ("less"

     * including even zero), which is why following assignment doesn't

     * have to reside inside the main loop below.

     */

    increment = len < BLAKE2S_BLOCKBYTES ? len : BLAKE2S_BLOCKBYTES;



    for (i = 0; i < 8; ++i) {

        v[i] = S->h[i];

    }



    do {

        for (i = 0; i < 16; ++i) {

            m[i] = load32(blocks + i * sizeof(m[i]));

        }



        /* blake2s_increment_counter */

        S->t[0] += increment;

        S->t[1] += (S->t[0] < increment);



        v[ 8] = blake2s_IV[0];

        v[ 9] = blake2s_IV[1];

        v[10] = blake2s_IV[2];
@@ -148,6 +174,12 @@ static void blake2s_compress(BLAKE2S_CTX *S, 
            G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \

            G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \

        } while (0)

#if defined(OPENSSL_SMALL_FOOTPRINT)

        /* almost 3x reduction on x86_64, 4.5x on ARMv8, 4x on ARMv4 */

        for (i = 0; i < 10; i++) {

            ROUND(i);

        }

#else

        ROUND(0);

        ROUND(1);

        ROUND(2);
@@ -158,13 +190,16 @@ static void blake2s_compress(BLAKE2S_CTX *S, 
        ROUND(7);

        ROUND(8);

        ROUND(9);

#endif



        for (i = 0; i < 8; ++i) {

        S->h[i] = S->h[i] ^ v[i] ^ v[i + 8];

            S->h[i] = v[i] ^= v[i + 8] ^ S->h[i];

        }



#undef G

#undef ROUND

        blocks += increment;

        len -= increment;

    } while (len);

}



/* Absorb the input data into the hash state.  Always returns 1. */
@@ -173,22 +208,41 @@ int BLAKE2s_Update(BLAKE2S_CTX *c, const void *data, size_t datalen) 
    const uint8_t *in = data;

    size_t fill;



    while(datalen > 0) {

    /*

     * Intuitively one would expect intermediate buffer, c->buf, to

     * store incomplete blocks. But in this case we are interested to

     * temporarily stash even complete blocks, because last one in the

     * stream has to be treated in special way, and at this point we

     * don't know if last block in *this* call is last one "ever". This

     * is the reason for why |datalen| is compared as >, and not >=.

     */

    fill = sizeof(c->buf) - c->buflen;

        /* Must be >, not >=, so that last block can be hashed differently */

    if (datalen > fill) {

        if (c->buflen) {

            memcpy(c->buf + c->buflen, in, fill); /* Fill buffer */

            blake2s_increment_counter(c, BLAKE2S_BLOCKBYTES);

            blake2s_compress(c, c->buf); /* Compress */

            blake2s_compress(c, c->buf, BLAKE2S_BLOCKBYTES);

            c->buflen = 0;

            in += fill;

            datalen -= fill;

        } else { /* datalen <= fill */

            memcpy(c->buf + c->buflen, in, datalen);

            c->buflen += datalen; /* Be lazy, do not compress */

            return 1;

        }

        if (datalen > BLAKE2S_BLOCKBYTES)  {

            size_t stashlen = datalen % BLAKE2S_BLOCKBYTES;

            /*

	     * If |datalen| is a multiple of the blocksize, stash

	     * last complete block, it can be final one...

             */

            stashlen = stashlen ? stashlen : BLAKE2S_BLOCKBYTES;

            datalen -= stashlen;

            blake2s_compress(c, in, datalen);

            in += datalen;

            datalen = stashlen;

        }

    }



    assert(datalen <= BLAKE2S_BLOCKBYTES);



    memcpy(c->buf + c->buflen, in, datalen);

    c->buflen += datalen; /* Be lazy, do not compress */



    return 1;

}
@@ -201,11 +255,10 @@ int BLAKE2s_Final(unsigned char *md, BLAKE2S_CTX *c) 
{

    int i;



    blake2s_increment_counter(c, (uint32_t)c->buflen);

    blake2s_set_lastblock(c);

    /* Padding */

    memset(c->buf + c->buflen, 0, sizeof(c->buf) - c->buflen);

    blake2s_compress(c, c->buf);

    blake2s_compress(c, c->buf, c->buflen);



    /* Output full hash to temp buffer */

    for (i = 0; i < 8; ++i) {

test/evptests.txt

+16 −0
Original line number Diff line number Diff line
@@ -34,6 +34,14 @@ Digest = BLAKE2s256 
Input = 3132333435363738393031323334353637383930313233343536373839303132333435363738393031323334353637383930313233343536373839303132333435363738393031323334353637383930

Output = fdaedb290a0d5af9870864fec2e090200989dc9cd53a3c092129e8535e8b4f66



Digest = BLAKE2s256

Input = 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F

Output = 1FA877DE67259D19863A2A34BCC6962A2B25FCBF5CBECD7EDE8F1FA36688A796



Digest = BLAKE2s256

Input = 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F8081

Output = C80ABEEBB669AD5DEEB5F5EC8EA6B7A05DDF7D31EC4C0A2EE20B0B98CAEC6746



Digest = BLAKE2b512

Input = 

Output = 786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce
@@ -62,6 +70,14 @@ Digest = BLAKE2b512 
Input = 3132333435363738393031323334353637383930313233343536373839303132333435363738393031323334353637383930313233343536373839303132333435363738393031323334353637383930

Output = 686f41ec5afff6e87e1f076f542aa466466ff5fbde162c48481ba48a748d842799f5b30f5b67fc684771b33b994206d05cc310f31914edd7b97e41860d77d282



Digest = BLAKE2b512

Input = 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F

Output = 2319E3789C47E2DAA5FE807F61BEC2A1A6537FA03F19FF32E87EECBFD64B7E0E8CCFF439AC333B040F19B0C4DDD11A61E24AC1FE0F10A039806C5DCC0DA3D115



Digest = BLAKE2b512

Input = 000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F202122232425262728292A2B2C2D2E2F303132333435363738393A3B3C3D3E3F404142434445464748494A4B4C4D4E4F505152535455565758595A5B5C5D5E5F606162636465666768696A6B6C6D6E6F707172737475767778797A7B7C7D7E7F8081

Output = DF0A9D0C212843A6A934E3902B2DD30D17FBA5F969D2030B12A546D8A6A45E80CF5635F071F0452E9C919275DA99BED51EB1173C1AF0518726B75B0EC3BAE2B5



# SHA(1) tests (from shatest.c)

Digest = SHA1

Input = 616263