123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469 |
- #if !defined(MBEDTLS_CONFIG_FILE)
- #include "mbedtls/config.h"
- #else
- #include MBEDTLS_CONFIG_FILE
- #endif
- #if defined(MBEDTLS_AESNI_C)
- #include "mbedtls/aesni.h"
- #include <string.h>
- #ifndef asm
- #define asm __asm
- #endif
- #if defined(MBEDTLS_HAVE_X86_64)
- int mbedtls_aesni_has_support( unsigned int what )
- {
- #if 0
- static int done = 0;
- static unsigned int c = 0;
- if( ! done )
- {
- asm( "movl $1, %%eax \n\t"
- "cpuid \n\t"
- : "=c" (c)
- :
- : "eax", "ebx", "edx" );
- done = 1;
- }
- return( ( c & what ) != 0 );
- #else
-
- return 1;
- #endif
- }
- #define AESDEC ".byte 0x66,0x0F,0x38,0xDE,"
- #define AESDECLAST ".byte 0x66,0x0F,0x38,0xDF,"
- #define AESENC ".byte 0x66,0x0F,0x38,0xDC,"
- #define AESENCLAST ".byte 0x66,0x0F,0x38,0xDD,"
- #define AESIMC ".byte 0x66,0x0F,0x38,0xDB,"
- #define AESKEYGENA ".byte 0x66,0x0F,0x3A,0xDF,"
- #define PCLMULQDQ ".byte 0x66,0x0F,0x3A,0x44,"
- #define xmm0_xmm0 "0xC0"
- #define xmm0_xmm1 "0xC8"
- #define xmm0_xmm2 "0xD0"
- #define xmm0_xmm3 "0xD8"
- #define xmm0_xmm4 "0xE0"
- #define xmm1_xmm0 "0xC1"
- #define xmm1_xmm2 "0xD1"
- int mbedtls_aesni_crypt_ecb( mbedtls_aes_context *ctx,
- int mode,
- const unsigned char input[16],
- unsigned char output[16] )
- {
- asm( "movdqu (%3), %%xmm0 \n\t"
- "movdqu (%1), %%xmm1 \n\t"
- "pxor %%xmm1, %%xmm0 \n\t"
- "add $16, %1 \n\t"
- "subl $1, %0 \n\t"
- "test %2, %2 \n\t"
- "jz 2f \n\t"
- "1: \n\t"
- "movdqu (%1), %%xmm1 \n\t"
- AESENC xmm1_xmm0 "\n\t"
- "add $16, %1 \n\t"
- "subl $1, %0 \n\t"
- "jnz 1b \n\t"
- "movdqu (%1), %%xmm1 \n\t"
- AESENCLAST xmm1_xmm0 "\n\t"
- "jmp 3f \n\t"
- "2: \n\t"
- "movdqu (%1), %%xmm1 \n\t"
- AESDEC xmm1_xmm0 "\n\t"
- "add $16, %1 \n\t"
- "subl $1, %0 \n\t"
- "jnz 2b \n\t"
- "movdqu (%1), %%xmm1 \n\t"
- AESDECLAST xmm1_xmm0 "\n\t"
- "3: \n\t"
- "movdqu %%xmm0, (%4) \n\t"
- :
- : "r" (ctx->nr), "r" (ctx->rk), "r" (mode), "r" (input), "r" (output)
- : "memory", "cc", "xmm0", "xmm1" );
- return( 0 );
- }
- void mbedtls_aesni_gcm_mult( unsigned char c[16],
- const unsigned char a[16],
- const unsigned char b[16] )
- {
- unsigned char aa[16], bb[16], cc[16];
- size_t i;
-
- for( i = 0; i < 16; i++ )
- {
- aa[i] = a[15 - i];
- bb[i] = b[15 - i];
- }
- asm( "movdqu (%0), %%xmm0 \n\t"
- "movdqu (%1), %%xmm1 \n\t"
-
- "movdqa %%xmm1, %%xmm2 \n\t"
- "movdqa %%xmm1, %%xmm3 \n\t"
- "movdqa %%xmm1, %%xmm4 \n\t"
- PCLMULQDQ xmm0_xmm1 ",0x00 \n\t"
- PCLMULQDQ xmm0_xmm2 ",0x11 \n\t"
- PCLMULQDQ xmm0_xmm3 ",0x10 \n\t"
- PCLMULQDQ xmm0_xmm4 ",0x01 \n\t"
- "pxor %%xmm3, %%xmm4 \n\t"
- "movdqa %%xmm4, %%xmm3 \n\t"
- "psrldq $8, %%xmm4 \n\t"
- "pslldq $8, %%xmm3 \n\t"
- "pxor %%xmm4, %%xmm2 \n\t"
- "pxor %%xmm3, %%xmm1 \n\t"
-
- "movdqa %%xmm1, %%xmm3 \n\t"
- "movdqa %%xmm2, %%xmm4 \n\t"
- "psllq $1, %%xmm1 \n\t"
- "psllq $1, %%xmm2 \n\t"
- "psrlq $63, %%xmm3 \n\t"
- "psrlq $63, %%xmm4 \n\t"
- "movdqa %%xmm3, %%xmm5 \n\t"
- "pslldq $8, %%xmm3 \n\t"
- "pslldq $8, %%xmm4 \n\t"
- "psrldq $8, %%xmm5 \n\t"
- "por %%xmm3, %%xmm1 \n\t"
- "por %%xmm4, %%xmm2 \n\t"
- "por %%xmm5, %%xmm2 \n\t"
-
-
- "movdqa %%xmm1, %%xmm3 \n\t"
- "movdqa %%xmm1, %%xmm4 \n\t"
- "movdqa %%xmm1, %%xmm5 \n\t"
- "psllq $63, %%xmm3 \n\t"
- "psllq $62, %%xmm4 \n\t"
- "psllq $57, %%xmm5 \n\t"
-
- "pxor %%xmm4, %%xmm3 \n\t"
- "pxor %%xmm5, %%xmm3 \n\t"
- "pslldq $8, %%xmm3 \n\t"
- "pxor %%xmm3, %%xmm1 \n\t"
-
- "movdqa %%xmm1,%%xmm0 \n\t"
- "movdqa %%xmm1,%%xmm4 \n\t"
- "movdqa %%xmm1,%%xmm5 \n\t"
- "psrlq $1, %%xmm0 \n\t"
- "psrlq $2, %%xmm4 \n\t"
- "psrlq $7, %%xmm5 \n\t"
- "pxor %%xmm4, %%xmm0 \n\t"
- "pxor %%xmm5, %%xmm0 \n\t"
-
-
- "movdqa %%xmm1,%%xmm3 \n\t"
- "movdqa %%xmm1,%%xmm4 \n\t"
- "movdqa %%xmm1,%%xmm5 \n\t"
- "psllq $63, %%xmm3 \n\t"
- "psllq $62, %%xmm4 \n\t"
- "psllq $57, %%xmm5 \n\t"
- "pxor %%xmm4, %%xmm3 \n\t"
- "pxor %%xmm5, %%xmm3 \n\t"
- "psrldq $8, %%xmm3 \n\t"
- "pxor %%xmm3, %%xmm0 \n\t"
- "pxor %%xmm1, %%xmm0 \n\t"
- "pxor %%xmm2, %%xmm0 \n\t"
- "movdqu %%xmm0, (%2) \n\t"
- :
- : "r" (aa), "r" (bb), "r" (cc)
- : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" );
-
- for( i = 0; i < 16; i++ )
- c[i] = cc[15 - i];
- return;
- }
- void mbedtls_aesni_inverse_key( unsigned char *invkey,
- const unsigned char *fwdkey, int nr )
- {
- unsigned char *ik = invkey;
- const unsigned char *fk = fwdkey + 16 * nr;
- memcpy( ik, fk, 16 );
- for( fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16 )
- asm( "movdqu (%0), %%xmm0 \n\t"
- AESIMC xmm0_xmm0 "\n\t"
- "movdqu %%xmm0, (%1) \n\t"
- :
- : "r" (fk), "r" (ik)
- : "memory", "xmm0" );
- memcpy( ik, fk, 16 );
- }
- static void aesni_setkey_enc_128( unsigned char *rk,
- const unsigned char *key )
- {
- asm( "movdqu (%1), %%xmm0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
- "jmp 2f \n\t"
-
- "1: \n\t"
- "pshufd $0xff, %%xmm1, %%xmm1 \n\t"
- "pxor %%xmm0, %%xmm1 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm1 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm1 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm1, %%xmm0 \n\t"
- "add $16, %0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
- "ret \n\t"
-
- "2: \n\t"
- AESKEYGENA xmm0_xmm1 ",0x01 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x02 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x04 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x08 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x10 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x20 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x40 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x80 \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x1B \n\tcall 1b \n\t"
- AESKEYGENA xmm0_xmm1 ",0x36 \n\tcall 1b \n\t"
- :
- : "r" (rk), "r" (key)
- : "memory", "cc", "0" );
- }
- static void aesni_setkey_enc_192( unsigned char *rk,
- const unsigned char *key )
- {
- asm( "movdqu (%1), %%xmm0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
- "add $16, %0 \n\t"
- "movq 16(%1), %%xmm1 \n\t"
- "movq %%xmm1, (%0) \n\t"
- "add $8, %0 \n\t"
- "jmp 2f \n\t"
-
- "1: \n\t"
- "pshufd $0x55, %%xmm2, %%xmm2 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm2, %%xmm0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
- "add $16, %0 \n\t"
- "pshufd $0xff, %%xmm0, %%xmm2 \n\t"
- "pxor %%xmm1, %%xmm2 \n\t"
- "pslldq $4, %%xmm1 \n\t"
- "pxor %%xmm2, %%xmm1 \n\t"
- "movq %%xmm1, (%0) \n\t"
- "add $8, %0 \n\t"
- "ret \n\t"
- "2: \n\t"
- AESKEYGENA xmm1_xmm2 ",0x01 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x02 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x04 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x08 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x10 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x20 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x40 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x80 \n\tcall 1b \n\t"
- :
- : "r" (rk), "r" (key)
- : "memory", "cc", "0" );
- }
- static void aesni_setkey_enc_256( unsigned char *rk,
- const unsigned char *key )
- {
- asm( "movdqu (%1), %%xmm0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
- "add $16, %0 \n\t"
- "movdqu 16(%1), %%xmm1 \n\t"
- "movdqu %%xmm1, (%0) \n\t"
- "jmp 2f \n\t"
-
- "1: \n\t"
- "pshufd $0xff, %%xmm2, %%xmm2 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm0, %%xmm2 \n\t"
- "pslldq $4, %%xmm0 \n\t"
- "pxor %%xmm2, %%xmm0 \n\t"
- "add $16, %0 \n\t"
- "movdqu %%xmm0, (%0) \n\t"
-
- AESKEYGENA xmm0_xmm2 ",0x00 \n\t"
- "pshufd $0xaa, %%xmm2, %%xmm2 \n\t"
- "pxor %%xmm1, %%xmm2 \n\t"
- "pslldq $4, %%xmm1 \n\t"
- "pxor %%xmm1, %%xmm2 \n\t"
- "pslldq $4, %%xmm1 \n\t"
- "pxor %%xmm1, %%xmm2 \n\t"
- "pslldq $4, %%xmm1 \n\t"
- "pxor %%xmm2, %%xmm1 \n\t"
- "add $16, %0 \n\t"
- "movdqu %%xmm1, (%0) \n\t"
- "ret \n\t"
-
- "2: \n\t"
- AESKEYGENA xmm1_xmm2 ",0x01 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x02 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x04 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x08 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x10 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x20 \n\tcall 1b \n\t"
- AESKEYGENA xmm1_xmm2 ",0x40 \n\tcall 1b \n\t"
- :
- : "r" (rk), "r" (key)
- : "memory", "cc", "0" );
- }
- int mbedtls_aesni_setkey_enc( unsigned char *rk,
- const unsigned char *key,
- size_t bits )
- {
- switch( bits )
- {
- case 128: aesni_setkey_enc_128( rk, key ); break;
- case 192: aesni_setkey_enc_192( rk, key ); break;
- case 256: aesni_setkey_enc_256( rk, key ); break;
- default : return( MBEDTLS_ERR_AES_INVALID_KEY_LENGTH );
- }
- return( 0 );
- }
- #endif
- #endif
|