Merge remote-tracking branch 'yawning/feature17783_take2'

src/common/crypto.c

@@ -64,6 +64,8 @@
 #include "sandbox.h"
 #include "util_format.h"
 
+#include "keccak-tiny/keccak-tiny.h"
+
 #ifdef ANDROID
 /* Android's OpenSSL seems to have removed all of its Engine support. */
 #define DISABLE_ENGINES
@@ -1616,8 +1618,11 @@ crypto_digest256(char *digest, const char *m, size_t len,
 {
   tor_assert(m);
   tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256);
-  return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  if (algorithm == DIGEST_SHA256)
+    return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
+  else
+    return (sha3_256((uint8_t *)digest, DIGEST256_LEN, (const uint8_t *)m, len) == -1);
 }
 
 /** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
@@ -1629,8 +1634,11 @@ crypto_digest512(char *digest, const char *m, size_t len,
 {
   tor_assert(m);
   tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512);
-  return (SHA512((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  if (algorithm == DIGEST_SHA512)
+    return (SHA512((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
+  else
+    return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) == -1);
 }
 
 /** Set the digests_t in <b>ds_out</b> to contain every digest on the
@@ -1646,11 +1654,13 @@ crypto_digest_all(digests_t *ds_out, const char *m, size_t len)
     return -1;
   for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) {
       switch (i) {
-        case DIGEST_SHA256:
+        case DIGEST_SHA256: /* FALLSTHROUGH */
+        case DIGEST_SHA3_256:
           if (crypto_digest256(ds_out->d[i], m, len, i) < 0)
             return -1;
           break;
         case DIGEST_SHA512:
+        case DIGEST_SHA3_512: /* FALLSTHROUGH */
           if (crypto_digest512(ds_out->d[i], m, len, i) < 0)
             return -1;
           break;
@@ -1672,6 +1682,10 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg)
       return "sha256";
     case DIGEST_SHA512:
       return "sha512";
+    case DIGEST_SHA3_256:
+      return "sha3-256";
+    case DIGEST_SHA3_512:
+      return "sha3-512";
     default:
       tor_fragile_assert();
       return "??unknown_digest??";
@@ -1689,16 +1703,41 @@ crypto_digest_algorithm_parse_name(const char *name)
     return DIGEST_SHA256;
   else if (!strcmp(name, "sha512"))
     return DIGEST_SHA512;
+  else if (!strcmp(name, "sha3-256"))
+    return DIGEST_SHA3_256;
+  else if (!strcmp(name, "sha3-512"))
+    return DIGEST_SHA3_512;
   else
     return -1;
 }
 
+/** Given an algorithm, return the digest length in bytes. */
+static inline size_t
+crypto_digest_algorithm_get_length(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return DIGEST_LEN;
+    case DIGEST_SHA256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA512:
+      return DIGEST512_LEN;
+    case DIGEST_SHA3_256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA3_512:
+      return DIGEST512_LEN;
+    default:
+      tor_assert(0);
+  }
+}
+
 /** Intermediate information about the digest of a stream of data. */
 struct crypto_digest_t {
   union {
     SHA_CTX sha1; /**< state for SHA1 */
     SHA256_CTX sha2; /**< state for SHA256 */
     SHA512_CTX sha512; /**< state for SHA512 */
+    keccak_state sha3; /**< state for SHA3-[256,512] */
   } d; /**< State for the digest we're using.  Only one member of the
         * union is usable, depending on the value of <b>algorithm</b>. */
   digest_algorithm_bitfield_t algorithm : 8; /**< Which algorithm is in use? */
@@ -1722,9 +1761,12 @@ crypto_digest_t *
 crypto_digest256_new(digest_algorithm_t algorithm)
 {
   crypto_digest_t *r;
-  tor_assert(algorithm == DIGEST_SHA256);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
   r = tor_malloc(sizeof(crypto_digest_t));
-  SHA256_Init(&r->d.sha2);
+  if (algorithm == DIGEST_SHA256)
+    SHA256_Init(&r->d.sha2);
+  else
+    keccak_digest_init(&r->d.sha3, 256);
   r->algorithm = algorithm;
   return r;
 }
@@ -1735,9 +1777,12 @@ crypto_digest_t *
 crypto_digest512_new(digest_algorithm_t algorithm)
 {
   crypto_digest_t *r;
-  tor_assert(algorithm == DIGEST_SHA512);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
   r = tor_malloc(sizeof(crypto_digest_t));
-  SHA512_Init(&r->d.sha512);
+  if (algorithm == DIGEST_SHA512)
+    SHA512_Init(&r->d.sha512);
+  else
+    keccak_digest_init(&r->d.sha3, 512);
   r->algorithm = algorithm;
   return r;
 }
@@ -1776,6 +1821,10 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
     case DIGEST_SHA512:
       SHA512_Update(&digest->d.sha512, (void*)data, len);
       break;
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
     default:
       tor_fragile_assert();
       break;
@@ -1794,26 +1843,38 @@ crypto_digest_get_digest(crypto_digest_t *digest,
   crypto_digest_t tmpenv;
   tor_assert(digest);
   tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
   /* memcpy into a temporary ctx, since SHA*_Final clears the context */
   memcpy(&tmpenv, digest, sizeof(crypto_digest_t));
   switch (digest->algorithm) {
     case DIGEST_SHA1:
-      tor_assert(out_len <= DIGEST_LEN);
       SHA1_Final(r, &tmpenv.d.sha1);
       break;
     case DIGEST_SHA256:
-      tor_assert(out_len <= DIGEST256_LEN);
       SHA256_Final(r, &tmpenv.d.sha2);
       break;
     case DIGEST_SHA512:
-      tor_assert(out_len <= DIGEST512_LEN);
       SHA512_Final(r, &tmpenv.d.sha512);
       break;
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      tor_assert(0); /* This is fatal, because it should never happen. */
     default:
       log_warn(LD_BUG, "Called with unknown algorithm %d", digest->algorithm);
       /* If fragile_assert is not enabled, then we should at least not
        * leak anything. */
       memwipe(r, 0xff, sizeof(r));
+      memwipe(&tmpenv, 0, sizeof(crypto_digest_t));
       tor_fragile_assert();
       break;
   }
@@ -1878,10 +1939,12 @@ crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
     case DIGEST_SHA1:
       d = crypto_digest_new();
       break;
-    case DIGEST_SHA256:
+    case DIGEST_SHA256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_256:
       d = crypto_digest256_new(alg);
       break;
-    case DIGEST_SHA512:
+    case DIGEST_SHA512: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
       d = crypto_digest512_new(alg);
       break;
     default:
@@ -1923,6 +1986,56 @@ crypto_hmac_sha256(char *hmac_out,
   tor_assert(rv);
 }
 
+/** Internal state for a eXtendable-Output Function (XOF). */
+struct crypto_xof_t {
+  keccak_state s;
+};
+
+/** Allocate a new XOF object backed by SHAKE-256.  The security level
+ * provided is a function of the length of the output used.  Read and
+ * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
+ * Functions" before using this construct.
+ */
+crypto_xof_t *
+crypto_xof_new(void)
+{
+  crypto_xof_t *xof;
+  xof = tor_malloc(sizeof(crypto_xof_t));
+  keccak_xof_init(&xof->s, 256);
+  return xof;
+}
+
+/** Absorb bytes into a XOF object.  Must not be called after a call to
+ * crypto_xof_squeeze_bytes() for the same instance, and will assert
+ * if attempted.
+ */
+void
+crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
+{
+  int i = keccak_xof_absorb(&xof->s, data, len);
+  tor_assert(i == 0);
+}
+
+/** Squeeze bytes out of a XOF object.  Calling this routine will render
+ * the XOF instance ineligible to absorb further data.
+ */
+void
+crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
+{
+  int i = keccak_xof_squeeze(&xof->s, out, len);
+  tor_assert(i == 0);
+}
+
+/** Cleanse and deallocate a XOF object. */
+void
+crypto_xof_free(crypto_xof_t *xof)
+{
+  if (!xof)
+    return;
+  memwipe(xof, 0, sizeof(crypto_xof_t));
+  tor_free(xof);
+}
+
 /* DH */
 
 /** Our DH 'g' parameter */

src/common/crypto.h

@@ -96,8 +96,10 @@ typedef enum {
   DIGEST_SHA1 = 0,
   DIGEST_SHA256 = 1,
   DIGEST_SHA512 = 2,
+  DIGEST_SHA3_256 = 3,
+  DIGEST_SHA3_512 = 4,
 } digest_algorithm_t;
-#define  N_DIGEST_ALGORITHMS (DIGEST_SHA512+1)
+#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
 #define digest_algorithm_bitfield_t ENUM_BF(digest_algorithm_t)
 
 /** A set of all the digests we know how to compute, taken on a single
@@ -115,6 +117,7 @@ typedef struct {
 typedef struct crypto_pk_t crypto_pk_t;
 typedef struct crypto_cipher_t crypto_cipher_t;
 typedef struct crypto_digest_t crypto_digest_t;
+typedef struct crypto_xof_t crypto_xof_t;
 typedef struct crypto_dh_t crypto_dh_t;
 
 /* global state */
@@ -244,6 +247,10 @@ void crypto_digest_assign(crypto_digest_t *into,
 void crypto_hmac_sha256(char *hmac_out,
                         const char *key, size_t key_len,
                         const char *msg, size_t msg_len);
+crypto_xof_t *crypto_xof_new(void);
+void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
+void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
+void crypto_xof_free(crypto_xof_t *xof);
 
 /* Key negotiation */
 #define DH_TYPE_CIRCUIT 1

src/ext/README

@@ -65,6 +65,10 @@ ed25519/donna/*
     Andrew Moon's semi-portable ed25519-donna implementation of
     ed25519. Public domain.
 
+keccak-tiny/
+
+    David Leon Gil's portable Keccak implementation. CC0.
+
 readpassphrase.[ch]
 
     Portable readpassphrase implementation from OpenSSH portable, version

src/ext/include.am

@@ -135,3 +135,14 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS)
 LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a
 noinst_LIBRARIES += $(LIBED25519_DONNA)
 
+src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=
+
+src_ext_keccak_tiny_libkeccak_tiny_a_SOURCES= \
+	src/ext/keccak-tiny/keccak-tiny-unrolled.c
+
+LIBKECCAK_TINY_HDRS = \
+	src/ext/keccak-tiny/keccak-tiny.h
+
+LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a
+noinst_LIBRARIES += $(LIBKECCAK_TINY)
+

src/ext/keccak-tiny/README.markdown

@@ -0,0 +1,82 @@
+# libkeccak-tiny
+
+An implementation of the FIPS-202-defined SHA-3 and SHAKE functions
+in 120 cloc (156 lines). One C file, one header.
+
+The `Keccak-f[1600]` permutation is fully unrolled; it's nearly as fast
+as the Keccak team's optimized permutation.
+
+## Building
+
+    > clang -O3 -march=native -std=c11 -Wextra -dynamic -shared keccak-tiny.c -o libkeccak-tiny.dylib
+
+If you don't have a modern libc that includes the `memset_s` function,
+you can just add `-D"memset_s(W,WL,V,OL)=memset(W,V,OL)` to the command
+line.
+
+## Using
+
+Build the library, include the header, and do, e.g.,
+
+    shake256(out, 256, in, inlen);
+
+That's it.
+
+(Note: You can request less output from the fixed-output-length
+functions, but not more.)
+
+## TweetShake
+
+The relevant tweets:
+
+```C
+// @hashbreaker Inspired by TweetNaCl!
+// Keccak and SHA-3 are supposedly hard to implement. So, how many tweets does it take to get to the center of a sponge...?
+#define decshake(bits) int shake##bits(unsigned char* o, unsigned long, unsigned char*, unsigned long);                   /*begin keccak.h*/
+#define decsha3(bits) int sha3_##bits(unsigned char*,unsigned long,unsigned char*,unsigned long);
+decshake(128) decshake(256) decsha3(224) decsha3(256) decsha3(384) decsha3(512)                                             /*end keccak.h*/
+#define K static const /* Keccak constants: rho rotations, pi lanes, and iota RCs */                                      /*begin keccak.c*/
+typedef unsigned char byte;typedef byte*bytes;typedef unsigned long z;typedef unsigned long long u8;K u8 V=1ULL<<63;K u8 W=1ULL<<31;/*!gcc*/
+#define V (1ULL<<63)
+#define W (1ULL<31)
+K byte rho[24]={1,3,6,10,15,21,28,36,45,55,2,14,27,41,56,8,25,43,62,18,39,61,20,44};K u8 RC[24]={1,0x8082,V|0x808a,V|W|0x8000,0x808b,W|1,V|W
+|0x8081,V|0x8009,138,136,W|0x8009,W|10,W|0x808b,V|0x8b,V|0x8089,V|0x8003,V|0x8002,V|0x80,0x800a,V|W|0xa,V|W|0x8081,V|0x8080,W|1,V|W|0x8008};
+K byte pi[25]={10,7,11,17,18,3,5,16,8,21,24,4,15,23,19,13,12,2,20,14,22,9,6,1}; /**helpers:*/static inline z min(z a,z b){return (a<b)?a:b;}
+#define ROL(x, s) /* rotary shift */ (((x) << s) | ((x) >> (64-s)))              /**macros to fully unroll the Keccak-f[1600] permutation:*/
+#define R24(e) /* repeat 24 times */ e e e e e e e e e e e e e e e e e e e e e e e e
+#define L5(v,s,e) /* 5-unroll a loop */ v=0; e; v+=s; e; v+=s; e; v+=s; e; v+=s; e; v+=s;                              /**the permutation:*/
+static inline void keccakf(u8* a){u8 b[5]={0};u8 t=0;byte x,y,i=0; /*24 rounds:*/R24( L5(x,1,b[x]=0;L5(y,5, /*parity*/ b[x] ^= a[x+y]))
+L5(x,1,L5(y,5,/*theta*/a[y+x] ^= b[(x+4)%5] ^ ROL(b[(x+1)%5],1))) t=a[1];x=0;R24(b[0]=a[pi[x]];/*rho*/a[pi[x]]=ROL(t, rho[x]);t=b[0];x++;)
+L5(y,5,L5(x,1, /*chi*/ b[x] = a[y+x]) L5(x,1, a[y+x] = b[x] ^ ~b[(x+1)%5] & b[(x+2)%5])) /*iota*/ a[0] ^= RC[i]; i++; )}     /**keccak-f!**/
+#define FOR(i, ST, L, S) /*obvious*/ do { for (z i = 0; i < L; i += ST) { S; } } while (0)   /**now, the sponge construction in hash mode**/
+#define appl(NAME, S) /*macro to define array comprehensions*/ static inline void NAME(bytes dst, bytes src, z len) { FOR(i, 1, len, S); }
+/*helpers:*/ static inline void clear(bytes a) { FOR(i,1,200,a[i]=0); } appl(xorin, dst[i] ^= src[i])  appl(set, src[i] = dst[i])
+#define foldP(I, L, F) /* macro to fold app P F */ while (L >= r) { /*apply F*/ F(a, I, r); /*permute*/ keccakf(A); I += r; L -= r; }
+static inline int hash(bytes o,z olen,bytes in,z ilen,z r,byte D){ if((o == (void*)0)||((in == (void*)0)&&ilen != 0)||(r >= 200))return -1;
+/*absorb*/u8 A[25]={0};bytes a=(bytes)A;/*full blocks*/foldP(in,ilen,xorin);/*last block*/xorin(a,in,ilen);/**ds+padstart*/a[ilen]^=D;
+/*padend:*/a[r-1]^=0x80; /**permute**/keccakf(A); /**squeeze:**/foldP(o,olen,set);/*last bytes*/set(a,o,olen);/*done!*/clear(a);return 0;}
+#define defshake(bits) int shake##bits(bytes o, z olen, bytes in, z ilen) {return hash(o,olen,in,ilen,200-(bits/4),0x1f);}
+#define defsha3(bits) int sha3_##bits(bytes o,z olen,bytes in,z ilen) {return hash(o,min(olen,200-(bits/4)),in,ilen,200-(bits/4),0x06);}
+/*define the SHA3 and SHAKE instances:*/defshake(128) defshake(256) defsha3(224) defsha3(256) defsha3(384) defsha3(512)/*end keccak.c*/
+// ...chomp. 24 kinda legible tweets (3232 bytes). And a simple interface: shake256(digest, digestlen, in, inlen)
+// Clang recommended. GCC users will need to insert "#define V (1ULL<<63)" and "#define W (1ULL<31)" at the point marked "/*!gcc*/"
+// If you're using as a prefix MAC, you MUST replace the body of "clear" with "memset_s(a, 200, 0, 200)" to avoid misoptimization.
+// @everyone_who_is_still_using_sha1 Please stop using SHA-1.
+// Oh, one more thing: a C11-threaded, memmapped shake256sum in 10 tweets. (Your libc may need a shim for C11 thread support.)
+// echo -n string stdio stdint fcntl sys/mman sys/stat sys/types unistd threads|tr ' ' \\n|xargs -n1 -I_ echo '#include <_.h>'
+#include "kcksum_tweet.h"
+#define E(LABEL, MSG) if (err != 0) { strerror_r(err, serr, 1024); fprintf(stderr, "%s: '%s' %s\n", serr, fn, MSG); goto LABEL;}
+static mtx_t iomtx;void h(void* v);void h(void* v){char* fn=(char*)v;int err=0;char serr[1024]={0};/*open file*/int fd=open(fn, O_RDONLY);
+err=!fd;E(ret,"couldn't be opened.");/*stat it*/struct stat stat;err=fstat(fd,&stat);E(close,"doesn't exist.");err=!!(stat.st_mode&S_IFDIR);
+E(close,"not a regular file.");z length=(size_t)stat.st_size;/*mmap the file*/bytes in=length?mmap(0,length,PROT_READ,MAP_SHARED,fd,0):NULL;
+if(length&&(in==MAP_FAILED)){E(close,"mmap-ing failed.");}byte out[64]={0};/*hash it*/shake256(out,64,in,length);length&&munmap(in,length);
+/*lock io*/mtx_lock(&iomtx);printf("SHAKE256('%s') = ", fn);FOR(i,1,64,printf("%02x",out[i]));printf("\n");mtx_unlock(&iomtx);/*unlock io*/
+close:close(fd);ret:thrd_exit(err);}int main(int argc,char** argv){int err=0; mtx_init(&iomtx, mtx_plain); thrd_t t[4]; int res[4],i,j,k;
+for(i=1;i<argc;i+=4){for(j=0;j<4;j++){if((j+i)==argc){/*out of files*/goto join;} /*spawn*/ thrd_create(t + j,h,argv[i + j]);}
+join: for (k = 0; k < j; k++) { /*wait*/ err |= thrd_join(t[k], res + k); err |= res[k];} } mtx_destroy(&iomtx); return err; } /* done! */
+```
+
+
+## License
+
+[CC0](http://creativecommons.org/publicdomain/zero/1.0/)

src/ext/keccak-tiny/do.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+cc=$(which clang-3.6||which gcc-4.9||which clang||||which gcc)
+so=$(test -f /etc/asl.conf && printf dylib|| printf so)
+$cc "-Dinline=__attribute__((__always_inline__))" -O3 -march=native -std=c11 -Wextra -Wpedantic -Wall -dynamic -shared keccak-tiny.c -o libkeccak-tiny.$so
+$cc -Os -march=native -std=c11 -Wextra -Wpedantic -Wall -dynamic -shared keccak-tiny.c -o libkeccak-tiny-small.$so

src/ext/keccak-tiny/keccak-tiny-unrolled.c

@@ -0,0 +1,368 @@
+/** libkeccak-tiny
+ *
+ * A single-file implementation of SHA-3 and SHAKE.
+ *
+ * Implementor: David Leon Gil
+ * License: CC0, attribution kindly requested. Blame taken too,
+ * but not liability.
+ */
+#include "keccak-tiny.h"
+
+#include <string.h>
+#include "crypto.h"
+
+/******** The Keccak-f[1600] permutation ********/
+
+/*** Constants. ***/
+static const uint8_t rho[24] = \
+  { 1,  3,   6, 10, 15, 21,
+    28, 36, 45, 55,  2, 14,
+    27, 41, 56,  8, 25, 43,
+    62, 18, 39, 61, 20, 44};
+static const uint8_t pi[24] = \
+  {10,  7, 11, 17, 18, 3,
+    5, 16,  8, 21, 24, 4,
+   15, 23, 19, 13, 12, 2,
+   20, 14, 22,  9, 6,  1};
+static const uint64_t RC[24] = \
+  {1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL,
+   0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL,
+   0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL,
+   0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL,
+   0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL,
+   0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL};
+
+/*** Helper macros to unroll the permutation. ***/
+#define rol(x, s) (((x) << s) | ((x) >> (64 - s)))
+#define REPEAT6(e) e e e e e e
+#define REPEAT24(e) REPEAT6(e e e e)
+#define REPEAT5(e) e e e e e
+#define FOR5(v, s, e) \
+  v = 0;            \
+  REPEAT5(e; v += s;)
+
+/*** Keccak-f[1600] ***/
+static inline void keccakf(void* state) {
+  uint64_t* a = (uint64_t*)state;
+  uint64_t b[5] = {0};
+  uint64_t t = 0;
+  uint8_t x, y, i = 0;
+
+  REPEAT24(
+      // Theta
+      FOR5(x, 1,
+           b[x] = 0;
+           FOR5(y, 5,
+                b[x] ^= a[x + y]; ))
+      FOR5(x, 1,
+           FOR5(y, 5,
+                a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); ))
+      // Rho and pi
+      t = a[1];
+      x = 0;
+      REPEAT24(b[0] = a[pi[x]];
+               a[pi[x]] = rol(t, rho[x]);
+               t = b[0];
+               x++; )
+      // Chi
+      FOR5(y,
+         5,
+         FOR5(x, 1,
+              b[x] = a[y + x];)
+         FOR5(x, 1,
+              a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); ))
+      // Iota
+      a[0] ^= RC[i];
+      i++; )
+}
+
+/******** The FIPS202-defined functions. ********/
+
+/*** Some helper macros. ***/
+
+#define _(S) do { S } while (0)
+#define FOR(i, ST, L, S) \
+  _(for (size_t i = 0; i < L; i += ST) { S; })
+#define mkapply_ds(NAME, S)                                          \
+  static inline void NAME(uint8_t* dst,                              \
+                          const uint8_t* src,                        \
+                          size_t len) {                              \
+    FOR(i, 1, len, S);                                               \
+  }
+#define mkapply_sd(NAME, S)                                          \
+  static inline void NAME(const uint8_t* src,                        \
+                          uint8_t* dst,                              \
+                          size_t len) {                              \
+    FOR(i, 1, len, S);                                               \
+  }
+
+mkapply_ds(xorin, dst[i] ^= src[i])  // xorin
+mkapply_sd(setout, dst[i] = src[i])  // setout
+
+#define P keccakf
+#define Plen KECCAK_MAX_RATE
+
+#define KECCAK_DELIM_DIGEST 0x06
+#define KECCAK_DELIM_XOF 0x1f
+
+// Fold P*F over the full blocks of an input.
+#define foldP(I, L, F) \
+  while (L >= s->rate) {  \
+    F(s->a, I, s->rate);  \
+    P(s->a);              \
+    I += s->rate;         \
+    L -= s->rate;         \
+  }
+
+static inline void
+keccak_absorb_blocks(keccak_state *s, const uint8_t *buf, size_t nr_blocks)
+{
+  size_t blen = nr_blocks * s->rate;
+  foldP(buf, blen, xorin);
+}
+
+static int
+keccak_update(keccak_state *s, const uint8_t *buf, size_t len)
+{
+  if (s->finalized)
+    return -1;
+  if ((buf == NULL) && len != 0)
+    return -1;
+
+  size_t remaining = len;
+  while (remaining > 0) {
+    if (s->offset == 0) {
+      const size_t blocks = remaining / s->rate;
+      size_t direct_bytes = blocks * s->rate;
+      if (direct_bytes > 0) {
+        keccak_absorb_blocks(s, buf, blocks);
+        remaining -= direct_bytes;
+        buf += direct_bytes;
+      }
+    }
+
+    const size_t buf_avail = s->rate - s->offset;
+    const size_t buf_bytes = (buf_avail > remaining) ? remaining : buf_avail;
+    if (buf_bytes > 0) {
+      memcpy(&s->block[s->offset], buf, buf_bytes);
+      s->offset += buf_bytes;
+      remaining -= buf_bytes;
+      buf += buf_bytes;
+    }
+    if (s->offset == s->rate) {
+      keccak_absorb_blocks(s, s->block, 1);
+      s->offset = 0;
+    }
+  }
+  return 0;
+}
+
+static void
+keccak_finalize(keccak_state *s)
+{
+  // Xor in the DS and pad frame.
+  s->a[s->offset] ^= s->delim;
+  s->a[s->rate - 1] ^= 0x80;
+  // Xor in the last block.
+  xorin(s->a, s->block, s->offset);
+
+  memwipe(s->block, 0, sizeof(s->block));
+  s->finalized = 1;
+  s->offset = s->rate;
+}
+
+static inline void
+keccak_squeeze_blocks(keccak_state *s, uint8_t *out, size_t nr_blocks)
+{
+  for (size_t n = 0; n < nr_blocks; n++) {
+    keccakf(s->a);
+    setout(s->a, out, s->rate);
+    out += s->rate;
+  }
+}
+
+static int
+keccak_squeeze(keccak_state *s, uint8_t *out, size_t outlen)
+{
+  if (!s->finalized)
+    return -1;
+
+  size_t remaining = outlen;
+  while (remaining > 0) {
+    if (s->offset == s->rate) {
+      const size_t blocks = remaining / s->rate;
+      const size_t direct_bytes = blocks * s->rate;
+      if (blocks > 0) {
+        keccak_squeeze_blocks(s, out, blocks);
+        out += direct_bytes;
+        remaining -= direct_bytes;
+      }
+
+      if (remaining > 0) {
+        keccak_squeeze_blocks(s, s->block, 1);
+        s->offset = 0;
+      }
+    }
+
+    const size_t buf_bytes = s->rate - s->offset;
+    const size_t indirect_bytes = (buf_bytes > remaining) ? remaining : buf_bytes;
+    if (indirect_bytes > 0) {
+      memcpy(out, &s->block[s->offset], indirect_bytes);
+      out += indirect_bytes;
+      s->offset += indirect_bytes;
+      remaining -= indirect_bytes;
+    }
+  }
+  return 0;
+}
+
+int
+keccak_digest_init(keccak_state *s, size_t bits)
+{
+  if (s == NULL)
+    return -1;
+  if (bits != 224 && bits != 256 && bits != 384 && bits != 512)
+    return -1;
+
+  keccak_cleanse(s);
+  s->rate = KECCAK_RATE(bits);
+  s->delim = KECCAK_DELIM_DIGEST;
+  return 0;
+}
+
+int
+keccak_digest_update(keccak_state *s, const uint8_t *buf, size_t len)
+{
+  if (s == NULL)
+    return -1;
+  if (s->delim != KECCAK_DELIM_DIGEST)
+    return -1;
+
+  return keccak_update(s, buf, len);
+}
+
+int
+keccak_digest_sum(const keccak_state *s, uint8_t *out, size_t outlen)
+{
+  if (s == NULL)
+    return -1;
+  if (s->delim != KECCAK_DELIM_DIGEST)
+    return -1;
+  if (out == NULL || outlen > 4 * (KECCAK_MAX_RATE - s->rate) / 8)
+    return -1;
+
+  // Work in a copy so that incremental/rolling hashes are easy.
+  keccak_state s_tmp;
+  keccak_clone(&s_tmp, s);
+  keccak_finalize(&s_tmp);
+  int ret = keccak_squeeze(&s_tmp, out, outlen);
+  keccak_cleanse(&s_tmp);
+  return ret;
+}
+
+int
+keccak_xof_init(keccak_state *s, size_t bits)
+{
+  if (s == NULL)
+    return -1;
+  if (bits != 128 && bits != 256)
+    return -1;
+
+  keccak_cleanse(s);
+  s->rate = KECCAK_RATE(bits);
+  s->delim = KECCAK_DELIM_XOF;
+  return 0;
+}
+
+int
+keccak_xof_absorb(keccak_state *s, const uint8_t *buf, size_t len)
+{
+  if (s == NULL)
+    return -1;
+  if (s->delim != KECCAK_DELIM_XOF)
+    return -1;
+
+  return keccak_update(s, buf, len);
+}
+
+int
+keccak_xof_squeeze(keccak_state *s, uint8_t *out, size_t outlen)
+{
+  if (s == NULL)
+    return -1;
+  if (s->delim != KECCAK_DELIM_XOF)
+    return -1;
+
+  if (!s->finalized)
+    keccak_finalize(s);
+
+  return keccak_squeeze(s, out, outlen);
+}
+
+void
+keccak_clone(keccak_state *out, const keccak_state *in)
+{
+  memcpy(out, in, sizeof(keccak_state));
+}
+
+void
+keccak_cleanse(keccak_state *s)
+{
+  memwipe(s, 0, sizeof(keccak_state));
+}
+
+/** The sponge-based hash construction. **/
+static inline int hash(uint8_t* out, size_t outlen,
+                       const uint8_t* in, size_t inlen,
+                       size_t bits, uint8_t delim) {
+  if ((out == NULL) || ((in == NULL) && inlen != 0)) {
+    return -1;
+  }
+
+  int ret = 0;
+  keccak_state s;
+
+  switch (delim) {
+    case KECCAK_DELIM_DIGEST:
+      ret |= keccak_digest_init(&s, bits);
+      ret |= keccak_digest_update(&s, in, inlen);
+      // Use the internal API instead of sum to avoid the memcpy.
+      keccak_finalize(&s);
+      ret |= keccak_squeeze(&s, out, outlen);
+      break;
+    case KECCAK_DELIM_XOF:
+      ret |= keccak_xof_init(&s, bits);
+      ret |= keccak_xof_absorb(&s, in, inlen);
+      ret |= keccak_xof_squeeze(&s, out, outlen);
+      break;
+    default:
+      return -1;
+  }
+  keccak_cleanse(&s);
+  return ret;
+}
+
+/*** Helper macros to define SHA3 and SHAKE instances. ***/
+#define defshake(bits)                                            \
+  int shake##bits(uint8_t* out, size_t outlen,                    \
+                  const uint8_t* in, size_t inlen) {              \
+    return hash(out, outlen, in, inlen, bits, KECCAK_DELIM_XOF);  \
+  }
+#define defsha3(bits)                                             \
+  int sha3_##bits(uint8_t* out, size_t outlen,                    \
+                  const uint8_t* in, size_t inlen) {              \
+    if (outlen > (bits/8)) {                                      \
+      return -1;                                                  \
+    }                                                             \
+    return hash(out, outlen, in, inlen, bits, KECCAK_DELIM_DIGEST);  \
+  }
+
+/*** FIPS202 SHAKE VOFs ***/
+defshake(128)
+defshake(256)
+
+/*** FIPS202 SHA3 FOFs ***/
+defsha3(224)
+defsha3(256)
+defsha3(384)
+defsha3(512)

src/ext/keccak-tiny/keccak-tiny.c

@@ -0,0 +1,163 @@
+/** libkeccak-tiny
+ *
+ * A single-file implementation of SHA-3 and SHAKE.
+ *
+ * Implementor: David Leon Gil
+ * License: CC0, attribution kindly requested. Blame taken too,
+ * but not liability.
+ */
+#include "keccak-tiny.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/******** The Keccak-f[1600] permutation ********/
+
+/*** Constants. ***/
+static const uint8_t rho[24] = \
+  { 1,  3,   6, 10, 15, 21,
+    28, 36, 45, 55,  2, 14,
+    27, 41, 56,  8, 25, 43,
+    62, 18, 39, 61, 20, 44};
+static const uint8_t pi[24] = \
+  {10,  7, 11, 17, 18, 3,
+    5, 16,  8, 21, 24, 4,
+   15, 23, 19, 13, 12, 2,
+   20, 14, 22,  9, 6,  1};
+static const uint64_t RC[24] = \
+  {1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL,
+   0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL,
+   0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL,
+   0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL,
+   0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL,
+   0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL};
+
+/*** Helper macros to unroll the permutation. ***/
+#define rol(x, s) (((x) << s) | ((x) >> (64 - s)))
+#define REPEAT6(e) e e e e e e
+#define REPEAT24(e) REPEAT6(e e e e)
+#define REPEAT5(e) e e e e e
+#define FOR5(v, s, e) \
+  v = 0;            \
+  REPEAT5(e; v += s;)
+
+/*** Keccak-f[1600] ***/
+static inline void keccakf(void* state) {
+  uint64_t* a = (uint64_t*)state;
+  uint64_t b[5] = {0};
+  uint64_t t = 0;
+  uint8_t x, y;
+
+  for (int i = 0; i < 24; i++) {
+    // Theta
+    FOR5(x, 1,
+         b[x] = 0;
+         FOR5(y, 5,
+              b[x] ^= a[x + y]; ))
+    FOR5(x, 1,
+         FOR5(y, 5,
+              a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); ))
+    // Rho and pi
+    t = a[1];
+    x = 0;
+    REPEAT24(b[0] = a[pi[x]];
+             a[pi[x]] = rol(t, rho[x]);
+             t = b[0];
+             x++; )
+    // Chi
+    FOR5(y,
+       5,
+       FOR5(x, 1,
+            b[x] = a[y + x];)
+       FOR5(x, 1,
+            a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); ))
+    // Iota
+    a[0] ^= RC[i];
+  }
+}
+
+/******** The FIPS202-defined functions. ********/
+
+/*** Some helper macros. ***/
+
+#define _(S) do { S } while (0)
+#define FOR(i, ST, L, S) \
+  _(for (size_t i = 0; i < L; i += ST) { S; })
+#define mkapply_ds(NAME, S)                                          \
+  static inline void NAME(uint8_t* dst,                              \
+                          const uint8_t* src,                        \
+                          size_t len) {                              \
+    FOR(i, 1, len, S);                                               \
+  }
+#define mkapply_sd(NAME, S)                                          \
+  static inline void NAME(const uint8_t* src,                        \
+                          uint8_t* dst,                              \
+                          size_t len) {                              \
+    FOR(i, 1, len, S);                                               \
+  }
+
+mkapply_ds(xorin, dst[i] ^= src[i])  // xorin
+mkapply_sd(setout, dst[i] = src[i])  // setout
+
+#define P keccakf
+#define Plen 200
+
+// Fold P*F over the full blocks of an input.
+#define foldP(I, L, F) \
+  while (L >= rate) {  \
+    F(a, I, rate);     \
+    P(a);              \
+    I += rate;         \
+    L -= rate;         \
+  }
+
+/** The sponge-based hash construction. **/
+static inline int hash(uint8_t* out, size_t outlen,
+                       const uint8_t* in, size_t inlen,
+                       size_t rate, uint8_t delim) {
+  if ((out == NULL) || ((in == NULL) && inlen != 0) || (rate >= Plen)) {
+    return -1;
+  }
+  uint8_t a[Plen] = {0};
+  // Absorb input.
+  foldP(in, inlen, xorin);
+  // Xor in the DS and pad frame.
+  a[inlen] ^= delim;
+  a[rate - 1] ^= 0x80;
+  // Xor in the last block.
+  xorin(a, in, inlen);
+  // Apply P
+  P(a);
+  // Squeeze output.
+  foldP(out, outlen, setout);
+  setout(a, out, outlen);
+  memset_s(a, 200, 0, 200);
+  return 0;
+}
+
+/*** Helper macros to define SHA3 and SHAKE instances. ***/
+#define defshake(bits)                                            \
+  int shake##bits(uint8_t* out, size_t outlen,                    \
+                  const uint8_t* in, size_t inlen) {              \
+    return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x1f);  \
+  }
+#define defsha3(bits)                                             \
+  int sha3_##bits(uint8_t* out, size_t outlen,                    \
+                  const uint8_t* in, size_t inlen) {              \
+    if (outlen > (bits/8)) {                                      \
+      return -1;                                                  \
+    }                                                             \
+    return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x06);  \
+  }
+
+/*** FIPS202 SHAKE VOFs ***/
+defshake(128)
+defshake(256)
+
+/*** FIPS202 SHA3 FOFs ***/
+defsha3(224)
+defsha3(256)
+defsha3(384)
+defsha3(512)

src/ext/keccak-tiny/keccak-tiny.h

@@ -0,0 +1,66 @@
+#ifndef KECCAK_FIPS202_H
+#define KECCAK_FIPS202_H
+
+#include <stddef.h>
+#include "torint.h"
+
+#define KECCAK_MAX_RATE 200
+
+/* Calculate the rate (block size) from the security target. */
+#define KECCAK_RATE(bits) (KECCAK_MAX_RATE - (bits / 4))
+
+/* The internal structure of a FIPS202 hash/xof instance.  Most callers
+ * should treat this as an opaque structure.
+ */
+typedef struct keccak_state {
+  uint8_t a[KECCAK_MAX_RATE];
+  size_t rate;
+  uint8_t delim;
+
+  uint8_t block[KECCAK_MAX_RATE];
+  size_t offset;
+
+  uint8_t finalized : 1;
+} keccak_state;
+
+/* Initialize a Keccak instance suitable for SHA-3 hash functions. */
+int keccak_digest_init(keccak_state *s, size_t bits);
+
+/* Feed more data into the SHA-3 hash instance. */
+int keccak_digest_update(keccak_state *s, const uint8_t *buf, size_t len);
+
+/* Calculate the SHA-3 hash digest.  The state is unmodified to support
+ * calculating multiple/rolling digests.
+ */
+int keccak_digest_sum(const keccak_state *s, uint8_t *out, size_t outlen);
+
+/* Initialize a Keccak instance suitable for XOFs (SHAKE-128/256). */
+int keccak_xof_init(keccak_state *s, size_t bits);
+
+/* Absorb more data into the XOF.  Must not be called after a squeeze call. */
+int keccak_xof_absorb(keccak_state *s, const uint8_t *buf, size_t len);
+
+/* Squeeze data out of the XOF. Must not attempt to absorb additional data,
+ * after a squeeze has been called.
+ */
+int keccak_xof_squeeze(keccak_state *s, uint8_t *out, size_t outlen);
+
+/* Clone an existing hash/XOF instance. */
+void keccak_clone(keccak_state *out, const keccak_state *in);
+
+/* Cleanse sensitive data from a given hash instance. */
+void keccak_cleanse(keccak_state *s);
+
+#define decshake(bits) \
+  int shake##bits(uint8_t*, size_t, const uint8_t*, size_t);
+
+#define decsha3(bits) \
+  int sha3_##bits(uint8_t*, size_t, const uint8_t*, size_t);
+
+decshake(128)
+decshake(256)
+decsha3(224)
+decsha3(256)
+decsha3(384)
+decsha3(512)
+#endif

src/or/include.am

@@ -109,7 +109,7 @@ src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
 src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
 src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \
-	src/common/libor-crypto.a $(LIBDONNA) \
+	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
 	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
@@ -120,7 +120,7 @@ src_or_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
 src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \
-	src/common/libor-crypto-testing.a $(LIBDONNA) \
+	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
 	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@

src/test/bench.c

@@ -442,6 +442,45 @@ bench_siphash(void)
   }
 }
 
+static void
+bench_digest(void)
+{
+  char buf[8192];
+  char out[DIGEST512_LEN];
+  const int lens[] = { 1, 16, 32, 64, 128, 512, 1024, 2048, -1 };
+  const int N = 300000;
+  uint64_t start, end;
+  crypto_rand(buf, sizeof(buf));
+
+  for (int alg = 0; alg < N_DIGEST_ALGORITHMS; alg++) {
+    for (int i = 0; lens[i] > 0; ++i) {
+      reset_perftime();
+      start = perftime();
+      for (int j = 0; j < N; ++j) {
+        switch (alg) {
+          case DIGEST_SHA1:
+            crypto_digest(out, buf, lens[i]);
+            break;
+          case DIGEST_SHA256:
+          case DIGEST_SHA3_256:
+            crypto_digest256(out, buf, lens[i], alg);
+            break;
+          case DIGEST_SHA512:
+          case DIGEST_SHA3_512:
+            crypto_digest512(out, buf, lens[i], alg);
+            break;
+          default:
+            tor_assert(0);
+        }
+      }
+      end = perftime();
+      printf("%s(%d): %.2f ns per call\n",
+             crypto_digest_algorithm_get_name(alg),
+             lens[i], NANOCOUNT(start,end,N));
+    }
+  }
+}
+
 static void
 bench_cell_ops(void)
 {
@@ -589,6 +628,7 @@ typedef struct benchmark_t {
 static struct benchmark_t benchmarks[] = {
   ENT(dmap),
   ENT(siphash),
+  ENT(digest),
   ENT(aes),
   ENT(onion_TAP),
   ENT(onion_ntor),

src/test/include.am

@@ -151,8 +151,9 @@ src_test_test_switch_id_LDADD = \
 src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
 src_test_test_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \
-	src/common/libor-crypto-testing.a $(LIBDONNA) src/common/libor.a \
-	src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \
+	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	src/common/libor.a src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@
@@ -170,7 +171,7 @@ src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS)
 src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
 src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \
-	src/common/libor-crypto.a $(LIBDONNA) \
+	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
@@ -180,7 +181,7 @@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
 src_test_test_workqueue_LDADD = src/or/libtor-testing.a \
 	src/common/libor-testing.a \
-	src/common/libor-crypto-testing.a $(LIBDONNA) \
+	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
@@ -202,7 +203,7 @@ noinst_PROGRAMS+= src/test/test-ntor-cl
 src_test_test_ntor_cl_SOURCES  = src/test/test_ntor_cl.c
 src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
-	src/common/libor-crypto.a $(LIBDONNA) \
+	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
 src_test_test_ntor_cl_AM_CPPFLAGS =	       \

src/test/test_crypto.c

@@ -448,6 +448,380 @@ test_crypto_sha(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+static void
+test_crypto_sha3(void *arg)
+{
+  crypto_digest_t *d1 = NULL, *d2 = NULL;
+  int i;
+  char data[DIGEST512_LEN];
+  char d_out1[DIGEST512_LEN], d_out2[DIGEST512_LEN];
+  char *mem_op_hex_tmp=NULL;
+  char *large = NULL;
+
+  (void)arg;
+
+  /* Test SHA3-[256,512] with a test vectors from the Keccak Code Package.
+   *
+   * NB: The code package's test vectors have length expressed in bits.
+   */
+
+  /* Len = 8, Msg = CC */
+  const uint8_t keccak_kat_msg8[] = { 0xcc };
+  i = crypto_digest256(data, (const char*)keccak_kat_msg8, 1, DIGEST_SHA3_256);
+  test_memeq_hex(data, "677035391CD3701293D385F037BA3279"
+                       "6252BB7CE180B00B582DD9B20AAAD7F0");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, (const char*)keccak_kat_msg8, 1, DIGEST_SHA3_512);
+  test_memeq_hex(data, "3939FCC8B57B63612542DA31A834E5DC"
+                       "C36E2EE0F652AC72E02624FA2E5ADEEC"
+                       "C7DD6BB3580224B4D6138706FC6E8059"
+                       "7B528051230B00621CC2B22999EAA205");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 24, Msg = 1F877C */
+  const uint8_t keccak_kat_msg24[] = { 0x1f, 0x87, 0x7c };
+  i = crypto_digest256(data, (const char*)keccak_kat_msg24, 3, DIGEST_SHA3_256);
+  test_memeq_hex(data, "BC22345E4BD3F792A341CF18AC0789F1"
+                       "C9C966712A501B19D1B6632CCD408EC5");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, (const char*)keccak_kat_msg24, 3, DIGEST_SHA3_512);
+  test_memeq_hex(data, "CB20DCF54955F8091111688BECCEF48C"
+                       "1A2F0D0608C3A575163751F002DB30F4"
+                       "0F2F671834B22D208591CFAF1F5ECFE4"
+                       "3C49863A53B3225BDFD7C6591BA7658B");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1080, Msg = B771D5CEF... ...C35AC81B5 (SHA3-256 rate - 1) */
+  const uint8_t keccak_kat_msg1080[] = {
+    0xB7, 0x71, 0xD5, 0xCE, 0xF5, 0xD1, 0xA4, 0x1A, 0x93, 0xD1,
+    0x56, 0x43, 0xD7, 0x18, 0x1D, 0x2A, 0x2E, 0xF0, 0xA8, 0xE8,
+    0x4D, 0x91, 0x81, 0x2F, 0x20, 0xED, 0x21, 0xF1, 0x47, 0xBE,
+    0xF7, 0x32, 0xBF, 0x3A, 0x60, 0xEF, 0x40, 0x67, 0xC3, 0x73,
+    0x4B, 0x85, 0xBC, 0x8C, 0xD4, 0x71, 0x78, 0x0F, 0x10, 0xDC,
+    0x9E, 0x82, 0x91, 0xB5, 0x83, 0x39, 0xA6, 0x77, 0xB9, 0x60,
+    0x21, 0x8F, 0x71, 0xE7, 0x93, 0xF2, 0x79, 0x7A, 0xEA, 0x34,
+    0x94, 0x06, 0x51, 0x28, 0x29, 0x06, 0x5D, 0x37, 0xBB, 0x55,
+    0xEA, 0x79, 0x6F, 0xA4, 0xF5, 0x6F, 0xD8, 0x89, 0x6B, 0x49,
+    0xB2, 0xCD, 0x19, 0xB4, 0x32, 0x15, 0xAD, 0x96, 0x7C, 0x71,
+    0x2B, 0x24, 0xE5, 0x03, 0x2D, 0x06, 0x52, 0x32, 0xE0, 0x2C,
+    0x12, 0x74, 0x09, 0xD2, 0xED, 0x41, 0x46, 0xB9, 0xD7, 0x5D,
+    0x76, 0x3D, 0x52, 0xDB, 0x98, 0xD9, 0x49, 0xD3, 0xB0, 0xFE,
+    0xD6, 0xA8, 0x05, 0x2F, 0xBB,
+  };
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1080, 135, DIGEST_SHA3_256);
+  test_memeq_hex(data, "A19EEE92BB2097B64E823D597798AA18"
+                       "BE9B7C736B8059ABFD6779AC35AC81B5");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1080, 135, DIGEST_SHA3_512);
+  test_memeq_hex(data, "7575A1FB4FC9A8F9C0466BD5FCA496D1"
+                       "CB78696773A212A5F62D02D14E3259D1"
+                       "92A87EBA4407DD83893527331407B6DA"
+                       "DAAD920DBC46489B677493CE5F20B595");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1088, Msg = B32D95B0... ...8E380C04 (SHA3-256 rate) */
+  const uint8_t keccak_kat_msg1088[] = {
+    0xB3, 0x2D, 0x95, 0xB0, 0xB9, 0xAA, 0xD2, 0xA8, 0x81, 0x6D,
+    0xE6, 0xD0, 0x6D, 0x1F, 0x86, 0x00, 0x85, 0x05, 0xBD, 0x8C,
+    0x14, 0x12, 0x4F, 0x6E, 0x9A, 0x16, 0x3B, 0x5A, 0x2A, 0xDE,
+    0x55, 0xF8, 0x35, 0xD0, 0xEC, 0x38, 0x80, 0xEF, 0x50, 0x70,
+    0x0D, 0x3B, 0x25, 0xE4, 0x2C, 0xC0, 0xAF, 0x05, 0x0C, 0xCD,
+    0x1B, 0xE5, 0xE5, 0x55, 0xB2, 0x30, 0x87, 0xE0, 0x4D, 0x7B,
+    0xF9, 0x81, 0x36, 0x22, 0x78, 0x0C, 0x73, 0x13, 0xA1, 0x95,
+    0x4F, 0x87, 0x40, 0xB6, 0xEE, 0x2D, 0x3F, 0x71, 0xF7, 0x68,
+    0xDD, 0x41, 0x7F, 0x52, 0x04, 0x82, 0xBD, 0x3A, 0x08, 0xD4,
+    0xF2, 0x22, 0xB4, 0xEE, 0x9D, 0xBD, 0x01, 0x54, 0x47, 0xB3,
+    0x35, 0x07, 0xDD, 0x50, 0xF3, 0xAB, 0x42, 0x47, 0xC5, 0xDE,
+    0x9A, 0x8A, 0xBD, 0x62, 0xA8, 0xDE, 0xCE, 0xA0, 0x1E, 0x3B,
+    0x87, 0xC8, 0xB9, 0x27, 0xF5, 0xB0, 0x8B, 0xEB, 0x37, 0x67,
+    0x4C, 0x6F, 0x8E, 0x38, 0x0C, 0x04,
+  };
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1088, 136, DIGEST_SHA3_256);
+  test_memeq_hex(data, "DF673F4105379FF6B755EEAB20CEB0DC"
+                       "77B5286364FE16C59CC8A907AFF07732");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1088, 136, DIGEST_SHA3_512);
+  test_memeq_hex(data, "2E293765022D48996CE8EFF0BE54E87E"
+                       "FB94A14C72DE5ACD10D0EB5ECE029CAD"
+                       "FA3BA17A40B2FFA2163991B17786E51C"
+                       "ABA79E5E0FFD34CF085E2A098BE8BACB");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1096, Msg = 04410E310... ...601016A0D (SHA3-256 rate + 1) */
+  const uint8_t keccak_kat_msg1096[] = {
+    0x04, 0x41, 0x0E, 0x31, 0x08, 0x2A, 0x47, 0x58, 0x4B, 0x40,
+    0x6F, 0x05, 0x13, 0x98, 0xA6, 0xAB, 0xE7, 0x4E, 0x4D, 0xA5,
+    0x9B, 0xB6, 0xF8, 0x5E, 0x6B, 0x49, 0xE8, 0xA1, 0xF7, 0xF2,
+    0xCA, 0x00, 0xDF, 0xBA, 0x54, 0x62, 0xC2, 0xCD, 0x2B, 0xFD,
+    0xE8, 0xB6, 0x4F, 0xB2, 0x1D, 0x70, 0xC0, 0x83, 0xF1, 0x13,
+    0x18, 0xB5, 0x6A, 0x52, 0xD0, 0x3B, 0x81, 0xCA, 0xC5, 0xEE,
+    0xC2, 0x9E, 0xB3, 0x1B, 0xD0, 0x07, 0x8B, 0x61, 0x56, 0x78,
+    0x6D, 0xA3, 0xD6, 0xD8, 0xC3, 0x30, 0x98, 0xC5, 0xC4, 0x7B,
+    0xB6, 0x7A, 0xC6, 0x4D, 0xB1, 0x41, 0x65, 0xAF, 0x65, 0xB4,
+    0x45, 0x44, 0xD8, 0x06, 0xDD, 0xE5, 0xF4, 0x87, 0xD5, 0x37,
+    0x3C, 0x7F, 0x97, 0x92, 0xC2, 0x99, 0xE9, 0x68, 0x6B, 0x7E,
+    0x58, 0x21, 0xE7, 0xC8, 0xE2, 0x45, 0x83, 0x15, 0xB9, 0x96,
+    0xB5, 0x67, 0x7D, 0x92, 0x6D, 0xAC, 0x57, 0xB3, 0xF2, 0x2D,
+    0xA8, 0x73, 0xC6, 0x01, 0x01, 0x6A, 0x0D,
+  };
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1096, 137, DIGEST_SHA3_256);
+  test_memeq_hex(data, "D52432CF3B6B4B949AA848E058DCD62D"
+                       "735E0177279222E7AC0AF8504762FAA0");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1096, 137, DIGEST_SHA3_512);
+  test_memeq_hex(data, "BE8E14B6757FFE53C9B75F6DDE9A7B6C"
+                       "40474041DE83D4A60645A826D7AF1ABE"
+                       "1EEFCB7B74B62CA6A514E5F2697D585B"
+                       "FECECE12931BBE1D4ED7EBF7B0BE660E");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1144, Msg = EA40E83C...  ...66DFAFEC (SHA3-512 rate *2 - 1) */
+  const uint8_t keccak_kat_msg1144[] = {
+    0xEA, 0x40, 0xE8, 0x3C, 0xB1, 0x8B, 0x3A, 0x24, 0x2C, 0x1E,
+    0xCC, 0x6C, 0xCD, 0x0B, 0x78, 0x53, 0xA4, 0x39, 0xDA, 0xB2,
+    0xC5, 0x69, 0xCF, 0xC6, 0xDC, 0x38, 0xA1, 0x9F, 0x5C, 0x90,
+    0xAC, 0xBF, 0x76, 0xAE, 0xF9, 0xEA, 0x37, 0x42, 0xFF, 0x3B,
+    0x54, 0xEF, 0x7D, 0x36, 0xEB, 0x7C, 0xE4, 0xFF, 0x1C, 0x9A,
+    0xB3, 0xBC, 0x11, 0x9C, 0xFF, 0x6B, 0xE9, 0x3C, 0x03, 0xE2,
+    0x08, 0x78, 0x33, 0x35, 0xC0, 0xAB, 0x81, 0x37, 0xBE, 0x5B,
+    0x10, 0xCD, 0xC6, 0x6F, 0xF3, 0xF8, 0x9A, 0x1B, 0xDD, 0xC6,
+    0xA1, 0xEE, 0xD7, 0x4F, 0x50, 0x4C, 0xBE, 0x72, 0x90, 0x69,
+    0x0B, 0xB2, 0x95, 0xA8, 0x72, 0xB9, 0xE3, 0xFE, 0x2C, 0xEE,
+    0x9E, 0x6C, 0x67, 0xC4, 0x1D, 0xB8, 0xEF, 0xD7, 0xD8, 0x63,
+    0xCF, 0x10, 0xF8, 0x40, 0xFE, 0x61, 0x8E, 0x79, 0x36, 0xDA,
+    0x3D, 0xCA, 0x5C, 0xA6, 0xDF, 0x93, 0x3F, 0x24, 0xF6, 0x95,
+    0x4B, 0xA0, 0x80, 0x1A, 0x12, 0x94, 0xCD, 0x8D, 0x7E, 0x66,
+    0xDF, 0xAF, 0xEC,
+  };
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1144, 143, DIGEST_SHA3_512);
+  test_memeq_hex(data, "3A8E938C45F3F177991296B24565D9A6"
+                       "605516615D96A062C8BE53A0D6C5A648"
+                       "7BE35D2A8F3CF6620D0C2DBA2C560D68"
+                       "295F284BE7F82F3B92919033C9CE5D80");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1144, 143, DIGEST_SHA3_256);
+  test_memeq_hex(data, "E58A947E98D6DD7E932D2FE02D9992E6"
+                       "118C0C2C606BDCDA06E7943D2C95E0E5");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1152, Msg = 157D5B7E... ...79EE00C63 (SHA3-512 rate * 2) */
+  const uint8_t keccak_kat_msg1152[] = {
+    0x15, 0x7D, 0x5B, 0x7E, 0x45, 0x07, 0xF6, 0x6D, 0x9A, 0x26,
+    0x74, 0x76, 0xD3, 0x38, 0x31, 0xE7, 0xBB, 0x76, 0x8D, 0x4D,
+    0x04, 0xCC, 0x34, 0x38, 0xDA, 0x12, 0xF9, 0x01, 0x02, 0x63,
+    0xEA, 0x5F, 0xCA, 0xFB, 0xDE, 0x25, 0x79, 0xDB, 0x2F, 0x6B,
+    0x58, 0xF9, 0x11, 0xD5, 0x93, 0xD5, 0xF7, 0x9F, 0xB0, 0x5F,
+    0xE3, 0x59, 0x6E, 0x3F, 0xA8, 0x0F, 0xF2, 0xF7, 0x61, 0xD1,
+    0xB0, 0xE5, 0x70, 0x80, 0x05, 0x5C, 0x11, 0x8C, 0x53, 0xE5,
+    0x3C, 0xDB, 0x63, 0x05, 0x52, 0x61, 0xD7, 0xC9, 0xB2, 0xB3,
+    0x9B, 0xD9, 0x0A, 0xCC, 0x32, 0x52, 0x0C, 0xBB, 0xDB, 0xDA,
+    0x2C, 0x4F, 0xD8, 0x85, 0x6D, 0xBC, 0xEE, 0x17, 0x31, 0x32,
+    0xA2, 0x67, 0x91, 0x98, 0xDA, 0xF8, 0x30, 0x07, 0xA9, 0xB5,
+    0xC5, 0x15, 0x11, 0xAE, 0x49, 0x76, 0x6C, 0x79, 0x2A, 0x29,
+    0x52, 0x03, 0x88, 0x44, 0x4E, 0xBE, 0xFE, 0x28, 0x25, 0x6F,
+    0xB3, 0x3D, 0x42, 0x60, 0x43, 0x9C, 0xBA, 0x73, 0xA9, 0x47,
+    0x9E, 0xE0, 0x0C, 0x63,
+  };
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1152, 144, DIGEST_SHA3_512);
+  test_memeq_hex(data, "FE45289874879720CE2A844AE34BB735"
+                       "22775DCB6019DCD22B8885994672A088"
+                       "9C69E8115C641DC8B83E39F7311815A1"
+                       "64DC46E0BA2FCA344D86D4BC2EF2532C");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1152, 144, DIGEST_SHA3_256);
+  test_memeq_hex(data, "A936FB9AF87FB67857B3EAD5C76226AD"
+                       "84DA47678F3C2FFE5A39FDB5F7E63FFB");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Len = 1160, Msg = 836B34B5... ...11044C53 (SHA3-512 rate * 2 + 1) */
+  const uint8_t keccak_kat_msg1160[] = {
+    0x83, 0x6B, 0x34, 0xB5, 0x15, 0x47, 0x6F, 0x61, 0x3F, 0xE4,
+    0x47, 0xA4, 0xE0, 0xC3, 0xF3, 0xB8, 0xF2, 0x09, 0x10, 0xAC,
+    0x89, 0xA3, 0x97, 0x70, 0x55, 0xC9, 0x60, 0xD2, 0xD5, 0xD2,
+    0xB7, 0x2B, 0xD8, 0xAC, 0xC7, 0x15, 0xA9, 0x03, 0x53, 0x21,
+    0xB8, 0x67, 0x03, 0xA4, 0x11, 0xDD, 0xE0, 0x46, 0x6D, 0x58,
+    0xA5, 0x97, 0x69, 0x67, 0x2A, 0xA6, 0x0A, 0xD5, 0x87, 0xB8,
+    0x48, 0x1D, 0xE4, 0xBB, 0xA5, 0x52, 0xA1, 0x64, 0x57, 0x79,
+    0x78, 0x95, 0x01, 0xEC, 0x53, 0xD5, 0x40, 0xB9, 0x04, 0x82,
+    0x1F, 0x32, 0xB0, 0xBD, 0x18, 0x55, 0xB0, 0x4E, 0x48, 0x48,
+    0xF9, 0xF8, 0xCF, 0xE9, 0xEB, 0xD8, 0x91, 0x1B, 0xE9, 0x57,
+    0x81, 0xA7, 0x59, 0xD7, 0xAD, 0x97, 0x24, 0xA7, 0x10, 0x2D,
+    0xBE, 0x57, 0x67, 0x76, 0xB7, 0xC6, 0x32, 0xBC, 0x39, 0xB9,
+    0xB5, 0xE1, 0x90, 0x57, 0xE2, 0x26, 0x55, 0x2A, 0x59, 0x94,
+    0xC1, 0xDB, 0xB3, 0xB5, 0xC7, 0x87, 0x1A, 0x11, 0xF5, 0x53,
+    0x70, 0x11, 0x04, 0x4C, 0x53,
+  };
+  i = crypto_digest512(data, (const char*)keccak_kat_msg1160, 145, DIGEST_SHA3_512);
+  test_memeq_hex(data, "AFF61C6E11B98E55AC213B1A0BC7DE04"
+                       "05221AC5EFB1229842E4614F4A029C9B"
+                       "D14A0ED7FD99AF3681429F3F309FDB53"
+                       "166AA9A3CD9F1F1223D04B4A9015E94A");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest256(data, (const char*)keccak_kat_msg1160, 145, DIGEST_SHA3_256);
+  test_memeq_hex(data, "3A654B88F88086C2751EDAE6D3924814"
+                       "3CF6235C6B0B7969342C45A35194B67E");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* SHA3-[256,512] Empty case (wikipedia) */
+  i = crypto_digest256(data, "", 0, DIGEST_SHA3_256);
+  test_memeq_hex(data, "a7ffc6f8bf1ed76651c14756a061d662"
+                       "f580ff4de43b49fa82d80a4b80f8434a");
+  tt_int_op(i, OP_EQ, 0);
+  i = crypto_digest512(data, "", 0, DIGEST_SHA3_512);
+  test_memeq_hex(data, "a69f73cca23a9ac5c8b567dc185a756e"
+                       "97c982164fe25859e0d1dcc1475c80a6"
+                       "15b2123af1f5f94c11e3e9402c3ac558"
+                       "f500199d95b6d3e301758586281dcd26");
+  tt_int_op(i, OP_EQ, 0);
+
+  /* Incremental digest code with SHA3-256 */
+  d1 = crypto_digest256_new(DIGEST_SHA3_256);
+  tt_assert(d1);
+  crypto_digest_add_bytes(d1, "abcdef", 6);
+  d2 = crypto_digest_dup(d1);
+  tt_assert(d2);
+  crypto_digest_add_bytes(d2, "ghijkl", 6);
+  crypto_digest_get_digest(d2, d_out1, DIGEST256_LEN);
+  crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA3_256);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST256_LEN);
+  crypto_digest_assign(d2, d1);
+  crypto_digest_add_bytes(d2, "mno", 3);
+  crypto_digest_get_digest(d2, d_out1, DIGEST256_LEN);
+  crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA3_256);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST256_LEN);
+  crypto_digest_get_digest(d1, d_out1, DIGEST256_LEN);
+  crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA3_256);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST256_LEN);
+  crypto_digest_free(d1);
+  crypto_digest_free(d2);
+
+  /* Incremental digest code with SHA3-512 */
+  d1 = crypto_digest512_new(DIGEST_SHA3_512);
+  tt_assert(d1);
+  crypto_digest_add_bytes(d1, "abcdef", 6);
+  d2 = crypto_digest_dup(d1);
+  tt_assert(d2);
+  crypto_digest_add_bytes(d2, "ghijkl", 6);
+  crypto_digest_get_digest(d2, d_out1, DIGEST512_LEN);
+  crypto_digest512(d_out2, "abcdefghijkl", 12, DIGEST_SHA3_512);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST512_LEN);
+  crypto_digest_assign(d2, d1);
+  crypto_digest_add_bytes(d2, "mno", 3);
+  crypto_digest_get_digest(d2, d_out1, DIGEST512_LEN);
+  crypto_digest512(d_out2, "abcdefmno", 9, DIGEST_SHA3_512);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST512_LEN);
+  crypto_digest_get_digest(d1, d_out1, DIGEST512_LEN);
+  crypto_digest512(d_out2, "abcdef", 6, DIGEST_SHA3_512);
+  tt_mem_op(d_out1,OP_EQ, d_out2, DIGEST512_LEN);
+  crypto_digest_free(d1);
+
+  /* Attempt to exercise the incremental hashing code by creating a randomized
+   * 100 KiB buffer, and hashing rand[1, 5 * Rate] bytes at a time.  SHA3-512
+   * is used because it has a lowest rate of the family (the code is common,
+   * but the slower rate exercises more of it).
+   */
+  const size_t bufsz = 100 * 1024;
+  size_t j = 0;
+  large = tor_malloc(bufsz);
+  crypto_rand(large, bufsz);
+  d1 = crypto_digest512_new(DIGEST_SHA3_512); /* Running digest. */
+  while (j < bufsz) {
+    /* Pick how much data to add to the running digest. */
+    size_t incr = (size_t)crypto_rand_int_range(1, 72 * 5);
+    incr = MIN(bufsz - j, incr);
+
+    /* Add the data, and calculate the hash. */
+    crypto_digest_add_bytes(d1, large + j, incr);
+    crypto_digest_get_digest(d1, d_out1, DIGEST512_LEN);
+
+    /* One-shot hash the buffer up to the data that was just added,
+     * and ensure that the values match up.
+     *
+     * XXX/yawning: If this actually fails, it'll be rather difficult to
+     * reproduce.  Improvements welcome.
+     */
+    i = crypto_digest512(d_out2, large, j + incr, DIGEST_SHA3_512);
+    tt_int_op(i, OP_EQ, 0);
+    tt_mem_op(d_out1, OP_EQ, d_out2, DIGEST512_LEN);
+
+    j += incr;
+  }
+
+ done:
+  if (d1)
+    crypto_digest_free(d1);
+  if (d2)
+    crypto_digest_free(d2);
+  tor_free(large);
+  tor_free(mem_op_hex_tmp);
+}
+
+/** Run unit tests for our XOF. */
+static void
+test_crypto_sha3_xof(void *arg)
+{
+  uint8_t msg[255];
+  uint8_t out[512];
+  crypto_xof_t *xof;
+  char *mem_op_hex_tmp=NULL;
+
+  (void)arg;
+
+  /* SHAKE256 test vector (Len = 2040) from the Keccak Code Package. */
+  base16_decode((char *)msg, 255,
+                "3A3A819C48EFDE2AD914FBF00E18AB6BC4F14513AB27D0C178A188B61431"
+                "E7F5623CB66B23346775D386B50E982C493ADBBFC54B9A3CD383382336A1"
+                "A0B2150A15358F336D03AE18F666C7573D55C4FD181C29E6CCFDE63EA35F"
+                "0ADF5885CFC0A3D84A2B2E4DD24496DB789E663170CEF74798AA1BBCD457"
+                "4EA0BBA40489D764B2F83AADC66B148B4A0CD95246C127D5871C4F114186"
+                "90A5DDF01246A0C80A43C70088B6183639DCFDA4125BD113A8F49EE23ED3"
+                "06FAAC576C3FB0C1E256671D817FC2534A52F5B439F72E424DE376F4C565"
+                "CCA82307DD9EF76DA5B7C4EB7E085172E328807C02D011FFBF33785378D7"
+                "9DC266F6A5BE6BB0E4A92ECEEBAEB1", 510);
+  const char *squeezed_hex =
+                "8A5199B4A7E133E264A86202720655894D48CFF344A928CF8347F48379CE"
+                "F347DFC5BCFFAB99B27B1F89AA2735E23D30088FFA03B9EDB02B9635470A"
+                "B9F1038985D55F9CA774572DD006470EA65145469609F9FA0831BF1FFD84"
+                "2DC24ACADE27BD9816E3B5BF2876CB112232A0EB4475F1DFF9F5C713D9FF"
+                "D4CCB89AE5607FE35731DF06317949EEF646E9591CF3BE53ADD6B7DD2B60"
+                "96E2B3FB06E662EC8B2D77422DAAD9463CD155204ACDBD38E319613F39F9"
+                "9B6DFB35CA9365160066DB19835888C2241FF9A731A4ACBB5663727AAC34"
+                "A401247FBAA7499E7D5EE5B69D31025E63D04C35C798BCA1262D5673A9CF"
+                "0930B5AD89BD485599DC184528DA4790F088EBD170B635D9581632D2FF90"
+                "DB79665CED430089AF13C9F21F6D443A818064F17AEC9E9C5457001FA8DC"
+                "6AFBADBE3138F388D89D0E6F22F66671255B210754ED63D81DCE75CE8F18"
+                "9B534E6D6B3539AA51E837C42DF9DF59C71E6171CD4902FE1BDC73FB1775"
+                "B5C754A1ED4EA7F3105FC543EE0418DAD256F3F6118EA77114A16C15355B"
+                "42877A1DB2A7DF0E155AE1D8670ABCEC3450F4E2EEC9838F895423EF63D2"
+                "61138BAAF5D9F104CB5A957AEA06C0B9B8C78B0D441796DC0350DDEABB78"
+                "A33B6F1F9E68EDE3D1805C7B7E2CFD54E0FAD62F0D8CA67A775DC4546AF9"
+                "096F2EDB221DB42843D65327861282DC946A0BA01A11863AB2D1DFD16E39"
+                "73D4";
+
+  /* Test oneshot absorb/squeeze. */
+  xof = crypto_xof_new();
+  tt_assert(xof);
+  crypto_xof_add_bytes(xof, msg, sizeof(msg));
+  crypto_xof_squeeze_bytes(xof, out, sizeof(out));
+  test_memeq_hex(out, squeezed_hex);
+  crypto_xof_free(xof);
+  memset(out, 0, sizeof(out));
+
+  /* Test incremental absorb/squeeze. */
+  xof = crypto_xof_new();
+  tt_assert(xof);
+  for (size_t i = 0; i < sizeof(msg); i++)
+    crypto_xof_add_bytes(xof, msg + i, 1);
+  for (size_t i = 0; i < sizeof(out); i++)
+    crypto_xof_squeeze_bytes(xof, out + i, 1);
+  test_memeq_hex(out, squeezed_hex);
+
+done:
+  if (xof)
+    crypto_xof_free(xof);
+  tor_free(mem_op_hex_tmp);
+}
+
 /** Run unit tests for our public key crypto functions */
 static void
 test_crypto_pk(void *arg)
@@ -1950,6 +2324,8 @@ struct testcase_t crypto_tests[] = {
   { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL },
   { "pk_base64", test_crypto_pk_base64, TT_FORK, NULL, NULL },
   CRYPTO_LEGACY(digests),
+  { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL},
+  { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL},
   CRYPTO_LEGACY(dh),
   { "aes_iv_AES", test_crypto_aes_iv, TT_FORK, &passthrough_setup,
     (void*)"aes" },

src/tools/include.am

@@ -20,6 +20,7 @@ endif
 src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c
 src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \
+    $(LIBKECCAK_TINY) \
     $(LIBDONNA) \
         @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
         @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
@@ -31,6 +32,7 @@ src_tools_tor_cov_gencert_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_tools_tor_cov_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \
     src/common/libor-crypto-testing.a \
+    $(LIBKECCAK_TINY) \
     $(LIBDONNA) \
         @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
         @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
@@ -39,6 +41,7 @@ endif
 src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c
 src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_tools_tor_checkkey_LDADD = src/common/libor.a src/common/libor-crypto.a \
+    $(LIBKECCAK_TINY) \
     $(LIBDONNA) \
         @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
         @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@