Merge branch 'microdesc'

ChangeLog

@@ -1,7 +1,25 @@
 Changes in version 0.2.2.6-alpha - 2009-10-??
+  o Major features:
+    - Directory authorities can now create, vote, and serve on multiple
+      parallel formats of directory data as part of their voting process.
+      This is a partial implementation of Proposal 162: "Publish the
+      consensus in multiple flavors."
+    - Directory authorities can now agree on and publish small summaries of
+      router information that clients can use in place of regular server
+      descriptors.  This will eventually allow clients to use far less
+      bandwidth for downloading information about the network.  This begins
+      the implementation of of Proposal 158: "Clients download a consensus +
+      Microdescriptors".
+    - The directory voting system is now extensible to use multiple hash
+      algorithms for signatures and resource selection.  Newer formats are
+      signed with SHA256, with a possibility for moving to a better hash
+      algorithm in the future.
+
   o Code simplifications and refactorings:
     - Numerous changes, bugfixes, and workarounds from Nathan Freitas
       to help Tor build correctly for Android phones.
+    - Begun converting Tor's signature and message digest logic to handle
+      multiple hash algorithms.
 
   o Minor bugfixes:
     - Fix a crash bug when trying to initialize the evdns module in

doc/spec/proposals/162-consensus-flavors.txt

@@ -72,9 +72,11 @@ Spec modifications:
    design.
 
    In addition to the consensus currently served at
-   /tor/status-vote/(current|next)/consensus.z , authorities serve
-   another consensus of each flavor "F" from the location
-   /tor/status-vote/(current|next)/F/consensus.z.
+   /tor/status-vote/(current|next)/consensus.z  and
+   /tor/status-vote/(current|next)/consensus/<FP1>+<FP2>+<FP3>+....z ,
+   authorities serve another consensus of each flavor "F" from the
+   locations /tor/status-vote/(current|next)/consensus-F.z. and
+   /tor/status-vote/(current|next)/consensus-F/<FP1>+....z.
 
    When caches serve these documents, they do so from the same
    locations.
@@ -91,9 +93,18 @@ Spec modifications:
 
    3. Document format: detached signatures.
 
-   In addition to the current detached signature format, we allow
-   the first line to take the form,
-      "consensus-digest" SP flavor SP 1*(Algname "=" Digest) NL
+   We amend the detached signature format to include more than one
+   consensus-digest line, and more than one set of signatures.
+
+   After the consensus-digest line, we allow more lines of the form:
+      "additional-digest" SP flavor SP algname SP digest NL
+
+   Before the directory-signature lines, we allow more entries of the form:
+      "additional-signature" SP flavor SP algname SP identity SP
+           signing-key-digest NL signature.
+
+   [We do not use "consensus-digest" or "directory-signature" for flavored
+   consensuses, since this could confuse older Tors.]
 
    The consensus-signatures URL should contain the signatures
    for _all_ flavors of consensus.
@@ -139,11 +150,10 @@ Spec modifications:
     4.1. The "sha256" signature format.
 
     The 'SHA256' signature format for directory objects is defined as
-    the RSA signature of the OAEP+-padded SHA256 digest of the SHA256
-    digest of the item to be signed.  When checking signatures,
-    the signature MUST be treated as valid if the signature material
-    begins with SHA256(SHA256(document)); this allows us to add other
-    data later.
+    the RSA signature of the OAEP+-padded SHA256 digest of the item to
+    be signed.  When checking signatures, the signature MUST be treated
+    as valid if the signature material begins with SHA256(document);
+    this allows us to add other data later.
 
 Considerations:
 

src/common/container.c

@@ -459,6 +459,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b))
         (int (*)(const void *,const void*))compare);
 }
 
+/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>,
+ * return the most frequent member in the list.  Break ties in favor of
+ * later elements.  If the list is empty, return NULL.
+ */
+void *
+smartlist_get_most_frequent(const smartlist_t *sl,
+                            int (*compare)(const void **a, const void **b))
+{
+  const void *most_frequent = NULL;
+  int most_frequent_count = 0;
+
+  const void **cur = NULL;
+  int i, count=0;
+
+  if (!sl->num_used)
+    return NULL;
+  for (i = 0; i < sl->num_used; ++i) {
+    const void *item = sl->list[i];
+    if (cur && 0 == compare(cur, &item)) {
+      ++count;
+    } else {
+      if (cur && count >= most_frequent_count) {
+        most_frequent = *cur;
+        most_frequent_count = count;
+      }
+      cur = &item;
+      count = 1;
+    }
+  }
+  if (cur && count >= most_frequent_count) {
+    most_frequent = *cur;
+    most_frequent_count = count;
+  }
+  return (void*)most_frequent;
+}
+
 /** Given a sorted smartlist <b>sl</b> and the comparison function used to
  * sort it, remove all duplicate members.  If free_fn is provided, calls
  * free_fn on each duplicate.  Otherwise, just removes them.  Preserves order.
@@ -550,6 +586,13 @@ smartlist_sort_strings(smartlist_t *sl)
   smartlist_sort(sl, _compare_string_ptrs);
 }
 
+/** Return the most frequent string in the sorted list <b>sl</b> */
+char *
+smartlist_get_most_frequent_string(smartlist_t *sl)
+{
+  return smartlist_get_most_frequent(sl, _compare_string_ptrs);
+}
+
 /** Remove duplicate strings from a sorted list, and free them with tor_free().
  */
 void
@@ -681,6 +724,37 @@ smartlist_uniq_digests(smartlist_t *sl)
   smartlist_uniq(sl, _compare_digests, _tor_free);
 }
 
+/** Helper: compare two DIGEST256_LEN digests. */
+static int
+_compare_digests256(const void **_a, const void **_b)
+{
+  return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN);
+}
+
+/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */
+void
+smartlist_sort_digests256(smartlist_t *sl)
+{
+  smartlist_sort(sl, _compare_digests256);
+}
+
+/** Return the most frequent member of the sorted list of DIGEST256_LEN
+ * digests in <b>sl</b> */
+char *
+smartlist_get_most_frequent_digest256(smartlist_t *sl)
+{
+  return smartlist_get_most_frequent(sl, _compare_digests256);
+}
+
+/** Remove duplicate 256-bit digests from a sorted list, and free them with
+ * tor_free().
+ */
+void
+smartlist_uniq_digests256(smartlist_t *sl)
+{
+  smartlist_uniq(sl, _compare_digests256, _tor_free);
+}
+
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
  * entry is declared using the C declaration <b>keydecl</b>.  All functions

src/common/container.h

@@ -93,13 +93,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx);
 void smartlist_insert(smartlist_t *sl, int idx, void *val);
 void smartlist_sort(smartlist_t *sl,
                     int (*compare)(const void **a, const void **b));
+void *smartlist_get_most_frequent(const smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b));
 void smartlist_uniq(smartlist_t *sl,
                     int (*compare)(const void **a, const void **b),
                     void (*free_fn)(void *elt));
+
 void smartlist_sort_strings(smartlist_t *sl);
 void smartlist_sort_digests(smartlist_t *sl);
+void smartlist_sort_digests256(smartlist_t *sl);
+
+char *smartlist_get_most_frequent_string(smartlist_t *sl);
+char *smartlist_get_most_frequent_digest256(smartlist_t *sl);
+
 void smartlist_uniq_strings(smartlist_t *sl);
 void smartlist_uniq_digests(smartlist_t *sl);
+void smartlist_uniq_digests256(smartlist_t *sl);
 void *smartlist_bsearch(smartlist_t *sl, const void *key,
                         int (*compare)(const void *key, const void **member))
   ATTR_PURE;

src/common/crypto.c

@@ -1448,6 +1448,52 @@ crypto_digest256(char *digest, const char *m, size_t len,
   return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
 }
 
+/** Set the digests_t in <b>ds_out</b> to contain every digest on the
+ * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
+ * success, -1 on failure. */
+int
+crypto_digest_all(digests_t *ds_out, const char *m, size_t len)
+{
+  digest_algorithm_t i;
+  tor_assert(ds_out);
+  memset(ds_out, 0, sizeof(*ds_out));
+  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
+    return -1;
+  for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) {
+    if (crypto_digest256(ds_out->d[i], m, len, i) < 0)
+      return -1;
+  }
+  return 0;
+}
+
+/** Return the name of an algorithm, as used in directory documents. */
+const char *
+crypto_digest_algorithm_get_name(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return "sha1";
+    case DIGEST_SHA256:
+      return "sha256";
+    default:
+      tor_fragile_assert();
+      return "??unknown_digest??";
+  }
+}
+
+/** Given the name of a digest algorithm, return its integer value, or -1 if
+ * the name is not recognized. */
+int
+crypto_digest_algorithm_parse_name(const char *name)
+{
+  if (!strcmp(name, "sha1"))
+    return DIGEST_SHA1;
+  else if (!strcmp(name, "sha256"))
+    return DIGEST_SHA256;
+  else
+    return -1;
+}
+
 /** Intermediate information about the digest of a stream of data. */
 struct crypto_digest_env_t {
   union {
@@ -2274,6 +2320,44 @@ digest_from_base64(char *digest, const char *d64)
 #endif
 }
 
+/** Base-64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the
+ * trailing = and newline characters, and store the nul-terminated result in
+ * the first BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>.  */
+int
+digest256_to_base64(char *d64, const char *digest)
+{
+  char buf[256];
+  base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN);
+  buf[BASE64_DIGEST256_LEN] = '\0';
+  memcpy(d64, buf, BASE64_DIGEST256_LEN+1);
+  return 0;
+}
+
+/** Given a base-64 encoded, nul-terminated digest in <b>d64</b> (without
+ * trailing newline or = characters), decode it and store the result in the
+ * first DIGEST256_LEN bytes at <b>digest</b>. */
+int
+digest256_from_base64(char *digest, const char *d64)
+{
+#ifdef USE_OPENSSL_BASE64
+  char buf_in[BASE64_DIGEST256_LEN+3];
+  char buf[256];
+  if (strlen(d64) != BASE64_DIGEST256_LEN)
+    return -1;
+  memcpy(buf_in, d64, BASE64_DIGEST256_LEN);
+  memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3);
+  if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN)
+    return -1;
+  memcpy(digest, buf, DIGEST256_LEN);
+  return 0;
+#else
+  if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN)
+    return 0;
+  else
+    return -1;
+#endif
+}
+
 /** Implements base32 encoding as in rfc3548.  Limitation: Requires
  * that srclen*8 is a multiple of 5.
  */

src/common/crypto.h

@@ -58,9 +58,22 @@
 #define HEX_DIGEST256_LEN 64
 
 typedef enum {
-  DIGEST_SHA1,
-  DIGEST_SHA256,
+  DIGEST_SHA1 = 0,
+  DIGEST_SHA256 = 1,
 } digest_algorithm_t;
+#define  N_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
+
+/** A set of all the digests we know how to compute, taken on a single
+ * string.  Any digests that are shorter than 256 bits are right-padded
+ * with 0 bits.
+ *
+ * Note that this representation wastes 12 bytes for the SHA1 case, so
+ * don't use it for anything where we need to allocate a whole bunch at
+ * once.
+ **/
+typedef struct {
+  char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN];
+} digests_t;
 
 typedef struct crypto_pk_env_t crypto_pk_env_t;
 typedef struct crypto_cipher_env_t crypto_cipher_env_t;
@@ -158,10 +171,13 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-/* SHA-1 */
+/* SHA-1 and other digests. */
 int crypto_digest(char *digest, const char *m, size_t len);
 int crypto_digest256(char *digest, const char *m, size_t len,
                      digest_algorithm_t algorithm);
+int crypto_digest_all(digests_t *ds_out, const char *m, size_t len);
+const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
+int crypto_digest_algorithm_parse_name(const char *name);
 crypto_digest_env_t *crypto_new_digest_env(void);
 crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm);
 void crypto_free_digest_env(crypto_digest_env_t *digest);
@@ -211,6 +227,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
 int digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
+int digest256_to_base64(char *d64, const char *digest);
+int digest256_from_base64(char *digest, const char *d64);
 
 /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the
  * 9th describes how much iteration to do. */

src/common/util.c

@@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest)
   return tor_mem_is_zero(digest, DIGEST_LEN);
 }
 
+/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
+int
+tor_digest256_is_zero(const char *digest)
+{
+  return tor_mem_is_zero(digest, DIGEST256_LEN);
+}
+
 /* Helper: common code to check whether the result of a strtol or strtoul or
  * strtoll is correct. */
 #define CHECK_STRTOX_RESULT()                           \
@@ -1729,7 +1736,8 @@ write_str_to_file(const char *fname, const char *str, int bin)
 struct open_file_t {
   char *tempname; /**< Name of the temporary file. */
   char *filename; /**< Name of the original file. */
-  int rename_on_close; /**< Are we using the temporary file or not? */
+  unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
+  unsigned binary:1; /**< Did we open in binary mode? */
   int fd; /**< fd for the open file. */
   FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
 };
@@ -1785,6 +1793,8 @@ start_writing_to_file(const char *fname, int open_flags, int mode,
     open_flags &= ~O_EXCL;
     new_file->rename_on_close = 1;
   }
+  if (open_flags & O_BINARY)
+    new_file->binary = 1;
 
   if ((new_file->fd = open(open_name, open_flags, mode)) < 0) {
     log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
@@ -1823,7 +1833,8 @@ fdopen_file(open_file_t *file_data)
   if (file_data->stdio_file)
     return file_data->stdio_file;
   tor_assert(file_data->fd >= 0);
-  if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) {
+  if (!(file_data->stdio_file = fdopen(file_data->fd,
+                                       file_data->binary?"ab":"a"))) {
     log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
              file_data->fd, strerror(errno));
   }

src/common/util.h

@@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE;
 const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE;
 int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE;
 int tor_digest_is_zero(const char *digest) ATTR_PURE;
+int tor_digest256_is_zero(const char *digest) ATTR_PURE;
 char *esc_for_log(const char *string) ATTR_MALLOC;
 const char *escaped(const char *string);
 struct smartlist_t;

src/or/Makefile.am

@@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \
 	connection.c connection_edge.c connection_or.c control.c \
 	cpuworker.c directory.c dirserv.c dirvote.c \
 	dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \
+	microdesc.c \
 	networkstatus.c onion.c policies.c \
 	reasons.c relay.c rendcommon.c rendclient.c rendmid.c \
 	rendservice.c rephist.c router.c routerlist.c routerparse.c \

src/or/control.c

@@ -1506,7 +1506,7 @@ getinfo_helper_dir(control_connection_t *control_conn,
     }
   } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
     if (directory_caches_dir_info(get_options())) {
-      const cached_dir_t *consensus = dirserv_get_consensus();
+      const cached_dir_t *consensus = dirserv_get_consensus("ns");
       if (consensus)
         *answer = tor_strdup(consensus->dir);
     }

src/or/directory.c

@@ -92,6 +92,7 @@ static void directory_initiate_command_rend(const char *address,
 #define ROUTERDESC_CACHE_LIFETIME (30*60)
 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
 #define ROBOTS_CACHE_LIFETIME (24*60*60)
+#define MICRODESC_CACHE_LIFETIME (48*60*60)
 
 /********* END VARIABLES ************/
 
@@ -610,7 +611,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn,
      * failed, and possibly retry them later.*/
     smartlist_t *failed = smartlist_create();
     dir_split_resource_into_fingerprints(conn->requested_resource+3,
-                                         failed, NULL, 0, 0);
+                                         failed, NULL, 0);
     if (smartlist_len(failed)) {
       dir_networkstatus_download_failed(failed, status_code);
       SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
@@ -647,7 +648,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status)
     return;
   failed = smartlist_create();
   dir_split_resource_into_fingerprints(conn->requested_resource+3,
-                                       failed, NULL, 1, 0);
+                                       failed, NULL, DSR_HEX);
   SMARTLIST_FOREACH(failed, char *, cp,
   {
     authority_cert_dl_failed(cp, status);
@@ -1564,7 +1565,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
       source = NS_FROM_DIR_BY_FP;
       which = smartlist_create();
       dir_split_resource_into_fingerprints(conn->requested_resource+3,
-                                           which, NULL, 0, 0);
+                                           which, NULL, 0);
     } else if (conn->requested_resource &&
                !strcmpstart(conn->requested_resource, "all")) {
       source = NS_FROM_DIR_ALL;
@@ -1623,7 +1624,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
     }
     log_info(LD_DIR,"Received consensus directory (size %d) from server "
              "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
-    if ((r=networkstatus_set_current_consensus(body, 0))<0) {
+    if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
       log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
              "Unable to load consensus directory downloaded from "
              "server '%s:%d'. I'll try again soon.",
@@ -1717,7 +1718,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
       which = smartlist_create();
       dir_split_resource_into_fingerprints(conn->requested_resource +
                                              (descriptor_digests ? 2 : 3),
-                                           which, NULL, 0, 0);
+                                           which, NULL, 0);
       n_asked_for = smartlist_len(which);
     }
     if (status_code != 200) {
@@ -2328,9 +2329,9 @@ client_likes_consensus(networkstatus_t *v, const char *want_url)
   int need_at_least;
   int have = 0;
 
-  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0);
+  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
   need_at_least = smartlist_len(want_authorities)/2+1;
-  SMARTLIST_FOREACH(want_authorities, const char *, d, {
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
     char want_digest[DIGEST_LEN];
     size_t want_len = strlen(d)/2;
     if (want_len > DIGEST_LEN)
@@ -2341,18 +2342,18 @@ client_likes_consensus(networkstatus_t *v, const char *want_url)
       continue;
     };
 
-    SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, {
-      if (vi->signature &&
+    SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
+      if (smartlist_len(vi->sigs) &&
           !memcmp(vi->identity_digest, want_digest, want_len)) {
         have++;
         break;
       };
-    });
+    } SMARTLIST_FOREACH_END(vi);
 
     /* early exit, if we already have enough */
     if (have >= need_at_least)
       break;
-  });
+  } SMARTLIST_FOREACH_END(d);
 
   SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
   smartlist_free(want_authorities);
@@ -2504,6 +2505,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     const char *request_type = NULL;
     const char *key = url + strlen("/tor/status/");
     long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
     if (!is_v3) {
       dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
       if (!strcmpstart(key, "fp/"))
@@ -2518,19 +2520,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     } else {
       networkstatus_t *v = networkstatus_get_latest_consensus();
       time_t now = time(NULL);
+      const char *want_fps = NULL;
+      char *flavor = NULL;
       #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
-      if (v &&
-          !strcmpstart(url, CONSENSUS_URL_PREFIX) &&
-          !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) {
+      #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
+      /* figure out the flavor if any, and who we wanted to sign the thing */
+      if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+        const char *f, *cp;
+        f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+        cp = strchr(f, '/');
+        if (cp) {
+          want_fps = cp+1;
+          flavor = tor_strndup(f, cp-f);
+        } else {
+          flavor = tor_strdup(f);
+        }
+      } else {
+        if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+          want_fps = url+strlen(CONSENSUS_URL_PREFIX);
+      }
+
+      /* XXXX MICRODESC NM NM should check document of correct flavor */
+      if (v && want_fps &&
+          !client_likes_consensus(v, want_fps)) {
         write_http_status_line(conn, 404, "Consensus not signed by sufficient "
                                           "number of requested authorities");
         smartlist_free(dir_fps);
         geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS);
+        tor_free(flavor);
         goto done;
       }
 
-      smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0"
-                                        "\0\0\0\0\0\0\0\0\0\0", 20));
+      {
+        char *fp = tor_malloc_zero(DIGEST_LEN);
+        if (flavor)
+          strlcpy(fp, flavor, DIGEST_LEN);
+        tor_free(flavor);
+        smartlist_add(dir_fps, fp);
+      }
       request_type = compressed?"v3.z":"v3";
       lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
     }
@@ -2618,7 +2645,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
       const char *item;
       tor_assert(!current); /* we handle current consensus specially above,
                              * since it wants to be spooled. */
-      if ((item = dirvote_get_pending_consensus()))
+      if ((item = dirvote_get_pending_consensus(FLAV_NS)))
         smartlist_add(items, (char*)item);
     } else if (!current && !strcmp(url, "consensus-signatures")) {
       /* XXXX the spec says that we should implement
@@ -2644,7 +2671,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
         flags = DGV_BY_ID |
           (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
       }
-      dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1);
+      dir_split_resource_into_fingerprints(url, fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
       SMARTLIST_FOREACH(fps, char *, fp, {
           if ((d = dirvote_get_vote(fp, flags)))
             smartlist_add(dir_items, (cached_dir_t*)d);
@@ -2697,6 +2725,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
+  if (!strcmpstart(url, "/tor/micro/d/")) {
+    smartlist_t *fps = smartlist_create();
+
+    dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
+                                      fps, NULL,
+                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
+
+    if (!dirserv_have_any_microdesc(fps)) {
+      write_http_status_line(conn, 404, "Not found");
+      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
+      smartlist_free(fps);
+      goto done;
+    }
+    dlen = dirserv_estimate_microdesc_size(fps, compressed);
+    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
+      log_info(LD_DIRSERV,
+               "Client asked for server descriptors, but we've been "
+               "writing too many bytes lately. Sending 503 Dir busy.");
+      write_http_status_line(conn, 503, "Directory busy, try again later");
+      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
+      smartlist_free(fps);
+      goto done;
+    }
+
+    write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
+    conn->dir_spool_src = DIR_SPOOL_MICRODESC;
+    conn->fingerprint_stack = fps;
+
+    if (compressed)
+      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
+
+    connection_dirserv_flushed_some(conn);
+    goto done;
+  }
+
   if (!strcmpstart(url,"/tor/server/") ||
       (!options->BridgeAuthoritativeDir &&
        !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
@@ -2778,7 +2841,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     } else if (!strcmpstart(url, "/tor/keys/fp/")) {
       smartlist_t *fps = smartlist_create();
       dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
-                                           fps, NULL, 1, 1);
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
       SMARTLIST_FOREACH(fps, char *, d, {
           authority_cert_t *c = authority_cert_get_newest_by_id(d);
           if (c) smartlist_add(certs, c);
@@ -2788,7 +2852,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
     } else if (!strcmpstart(url, "/tor/keys/sk/")) {
       smartlist_t *fps = smartlist_create();
       dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
-                                           fps, NULL, 1, 1);
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
       SMARTLIST_FOREACH(fps, char *, d, {
           authority_cert_t *c = authority_cert_get_by_sk_digest(d);
           if (c) smartlist_add(certs, c);
@@ -3523,19 +3588,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res,
 /** Given a directory <b>resource</b> request, containing zero
  * or more strings separated by plus signs, followed optionally by ".z", store
  * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
- * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.  If
- * decode_hex is true, then delete all elements that aren't hex digests, and
- * decode the rest.  If sort_uniq is true, then sort the list and remove
- * all duplicates.
+ * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
+ *
+ * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
+ * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
+ * a separator, delete all the elements that aren't base64-encoded digests,
+ * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
+ * 256 bits long; else they should be 160.
+ *
+ * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
  */
 int
 dir_split_resource_into_fingerprints(const char *resource,
                                      smartlist_t *fp_out, int *compressed_out,
-                                     int decode_hex, int sort_uniq)
+                                     int flags)
 {
+  const int decode_hex = flags & DSR_HEX;
+  const int decode_base64 = flags & DSR_BASE64;
+  const int digests_are_256 = flags & DSR_DIGEST256;
+  const int sort_uniq = flags & DSR_SORT_UNIQ;
+
+  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
+  const int hex_digest_len = digests_are_256 ?
+    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
+  const int base64_digest_len = digests_are_256 ?
+    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
   smartlist_t *fp_tmp = smartlist_create();
+
+  tor_assert(!(decode_hex && decode_base64));
   tor_assert(fp_out);
-  smartlist_split_string(fp_tmp, resource, "+", 0, 0);
+
+  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
   if (compressed_out)
     *compressed_out = 0;
   if (smartlist_len(fp_tmp)) {
@@ -3547,22 +3630,25 @@ dir_split_resource_into_fingerprints(const char *resource,
         *compressed_out = 1;
     }
   }
-  if (decode_hex) {
+  if (decode_hex || decode_base64) {
+    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
     int i;
     char *cp, *d = NULL;
     for (i = 0; i < smartlist_len(fp_tmp); ++i) {
       cp = smartlist_get(fp_tmp, i);
-      if (strlen(cp) != HEX_DIGEST_LEN) {
+      if (strlen(cp) != encoded_len) {
         log_info(LD_DIR,
                  "Skipping digest %s with non-standard length.", escaped(cp));
         smartlist_del_keeporder(fp_tmp, i--);
         goto again;
       }
-      d = tor_malloc_zero(DIGEST_LEN);
-      if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) {
-        log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
-        smartlist_del_keeporder(fp_tmp, i--);
-        goto again;
+      d = tor_malloc_zero(digest_len);
+      if (decode_hex ?
+          (base16_decode(d, digest_len, cp, hex_digest_len)<0) :
+          (base64_decode(d, digest_len, cp, base64_digest_len)<0)) {
+          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
+          smartlist_del_keeporder(fp_tmp, i--);
+          goto again;
       }
       smartlist_set(fp_tmp, i, d);
       d = NULL;
@@ -3572,26 +3658,18 @@ dir_split_resource_into_fingerprints(const char *resource,
     }
   }
   if (sort_uniq) {
-    smartlist_t *fp_tmp2 = smartlist_create();
-    int i;
-    if (decode_hex)
-      smartlist_sort_digests(fp_tmp);
-    else
+    if (decode_hex || decode_base64) {
+      if (digests_are_256) {
+        smartlist_sort_digests256(fp_tmp);
+        smartlist_uniq_digests256(fp_tmp);
+      } else {
+        smartlist_sort_digests(fp_tmp);
+        smartlist_uniq_digests(fp_tmp);
+      }
+    } else {
       smartlist_sort_strings(fp_tmp);
-    if (smartlist_len(fp_tmp))
-      smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0));
-    for (i = 1; i < smartlist_len(fp_tmp); ++i) {
-      char *cp = smartlist_get(fp_tmp, i);
-      char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1);
-
-      if ((decode_hex && memcmp(cp, last, DIGEST_LEN))
-          || (!decode_hex && strcasecmp(cp, last)))
-        smartlist_add(fp_tmp2, cp);
-      else
-        tor_free(cp);
+      smartlist_uniq_strings(fp_tmp);
     }
-    smartlist_free(fp_tmp);
-    fp_tmp = fp_tmp2;
   }
   smartlist_add_all(fp_out, fp_tmp);
   smartlist_free(fp_tmp);

src/or/dirserv.c

@@ -41,7 +41,7 @@ static time_t the_v2_networkstatus_is_dirty = 1;
 static cached_dir_t *the_directory = NULL;
 
 /** For authoritative directories: the current (v1) network status. */
-static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 };
+static cached_dir_t the_runningrouters;
 
 static void directory_remove_invalid(void);
 static cached_dir_t *dirserv_regenerate_directory(void);
@@ -1091,7 +1091,8 @@ dirserv_dump_directory_to_string(char **dir_out,
     return -1;
   }
   note_crypto_pk_op(SIGN_DIR);
-  if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) {
+  if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN,
+                                     private_key)<0) {
     tor_free(buf);
     return -1;
   }
@@ -1210,14 +1211,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now)
 static cached_dir_t *cached_directory = NULL;
 /** The v1 runningrouters document we'll serve (as a cache or as an authority)
  * if requested. */
-static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 };
+static cached_dir_t cached_runningrouters;
 
 /** Used for other dirservers' v2 network statuses.  Map from hexdigest to
  * cached_dir_t. */
 static digestmap_t *cached_v2_networkstatus = NULL;
 
-/** The v3 consensus network status that we're currently serving. */
-static cached_dir_t *cached_v3_networkstatus = NULL;
+/** Map from flavor name to the v3 consensuses that we're currently serving. */
+static strmap_t *cached_consensuses = NULL;
 
 /** Possibly replace the contents of <b>d</b> with the value of
  * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than
@@ -1385,17 +1386,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
   }
 }
 
-/** Replace the v3 consensus networkstatus that we're serving with
- * <b>networkstatus</b>, published at <b>published</b>.  No validation is
- * performed. */
+/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
+ * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
+ * validation is performed. */
 void
-dirserv_set_cached_networkstatus_v3(const char *networkstatus,
-                                    time_t published)
+dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           const char *flavor_name,
+                                           const digests_t *digests,
+                                           time_t published)
 {
-  if (cached_v3_networkstatus)
-    cached_dir_decref(cached_v3_networkstatus);
-  cached_v3_networkstatus = new_cached_dir(
-                                  tor_strdup(networkstatus), published);
+  cached_dir_t *new_networkstatus;
+  cached_dir_t *old_networkstatus;
+  if (!cached_consensuses)
+    cached_consensuses = strmap_new();
+
+  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  memcpy(&new_networkstatus->digests, digests, sizeof(digests_t));
+  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
+                                 new_networkstatus);
+  if (old_networkstatus)
+    cached_dir_decref(old_networkstatus);
 }
 
 /** Remove any v2 networkstatus from the directory cache that was published
@@ -1549,7 +1559,8 @@ generate_runningrouters(void)
     goto err;
   }
   note_crypto_pk_op(SIGN_DIR);
-  if (router_append_dirobj_signature(s, len, digest, private_key)<0)
+  if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN,
+                                     private_key)<0)
     goto err;
 
   set_cached_dir(&the_runningrouters, s, time(NULL));
@@ -1577,9 +1588,9 @@ dirserv_get_runningrouters(void)
 /** Return the latest downloaded consensus networkstatus in encoded, signed,
  * optionally compressed format, suitable for sending to clients. */
 cached_dir_t *
-dirserv_get_consensus(void)
+dirserv_get_consensus(const char *flavor_name)
 {
-  return cached_v3_networkstatus;
+  return strmap_get(cached_consensuses, flavor_name);
 }
 
 /** For authoritative directories: the current (v2) network status. */
@@ -1874,6 +1885,8 @@ version_from_platform(const char *platform)
  * The format argument has three possible values:
  *   NS_V2 - Output an entry suitable for a V2 NS opinion document
  *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
+ *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
+ *        consensus entry.
  *   NS_V3_VOTE - Output a complete V3 NS vote
  *   NS_CONTROL_PORT - Output a NS document for the control port
  */
@@ -1899,10 +1912,11 @@ routerstatus_format_entry(char *buf, size_t buf_len,
   tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
 
   r = tor_snprintf(buf, buf_len,
-                   "r %s %s %s %s %s %d %d\n",
+                   "r %s %s %s%s%s %s %d %d\n",
                    rs->nickname,
                    identity64,
-                   digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
                    published,
                    ipaddr,
                    (int)rs->or_port,
@@ -1916,7 +1930,7 @@ routerstatus_format_entry(char *buf, size_t buf_len,
    * this here, instead of in the caller. Then we could use the
    * networkstatus_type_t values, with an additional control port value
    * added -MP */
-  if (format == NS_V3_CONSENSUS)
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
     return 0;
 
   cp = buf + strlen(buf);
@@ -2432,6 +2446,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
   vote_timing_t timing;
   digestmap_t *omit_as_sybil = NULL;
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+  smartlist_t *microdescriptors = NULL;
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -2480,11 +2495,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
   omit_as_sybil = get_possible_sybil_list(routers);
 
   routerstatuses = smartlist_create();
+  microdescriptors = smartlist_create();
 
-  SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     if (ri->cache_info.published_on >= cutoff) {
       routerstatus_t *rs;
       vote_routerstatus_t *vrs;
+      microdesc_t *md;
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
@@ -2499,9 +2516,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
         rs->is_running = 0;
 
       vrs->version = version_from_platform(ri->platform);
+      md = dirvote_create_microdescriptor(ri);
+      if (md) {
+        char buf[128];
+        vote_microdesc_hash_t *h;
+        dirvote_format_microdesc_vote_line(buf, sizeof(buf), md);
+        h = tor_malloc(sizeof(vote_microdesc_hash_t));
+        h->microdesc_hash_line = tor_strdup(buf);
+        h->next = NULL;
+        vrs->microdesc = h;
+        md->last_listed = now;
+        smartlist_add(microdescriptors, md);
+      }
+
       smartlist_add(routerstatuses, vrs);
     }
-  });
+  } SMARTLIST_FOREACH_END(ri);
+
+  {
+    smartlist_t *added =
+      microdescs_add_list_to_cache(get_microdesc_cache(),
+                                   microdescriptors, SAVED_NOWHERE, 0);
+    smartlist_free(added);
+    smartlist_free(microdescriptors);
+  }
 
   smartlist_free(routers);
   digestmap_free(omit_as_sybil, NULL);
@@ -2570,12 +2608,17 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);
   memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
+  voter->sigs = smartlist_create();
+  {
+    document_signature_t *sig = tor_malloc_zero(sizeof(document_signature_t));
+    memcpy(sig->identity_digest, identity_digest, DIGEST_LEN);
+    memcpy(sig->signing_key_digest, signing_key_digest, DIGEST_LEN);
+  }
   voter->address = hostname;
   voter->addr = addr;
   voter->dir_port = options->DirPort;
   voter->or_port = options->ORPort;
   voter->contact = tor_strdup(contact);
-  memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN);
   if (options->V3AuthUseLegacyKey) {
     authority_cert_t *c = get_my_v3_legacy_cert();
     if (c) {
@@ -2743,7 +2786,8 @@ generate_v2_networkstatus_opinion(void)
   outp += strlen(outp);
 
   note_crypto_pk_op(SIGN_DIR);
-  if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) {
+  if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN,
+                                     private_key)<0) {
     log_warn(LD_BUG, "Unable to sign router status.");
     goto done;
   }
@@ -2826,7 +2870,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result,
       log_info(LD_DIRSERV,
                "Client requested 'all' network status objects; we have none.");
   } else if (!strcmpstart(key, "fp/")) {
-    dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1);
+    dir_split_resource_into_fingerprints(key+3, result, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
   }
 }
 
@@ -2891,10 +2936,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
   } else if (!strcmpstart(key, "d/")) {
     by_id = 0;
     key += strlen("d/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1);
+    dir_split_resource_into_fingerprints(key, fps_out, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
   } else if (!strcmpstart(key, "fp/")) {
     key += strlen("fp/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1);
+    dir_split_resource_into_fingerprints(key, fps_out, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
   } else {
     *msg = "Key not recognized";
     return -1;
@@ -2959,7 +3006,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
   } else if (!strcmpstart(key, "/tor/server/d/")) {
     smartlist_t *digests = smartlist_create();
     key += strlen("/tor/server/d/");
-    dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1);
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
     SMARTLIST_FOREACH(digests, const char *, d,
        {
          signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
@@ -2972,7 +3020,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
     smartlist_t *digests = smartlist_create();
     time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
     key += strlen("/tor/server/fp/");
-    dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1);
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
     SMARTLIST_FOREACH(digests, const char *, d,
        {
          if (router_digest_is_me(d)) {
@@ -3088,17 +3137,20 @@ dirserv_test_reachability(time_t now, int try_all)
     ctr = (ctr + 1) % 128;
 }
 
-/** Given a fingerprint <b>fp</b> which is either set if we're looking
- * for a v2 status, or zeroes if we're looking for a v3 status, return
- * a pointer to the appropriate cached dir object, or NULL if there isn't
- * one available. */
+/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
+ * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
+ * flavor name if we want a flavored v3 status, return a pointer to the
+ * appropriate cached dir object, or NULL if there isn't one available. */
 static cached_dir_t *
 lookup_cached_dir_by_fp(const char *fp)
 {
   cached_dir_t *d = NULL;
-  if (tor_digest_is_zero(fp) && cached_v3_networkstatus)
-    d = cached_v3_networkstatus;
-  else if (router_digest_is_me(fp) && the_v2_networkstatus)
+  if (tor_digest_is_zero(fp) && cached_consensuses)
+    d = strmap_get(cached_consensuses, "ns");
+  else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses &&
+           (d = strmap_get(cached_consensuses, fp))) {
+    /* this here interface is a nasty hack XXXX022 */;
+  } else if (router_digest_is_me(fp) && the_v2_networkstatus)
     d = the_v2_networkstatus;
   else if (cached_v2_networkstatus)
     d = digestmap_get(cached_v2_networkstatus, fp);
@@ -3184,6 +3236,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src)
   return 0;
 }
 
+/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of
+ * a microdescriptor we have. */
+int
+dirserv_have_any_microdesc(const smartlist_t *fps)
+{
+  microdesc_cache_t *cache = get_microdesc_cache();
+  SMARTLIST_FOREACH(fps, const char *, fp,
+                    if (microdesc_cache_lookup_by_digest256(cache, fp))
+                      return 1);
+  return 0;
+}
+
 /** Return an approximate estimate of the number of bytes that will
  * be needed to transmit the server descriptors (if is_serverdescs --
  * they can be either d/ or fp/ queries) or networkstatus objects (if
@@ -3215,6 +3279,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
   return result;
 }
 
+/** Given a list of microdescriptor hashes, guess how many bytes will be
+ * needed to transmit them, and return the guess. */
+size_t
+dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed)
+{
+  size_t result = smartlist_len(fps) * microdesc_average_size(NULL);
+  if (compressed)
+    result /= 2;
+  return result;
+}
+
 /** When we're spooling data onto our outbuf, add more whenever we dip
  * below this threshold. */
 #define DIRSERV_BUFFER_MIN 16384
@@ -3278,6 +3353,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
 #endif
     body = signed_descriptor_get_body(sd);
     if (conn->zlib_state) {
+      /* XXXX022 This 'last' business should actually happen on the last
+       * routerinfo, not on the last fingerprint. */
       int last = ! smartlist_len(conn->fingerprint_stack);
       connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn,
                                    last);
@@ -3301,6 +3378,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
   return 0;
 }
 
+/** Spooling helper: called when we're sending a bunch of microdescriptors,
+ * and the outbuf has become too empty. Pulls some entries from
+ * fingerprint_stack, and writes the corresponding microdescs onto outbuf.  If
+ * we run out of entries, flushes the zlib state and sets the spool source to
+ * NONE.  Returns 0 on success, negative on failure.
+ */
+static int
+connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn)
+{
+  microdesc_cache_t *cache = get_microdesc_cache();
+  while (smartlist_len(conn->fingerprint_stack) &&
+         buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) {
+    char *fp256 = smartlist_pop_last(conn->fingerprint_stack);
+    microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256);
+    tor_free(fp256);
+    if (!md)
+      continue;
+    if (conn->zlib_state) {
+      /* XXXX022 This 'last' business should actually happen on the last
+       * routerinfo, not on the last fingerprint. */
+      int last = !smartlist_len(conn->fingerprint_stack);
+      connection_write_to_buf_zlib(md->body, md->bodylen, conn, last);
+      if (last) {
+        tor_zlib_free(conn->zlib_state);
+        conn->zlib_state = NULL;
+      }
+    } else {
+      connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn));
+    }
+  }
+  if (!smartlist_len(conn->fingerprint_stack)) {
+    conn->dir_spool_src = DIR_SPOOL_NONE;
+    smartlist_free(conn->fingerprint_stack);
+    conn->fingerprint_stack = NULL;
+  }
+  return 0;
+}
+
 /** Spooling helper: Called when we're sending a directory or networkstatus,
  * and the outbuf has become too empty.  Pulls some bytes from
  * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and
@@ -3408,6 +3523,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn)
     case DIR_SPOOL_SERVER_BY_DIGEST:
     case DIR_SPOOL_SERVER_BY_FP:
       return connection_dirserv_add_servers_to_outbuf(conn);
+    case DIR_SPOOL_MICRODESC:
+      return connection_dirserv_add_microdescs_to_outbuf(conn);
     case DIR_SPOOL_CACHED_DIR:
       return connection_dirserv_add_dir_bytes_to_outbuf(conn);
     case DIR_SPOOL_NETWORKSTATUS:
@@ -3433,6 +3550,9 @@ dirserv_free_all(void)
     digestmap_free(cached_v2_networkstatus, _free_cached_dir);
     cached_v2_networkstatus = NULL;
   }
-  cached_dir_decref(cached_v3_networkstatus);
+  if (cached_consensuses) {
+    strmap_free(cached_consensuses, _free_cached_dir);
+    cached_consensuses = NULL;
+  }
 }
 

src/or/main.c

@@ -1985,6 +1985,7 @@ tor_free_all(int postfork)
   connection_free_all();
   buf_shrink_freelists(1);
   memarea_clear_freelist();
+  microdesc_free_all();
   if (!postfork) {
     config_free_all();
     router_free_all();

src/or/microdesc.c

@@ -0,0 +1,391 @@
+/* Copyright (c) 2009, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+/** A data structure to hold a bunch of cached microdescriptors.  There are
+ * two active files in the cache: a "cache file" that we mmap, and a "journal
+ * file" that we append to.  Periodically, we rebuild the cache file to hold
+ * only the microdescriptors that we want to keep */
+struct microdesc_cache_t {
+  /** Map from sha256-digest to microdesc_t for every microdesc_t in the
+   * cache. */
+  HT_HEAD(microdesc_map, microdesc_t) map;
+
+  /** Name of the cache file. */
+  char *cache_fname;
+  /** Name of the journal file. */
+  char *journal_fname;
+  /** Mmap'd contents of the cache file, or NULL if there is none. */
+  tor_mmap_t *cache_content;
+  /** Number of bytes used in the journal file. */
+  size_t journal_len;
+
+  /** Total bytes of microdescriptor bodies we have added to this cache */
+  uint64_t total_len_seen;
+  /** Total number of microdescriptors we have added to this cache */
+  unsigned n_seen;
+};
+
+/** Helper: computes a hash of <b>md</b> to place it in a hash table. */
+static INLINE unsigned int
+_microdesc_hash(microdesc_t *md)
+{
+  unsigned *d = (unsigned*)md->digest;
+#if SIZEOF_INT == 4
+  return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7];
+#else
+  return d[0] ^ d[1] ^ d[2] ^ d[3];
+#endif
+}
+
+/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */
+static INLINE int
+_microdesc_eq(microdesc_t *a, microdesc_t *b)
+{
+  return !memcmp(a->digest, b->digest, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(microdesc_map, microdesc_t, node,
+             _microdesc_hash, _microdesc_eq);
+HT_GENERATE(microdesc_map, microdesc_t, node,
+             _microdesc_hash, _microdesc_eq, 0.6,
+             _tor_malloc, _tor_realloc, _tor_free);
+
+/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
+ * On success, return the total number of bytes written, and set
+ * *<b>annotation_len_out</b> to the number of bytes written as
+ * annotations. */
+static size_t
+dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out)
+{
+  size_t r = 0;
+  /* XXXX drops unkown annotations. */
+  if (md->last_listed) {
+    char buf[ISO_TIME_LEN+1];
+    char annotation[ISO_TIME_LEN+32];
+    format_iso_time(buf, md->last_listed);
+    tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
+    fputs(annotation, f);
+    r += strlen(annotation);
+    *annotation_len_out = r;
+  } else {
+    *annotation_len_out = 0;
+  }
+
+  md->off = (off_t) ftell(f);
+  fwrite(md->body, 1, md->bodylen, f);
+  r += md->bodylen;
+  return r;
+}
+
+/** Holds a pointer to the current microdesc_cache_t object, or NULL if no
+ * such object has been allocated. */
+static microdesc_cache_t *the_microdesc_cache = NULL;
+
+/** Return a pointer to the microdescriptor cache, loading it if necessary. */
+microdesc_cache_t *
+get_microdesc_cache(void)
+{
+  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+    microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
+    HT_INIT(microdesc_map, &cache->map);
+    cache->cache_fname = get_datadir_fname("cached-microdescs");
+    cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+    microdesc_cache_reload(cache);
+    the_microdesc_cache = cache;
+  }
+  return the_microdesc_cache;
+}
+
+/* There are three sources of microdescriptors:
+   1) Generated by us while acting as a directory authority.
+   2) Loaded from the cache on disk.
+   3) Downloaded.
+*/
+
+/** Decode the microdescriptors from the string starting at <b>s</b> and
+ * ending at <b>eos</b>, and store them in <b>cache</b>.  If <b>no-save</b>,
+ * mark them as non-writable to disk.  If <b>where</b> is SAVED_IN_CACHE,
+ * leave their bodies as pointers to the mmap'd cache.  If where is
+ * <b>SAVED_NOWHERE</b>, do not allow annotations.  Return a list of the added
+ * microdescriptors.  */
+smartlist_t *
+microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save)
+{
+  /*XXXX need an argument that sets last_listed as appropriate. */
+
+  smartlist_t *descriptors, *added;
+  const int allow_annotations = (where != SAVED_NOWHERE);
+  const int copy_body = (where != SAVED_IN_CACHE);
+
+  descriptors = microdescs_parse_from_string(s, eos,
+                                             allow_annotations,
+                                             copy_body);
+
+  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
+  smartlist_free(descriptors);
+  return added;
+}
+
+/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of
+ * a string to encode.  Frees any members of <b>descriptors</b> that it does
+ * not add. */
+smartlist_t *
+microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                             smartlist_t *descriptors, saved_location_t where,
+                             int no_save)
+{
+  smartlist_t *added;
+  open_file_t *open_file = NULL;
+  FILE *f = NULL;
+  //  int n_added = 0;
+  size_t size = 0;
+
+  if (where == SAVED_NOWHERE && !no_save) {
+    f = start_writing_to_stdio_file(cache->journal_fname,
+                                    OPEN_FLAGS_APPEND|O_BINARY,
+                                    0600, &open_file);
+    if (!f) {
+      log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
+               cache->journal_fname, strerror(errno));
+      return NULL;
+    }
+  }
+
+  added = smartlist_create();
+  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+    microdesc_t *md2;
+    md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2) {
+      /* We already had this one. */
+      if (md2->last_listed < md->last_listed)
+        md2->last_listed = md->last_listed;
+      microdesc_free(md);
+      continue;
+    }
+
+    /* Okay, it's a new one. */
+    if (f) {
+      size_t annotation_len;
+      size = dump_microdescriptor(f, md, &annotation_len);
+      md->saved_location = SAVED_IN_JOURNAL;
+      cache->journal_len += size;
+    } else {
+      md->saved_location = where;
+    }
+
+    md->no_save = no_save;
+
+    HT_INSERT(microdesc_map, &cache->map, md);
+    smartlist_add(added, md);
+    ++cache->n_seen;
+    cache->total_len_seen += md->bodylen;
+  } SMARTLIST_FOREACH_END(md);
+
+  if (f)
+    finish_writing_to_file(open_file); /*XXX Check me.*/
+
+  {
+    size_t old_content_len =
+      cache->cache_content ? cache->cache_content->size : 0;
+    if (cache->journal_len > 16384 + old_content_len &&
+        cache->journal_len > old_content_len * 2) {
+      microdesc_cache_rebuild(cache);
+    }
+  }
+
+  return added;
+}
+
+/** Remove every microdescriptor in <b>cache</b>. */
+void
+microdesc_cache_clear(microdesc_cache_t *cache)
+{
+  microdesc_t **entry, **next;
+  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+    microdesc_t *md = *entry;
+    next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
+    microdesc_free(md);
+  }
+  HT_CLEAR(microdesc_map, &cache->map);
+  if (cache->cache_content) {
+    tor_munmap_file(cache->cache_content);
+    cache->cache_content = NULL;
+  }
+  cache->total_len_seen = 0;
+  cache->n_seen = 0;
+}
+
+/** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
+ * for the first time.  Return 0 on success, -1 on failure. */
+int
+microdesc_cache_reload(microdesc_cache_t *cache)
+{
+  struct stat st;
+  char *journal_content;
+  smartlist_t *added;
+  tor_mmap_t *mm;
+  int total = 0;
+
+  microdesc_cache_clear(cache);
+
+  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+  if (mm) {
+    added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+                                    SAVED_IN_CACHE, 0);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+  }
+
+  journal_content = read_file_to_str(cache->journal_fname,
+                                     RFTS_IGNORE_MISSING, &st);
+  if (journal_content) {
+    added = microdescs_add_to_cache(cache, journal_content,
+                                    journal_content+st.st_size,
+                                    SAVED_IN_JOURNAL, 0);
+    if (added) {
+      total += smartlist_len(added);
+      smartlist_free(added);
+    }
+    tor_free(journal_content);
+  }
+  log_notice(LD_DIR, "Reloaded microdescriptor cache.  Found %d descriptors.",
+             total);
+  return 0;
+}
+
+/** Regenerate the main cache file for <b>cache</b>, clear the journal file,
+ * and update every microdesc_t in the cache with pointers to its new
+ * location. */
+int
+microdesc_cache_rebuild(microdesc_cache_t *cache)
+{
+  open_file_t *open_file;
+  FILE *f;
+  microdesc_t **mdp;
+  smartlist_t *wrote;
+  size_t size;
+  off_t off = 0;
+  int orig_size, new_size;
+
+  log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+  orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
+  orig_size += (int)cache->journal_len;
+
+  f = start_writing_to_stdio_file(cache->cache_fname,
+                                  OPEN_FLAGS_REPLACE|O_BINARY,
+                                  0600, &open_file);
+  if (!f)
+    return -1;
+
+  wrote = smartlist_create();
+
+  HT_FOREACH(mdp, microdesc_map, &cache->map) {
+    microdesc_t *md = *mdp;
+    size_t annotation_len;
+    if (md->no_save)
+      continue;
+
+    size = dump_microdescriptor(f, md, &annotation_len);
+    md->off = off + annotation_len;
+    off += size;
+    if (md->saved_location != SAVED_IN_CACHE) {
+      tor_free(md->body);
+      md->saved_location = SAVED_IN_CACHE;
+    }
+    smartlist_add(wrote, md);
+  }
+
+  finish_writing_to_file(open_file); /*XXX Check me.*/
+
+  if (cache->cache_content)
+    tor_munmap_file(cache->cache_content);
+  cache->cache_content = tor_mmap_file(cache->cache_fname);
+
+  if (!cache->cache_content && smartlist_len(wrote)) {
+    log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+            cache->cache_fname);
+    smartlist_free(wrote);
+    return -1;
+  }
+  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+    tor_assert(md->saved_location == SAVED_IN_CACHE);
+    md->body = (char*)cache->cache_content->data + md->off;
+    tor_assert(!memcmp(md->body, "onion-key", 9));
+  } SMARTLIST_FOREACH_END(md);
+
+  smartlist_free(wrote);
+
+  write_str_to_file(cache->journal_fname, "", 1);
+  cache->journal_len = 0;
+
+  new_size = (int)cache->cache_content->size;
+  log_info(LD_DIR, "Done rebuilding microdesc cache. "
+           "Saved %d bytes; %d still used.",
+           orig_size-new_size, new_size);
+
+  return 0;
+}
+
+/** Deallocate a single microdescriptor.  Note: the microdescriptor MUST have
+ * previously been removed from the cache if it had ever been inserted. */
+void
+microdesc_free(microdesc_t *md)
+{
+  /* Must be removed from hash table! */
+  if (md->onion_pkey)
+    crypto_free_pk_env(md->onion_pkey);
+  if (md->body && md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  if (md->family) {
+    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
+    smartlist_free(md->family);
+  }
+  tor_free(md->exitsummary);
+
+  tor_free(md);
+}
+
+/** Free all storage held in the microdesc.c module. */
+void
+microdesc_free_all(void)
+{
+  if (the_microdesc_cache) {
+    microdesc_cache_clear(the_microdesc_cache);
+    tor_free(the_microdesc_cache->cache_fname);
+    tor_free(the_microdesc_cache->journal_fname);
+    tor_free(the_microdesc_cache);
+  }
+}
+
+/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
+ * <b>d</b>, return it.  Otherwise return NULL. */
+microdesc_t *
+microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
+{
+  microdesc_t *md, search;
+  if (!cache)
+    cache = get_microdesc_cache();
+  memcpy(search.digest, d, DIGEST256_LEN);
+  md = HT_FIND(microdesc_map, &cache->map, &search);
+  return md;
+}
+
+/** Return the mean size of decriptors added to <b>cache</b> since it was last
+ * cleared.  Used to estimate the size of large downloads. */
+size_t
+microdesc_average_size(microdesc_cache_t *cache)
+{
+  if (!cache)
+    cache = get_microdesc_cache();
+  if (!cache->n_seen)
+    return 512;
+  return (size_t)(cache->total_len_seen / cache->n_seen);
+}
+

src/or/networkstatus.c

@@ -35,16 +35,22 @@ static networkstatus_t *current_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
-static networkstatus_t *consensus_waiting_for_certs = NULL;
-/** The encoded version of consensus_waiting_for_certs. */
-static char *consensus_waiting_for_certs_body = NULL;
-/** When did we set the current value of consensus_waiting_for_certs?  If this
- * is too recent, we shouldn't try to fetch a new consensus for a little while,
- * to give ourselves time to get certificates for this one. */
-static time_t consensus_waiting_for_certs_set_at = 0;
-/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long
- * that we should treat it as maybe being bad. */
-static int consensus_waiting_for_certs_dl_failed = 0;
+typedef struct consensus_waiting_for_certs_t {
+  /** The consensus itself. */
+  networkstatus_t *consensus;
+  /** The encoded version of the consensus, nul-terminated. */
+  char *body;
+  /** When did we set the current value of consensus_waiting_for_certs?  If
+   * this is too recent, we shouldn't try to fetch a new consensus for a
+   * little while, to give ourselves time to get certificates for this one. */
+  time_t set_at;
+  /** Set to 1 if we've been holding on to it for so long we should maybe
+   * treat it as being bad. */
+  int dl_failed;
+} consensus_waiting_for_certs_t;
+
+static consensus_waiting_for_certs_t
+       consensus_waiting_for_certs[N_CONSENSUS_FLAVORS];
 
 /** The last time we tried to download a networkstatus, or 0 for "never".  We
  * use this to rate-limit download attempts for directory caches (including
@@ -56,7 +62,7 @@ static time_t last_networkstatus_download_attempted = 0;
  * before the current consensus becomes invalid. */
 static time_t time_to_download_next_consensus = 0;
 /** Download status for the current consensus networkstatus. */
-static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS };
+static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS];
 
 /** True iff we have logged a warning about this OR's version being older than
  * listed by the authorities. */
@@ -89,6 +95,7 @@ networkstatus_reset_warnings(void)
 void
 networkstatus_reset_download_failures(void)
 {
+  int i;
   const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list();
   SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns,
      SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs,
@@ -97,7 +104,8 @@ networkstatus_reset_download_failures(void)
            rs->need_to_mirror = 1;
        }));;
 
-  download_status_reset(&consensus_dl_status);
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i)
+    download_status_reset(&consensus_dl_status[i]);
   if (v2_download_status_map) {
     digestmap_iter_t *iter;
     digestmap_t *map = v2_download_status_map;
@@ -170,7 +178,7 @@ router_reload_v2_networkstatus(void)
   return 0;
 }
 
-/** Read the cached v3 consensus networkstatus from the disk. */
+/** Read every cached v3 consensus networkstatus from the disk. */
 int
 router_reload_consensus_networkstatus(void)
 {
@@ -179,31 +187,46 @@ router_reload_consensus_networkstatus(void)
   struct stat st;
   or_options_t *options = get_options();
   const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
+  int flav;
 
   /* FFFF Suppress warnings if cached consensus is bad? */
+  for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    char buf[128];
+    const char *flavor = networkstatus_get_flavor_name(flav);
+    if (flav == FLAV_NS) {
+      filename = get_datadir_fname("cached-consensus");
+    } else {
+      tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
+      filename = get_datadir_fname(buf);
+    }
+    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor, flags) < -1) {
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
+                 flavor, filename);
+      }
+      tor_free(s);
+    }
+    tor_free(filename);
 
-  filename = get_datadir_fname("cached-consensus");
-  s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
-  if (s) {
-    if (networkstatus_set_current_consensus(s, flags) < -1) {
-      log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"",
-               filename);
+    if (flav == FLAV_NS) {
+      filename = get_datadir_fname("unverified-consensus");
+    } else {
+      tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
+      filename = get_datadir_fname(buf);
     }
-    tor_free(s);
-  }
-  tor_free(filename);
 
-  filename = get_datadir_fname("unverified-consensus");
-  s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
-  if (s) {
-    if (networkstatus_set_current_consensus(s,
+    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    if (s) {
+      if (networkstatus_set_current_consensus(s, flavor,
                                      flags|NSSET_WAS_WAITING_FOR_CERTS)) {
-      log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"",
-               filename);
+      log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
+               flavor, filename);
+    }
+      tor_free(s);
     }
-    tor_free(s);
+    tor_free(filename);
   }
-  tor_free(filename);
 
   if (!current_consensus ||
       (stat(options->FallbackNetworkstatusFile, &st)==0 &&
@@ -211,7 +234,7 @@ router_reload_consensus_networkstatus(void)
     s = read_file_to_str(options->FallbackNetworkstatusFile,
                          RFTS_IGNORE_MISSING, NULL);
     if (s) {
-      if (networkstatus_set_current_consensus(s,
+      if (networkstatus_set_current_consensus(s, "ns",
                                               flags|NSSET_ACCEPT_OBSOLETE)) {
         log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"",
                  options->FallbackNetworkstatusFile);
@@ -242,8 +265,14 @@ router_reload_consensus_networkstatus(void)
 static void
 vote_routerstatus_free(vote_routerstatus_t *rs)
 {
+  vote_microdesc_hash_t *h, *next;
   tor_free(rs->version);
   tor_free(rs->status.exitsummary);
+  for (h = rs->microdesc; h; h = next) {
+    tor_free(h->microdesc_hash_line);
+    next = h->next;
+    tor_free(h);
+  }
   tor_free(rs);
 }
 
@@ -273,7 +302,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns)
   tor_free(ns);
 }
 
-/** Clear all storage held in <b>ns</b>. */
+/** Free all storage held in <b>sig</b> */
+void
+document_signature_free(document_signature_t *sig)
+{
+  tor_free(sig->signature);
+  tor_free(sig);
+}
+
+/** Return a newly allocated copy of <b>sig</b> */
+document_signature_t *
+document_signature_dup(const document_signature_t *sig)
+{
+  document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t));
+  if (r->signature)
+    r->signature = tor_memdup(sig->signature, sig->signature_len);
+  return r;
+}
+
+/** Free all storage held in <b>ns</b>. */
 void
 networkstatus_vote_free(networkstatus_t *ns)
 {
@@ -295,14 +342,17 @@ networkstatus_vote_free(networkstatus_t *ns)
     smartlist_free(ns->supported_methods);
   }
   if (ns->voters) {
-    SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter,
-    {
+    SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) {
       tor_free(voter->nickname);
       tor_free(voter->address);
       tor_free(voter->contact);
-      tor_free(voter->signature);
+      if (voter->sigs) {
+        SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                          document_signature_free(sig));
+        smartlist_free(voter->sigs);
+      }
       tor_free(voter);
-    });
+    } SMARTLIST_FOREACH_END(voter);
     smartlist_free(ns->voters);
   }
   if (ns->cert)
@@ -341,34 +391,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote,
   return NULL;
 }
 
-/** Check whether the signature on <b>voter</b> is correctly signed by
- * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the
+/** Check whether the signature <b>sig</b> is correctly signed with the
+ * signing key in <b>cert</b>.  Return -1 if <b>cert</b> doesn't match the
  * signing key; otherwise set the good_signature or bad_signature flag on
  * <b>voter</b>, and return 0. */
-/* (private; exposed for testing.) */
 int
-networkstatus_check_voter_signature(networkstatus_t *consensus,
-                                    networkstatus_voter_info_t *voter,
-                                    authority_cert_t *cert)
+networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                       document_signature_t *sig,
+                                       const authority_cert_t *cert)
 {
-  char d[DIGEST_LEN];
+  char key_digest[DIGEST_LEN];
+  const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN;
   char *signed_digest;
   size_t signed_digest_len;
-  if (crypto_pk_get_digest(cert->signing_key, d)<0)
+
+  if (crypto_pk_get_digest(cert->signing_key, key_digest)<0)
     return -1;
-  if (memcmp(voter->signing_key_digest, d, DIGEST_LEN))
+  if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) ||
+      memcmp(sig->identity_digest, cert->cache_info.identity_digest,
+             DIGEST_LEN))
     return -1;
+
   signed_digest_len = crypto_pk_keysize(cert->signing_key);
   signed_digest = tor_malloc(signed_digest_len);
   if (crypto_pk_public_checksig(cert->signing_key,
                                 signed_digest,
-                                voter->signature,
-                                voter->signature_len) != DIGEST_LEN ||
-      memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) {
+                                sig->signature,
+                                sig->signature_len) < dlen ||
+      memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) {
     log_warn(LD_DIR, "Got a bad signature on a networkstatus vote");
-    voter->bad_signature = 1;
+    sig->bad_signature = 1;
   } else {
-    voter->good_signature = 1;
+    sig->good_signature = 1;
   }
   tor_free(signed_digest);
   return 0;
@@ -401,37 +455,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus,
 
   tor_assert(consensus->type == NS_TYPE_CONSENSUS);
 
-  SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter,
-  {
-    if (!voter->good_signature && !voter->bad_signature && voter->signature) {
-      /* we can try to check the signature. */
-      int is_v3_auth = trusteddirserver_get_by_v3_auth_digest(
-                                          voter->identity_digest) != NULL;
-      authority_cert_t *cert =
-        authority_cert_get_by_digests(voter->identity_digest,
-                                      voter->signing_key_digest);
-      if (!is_v3_auth) {
-        smartlist_add(unrecognized, voter);
-        ++n_unknown;
-        continue;
-      } else if (!cert || cert->expires < now) {
-        smartlist_add(need_certs_from, voter);
-        ++n_missing_key;
-        continue;
-      }
-      if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) {
-        smartlist_add(need_certs_from, voter);
-        ++n_missing_key;
-        continue;
+  SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *,
+                          voter) {
+    int good_here = 0;
+    int bad_here = 0;
+    int missing_key_here = 0;
+    SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+      if (!sig->good_signature && !sig->bad_signature &&
+          sig->signature) {
+        /* we can try to check the signature. */
+        int is_v3_auth = trusteddirserver_get_by_v3_auth_digest(
+                                              sig->identity_digest) != NULL;
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(sig->identity_digest,
+                                        sig->signing_key_digest);
+        tor_assert(!memcmp(sig->identity_digest, voter->identity_digest,
+                           DIGEST_LEN));
+
+        if (!is_v3_auth) {
+          smartlist_add(unrecognized, voter);
+          ++n_unknown;
+          continue;
+        } else if (!cert || cert->expires < now) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          continue;
+        }
+        if (networkstatus_check_document_signature(consensus, sig, cert) < 0) {
+          smartlist_add(need_certs_from, voter);
+          ++missing_key_here;
+          continue;
+        }
       }
-    }
-    if (voter->good_signature)
+      if (sig->good_signature)
+        ++good_here;
+      else if (sig->bad_signature)
+        ++bad_here;
+    } SMARTLIST_FOREACH_END(sig);
+    if (good_here)
       ++n_good;
-    else if (voter->bad_signature)
+    else if (bad_here)
       ++n_bad;
+    else if (missing_key_here)
+      ++n_missing_key;
     else
       ++n_no_signature;
-  });
+  } SMARTLIST_FOREACH_END(voter);
 
   /* Now see whether we're missing any voters entirely. */
   SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
@@ -1077,27 +1146,32 @@ static void
 update_consensus_networkstatus_downloads(time_t now)
 {
   or_options_t *options = get_options();
+  int i;
   if (!networkstatus_get_live_consensus(now))
     time_to_download_next_consensus = now; /* No live consensus? Get one now!*/
   if (time_to_download_next_consensus > now)
     return; /* Wait until the current consensus is older. */
   if (authdir_mode_v3(options))
     return; /* Authorities never fetch a consensus */
-  if (!download_status_is_ready(&consensus_dl_status, now,
+  /* XXXXNM Microdescs: may need to download more types. */
+  if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now,
                                 CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
     return; /* We failed downloading a consensus too recently. */
   if (connection_get_by_type_purpose(CONN_TYPE_DIR,
                                      DIR_PURPOSE_FETCH_CONSENSUS))
     return; /* There's an in-progress download.*/
 
-  if (consensus_waiting_for_certs) {
-    /* XXXX make sure this doesn't delay sane downloads. */
-    if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now)
-      return; /* We're still getting certs for this one. */
-    else {
-      if (!consensus_waiting_for_certs_dl_failed) {
-        download_status_failed(&consensus_dl_status, 0);
-        consensus_waiting_for_certs_dl_failed=1;
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (waiting->consensus) {
+      /* XXXX make sure this doesn't delay sane downloads. */
+      if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now)
+        return; /* We're still getting certs for this one. */
+      else {
+        if (!waiting->dl_failed) {
+          download_status_failed(&consensus_dl_status[FLAV_NS], 0);
+          waiting->dl_failed=1;
+        }
       }
     }
   }
@@ -1113,7 +1187,8 @@ update_consensus_networkstatus_downloads(time_t now)
 void
 networkstatus_consensus_download_failed(int status_code)
 {
-  download_status_failed(&consensus_dl_status, status_code);
+  /* XXXXNM Microdescs: may need to handle more types. */
+  download_status_failed(&consensus_dl_status[FLAV_NS], status_code);
   /* Retry immediately, if appropriate. */
   update_consensus_networkstatus_downloads(time(NULL));
 }
@@ -1219,10 +1294,14 @@ update_networkstatus_downloads(time_t now)
 void
 update_certificate_downloads(time_t now)
 {
-  if (consensus_waiting_for_certs)
-    authority_certs_fetch_missing(consensus_waiting_for_certs, now);
-  else
-    authority_certs_fetch_missing(current_consensus, now);
+  int i;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    if (consensus_waiting_for_certs[i].consensus)
+      authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus,
+                                    now);
+  }
+
+  authority_certs_fetch_missing(current_consensus, now);
 }
 
 /** Return 1 if we have a consensus but we don't have enough certificates
@@ -1230,7 +1309,8 @@ update_certificate_downloads(time_t now)
 int
 consensus_is_waiting_for_certs(void)
 {
-  return consensus_waiting_for_certs ? 1 : 0;
+  return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus
+    ? 1 : 0;
 }
 
 /** Return the network status with a given identity digest. */
@@ -1399,16 +1479,29 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c,
  * user, and -2 for more serious problems.
  */
 int
-networkstatus_set_current_consensus(const char *consensus, unsigned flags)
+networkstatus_set_current_consensus(const char *consensus,
+                                    const char *flavor,
+                                    unsigned flags)
 {
-  networkstatus_t *c;
+  networkstatus_t *c=NULL;
   int r, result = -1;
   time_t now = time(NULL);
   char *unverified_fname = NULL, *consensus_fname = NULL;
+  int flav = networkstatus_parse_flavor_name(flavor);
   const unsigned from_cache = flags & NSSET_FROM_CACHE;
   const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS;
   const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS);
   const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE;
+  const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR;
+  const digests_t *current_digests = NULL;
+  consensus_waiting_for_certs_t *waiting = NULL;
+  time_t current_valid_after = 0;
+
+  if (flav < 0) {
+    /* XXXX we don't handle unrecognized flavors yet. */
+    log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor);
+    return -2;
+  }
 
   /* Make sure it's parseable. */
   c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
@@ -1418,33 +1511,70 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags)
     goto done;
   }
 
+  if (c->flavor != flav) {
+    /* This wasn't the flavor we thought we were getting. */
+    if (require_flavor) {
+      log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)",
+               networkstatus_get_flavor_name(c->flavor), flavor);
+      goto done;
+    }
+    flav = c->flavor;
+    flavor = networkstatus_get_flavor_name(flav);
+  }
+
+  if (flav != USABLE_CONSENSUS_FLAVOR &&
+      !directory_caches_dir_info(get_options())) {
+    /* This consensus is totally boring to us: we won't use it, and we won't
+     * serve it.  Drop it. */
+    result = -1;
+    goto done;
+  }
+
   if (from_cache && !accept_obsolete &&
       c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) {
     /* XXX022 when we try to make fallbackconsensus work again, we should
      * consider taking this out. Until then, believing obsolete consensuses
      * is causing more harm than good. See also bug 887. */
-    log_info(LD_DIR, "Loaded an obsolete consensus. Discarding.");
+    log_info(LD_DIR, "Loaded an expired consensus. Discarding.");
     goto done;
   }
 
-  if (current_consensus &&
-      !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest,
-              DIGEST_LEN)) {
+  if (!strcmp(flavor, "ns")) {
+    consensus_fname = get_datadir_fname("cached-consensus");
+    unverified_fname = get_datadir_fname("unverified-consensus");
+    if (current_consensus) {
+      current_digests = &current_consensus->digests;
+      current_valid_after = current_consensus->valid_after;
+    }
+  } else {
+    cached_dir_t *cur;
+    char buf[128];
+    tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
+    consensus_fname = get_datadir_fname(buf);
+    tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
+    unverified_fname = get_datadir_fname(buf);
+    cur = dirserv_get_consensus(flavor);
+    if (cur) {
+      current_digests = &cur->digests;
+      current_valid_after = cur->published;
+    }
+  }
+
+  if (current_digests &&
+      !memcmp(&c->digests, current_digests, sizeof(c->digests))) {
     /* We already have this one. That's a failure. */
-    log_info(LD_DIR, "Got a consensus we already have");
+    log_info(LD_DIR, "Got a %s consensus we already have", flavor);
     goto done;
   }
 
-  if (current_consensus && c->valid_after <= current_consensus->valid_after) {
+  if (current_valid_after && c->valid_after <= current_valid_after) {
     /* We have a newer one.  There's no point in accepting this one,
      * even if it's great. */
-    log_info(LD_DIR, "Got a consensus at least as old as the one we have");
+    log_info(LD_DIR, "Got a %s consensus at least as old as the one we have",
+             flavor);
     goto done;
   }
 
-  consensus_fname = get_datadir_fname("cached-consensus");
-  unverified_fname = get_datadir_fname("unverified-consensus");
-
   /* Make sure it's signed enough. */
   if ((r=networkstatus_check_consensus_signature(c, 1))<0) {
     if (r == -1) {
@@ -1453,16 +1583,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags)
         log_info(LD_DIR,
                  "Not enough certificates to check networkstatus consensus");
       }
-      if (!current_consensus ||
-          c->valid_after > current_consensus->valid_after) {
-        if (consensus_waiting_for_certs)
-          networkstatus_vote_free(consensus_waiting_for_certs);
-        tor_free(consensus_waiting_for_certs_body);
-        consensus_waiting_for_certs = c;
+      if (!current_valid_after ||
+          c->valid_after > current_valid_after) {
+        waiting = &consensus_waiting_for_certs[flav];
+        if (waiting->consensus)
+          networkstatus_vote_free(waiting->consensus);
+        tor_free(waiting->body);
+        waiting->consensus = c;
         c = NULL; /* Prevent free. */
-        consensus_waiting_for_certs_body = tor_strdup(consensus);
-        consensus_waiting_for_certs_set_at = now;
-        consensus_waiting_for_certs_dl_failed = 0;
+        waiting->body = tor_strdup(consensus);
+        waiting->set_at = now;
+        waiting->dl_failed = 0;
         if (!from_cache) {
           write_str_to_file(unverified_fname, consensus, 0);
         }
@@ -1491,56 +1622,65 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags)
     }
   }
 
-  if (!from_cache)
+  if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR)
     control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED");
 
   /* Are we missing any certificates at all? */
   if (r != 1 && dl_certs)
     authority_certs_fetch_missing(c, now);
 
-  notify_control_networkstatus_changed(current_consensus, c);
+  if (flav == USABLE_CONSENSUS_FLAVOR) {
+    notify_control_networkstatus_changed(current_consensus, c);
 
-  if (current_consensus) {
-    networkstatus_copy_old_consensus_info(c, current_consensus);
-    networkstatus_vote_free(current_consensus);
+    if (current_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_consensus);
+      networkstatus_vote_free(current_consensus);
+    }
   }
 
-  if (consensus_waiting_for_certs &&
-      consensus_waiting_for_certs->valid_after <= c->valid_after) {
-    networkstatus_vote_free(consensus_waiting_for_certs);
-    consensus_waiting_for_certs = NULL;
-    if (consensus != consensus_waiting_for_certs_body)
-      tor_free(consensus_waiting_for_certs_body);
+  waiting = &consensus_waiting_for_certs[flav];
+  if (waiting->consensus &&
+      waiting->consensus->valid_after <= c->valid_after) {
+    networkstatus_vote_free(waiting->consensus);
+    waiting->consensus = NULL;
+    if (consensus != waiting->body)
+      tor_free(waiting->body);
     else
-      consensus_waiting_for_certs_body = NULL;
-    consensus_waiting_for_certs_set_at = 0;
-    consensus_waiting_for_certs_dl_failed = 0;
+      waiting->body = NULL;
+    waiting->set_at = 0;
+    waiting->dl_failed = 0;
     unlink(unverified_fname);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
   if (c->valid_after <= now && now <= c->valid_until) {
-    download_status_reset(&consensus_dl_status);
+    download_status_reset(&consensus_dl_status[flav]);
   } else {
     if (!from_cache)
-      download_status_failed(&consensus_dl_status, 0);
+      download_status_failed(&consensus_dl_status[flav], 0);
   }
 
-  current_consensus = c;
-  c = NULL; /* Prevent free. */
+  if (directory_caches_dir_info(get_options())) {
+    dirserv_set_cached_consensus_networkstatus(consensus,
+                                               flavor,
+                                               &c->digests,
+                                               c->valid_after);
+  }
 
-  update_consensus_networkstatus_fetch_time(now);
-  dirvote_recalculate_timing(get_options(), now);
-  routerstatus_list_update_named_server_map();
+  if (flav == USABLE_CONSENSUS_FLAVOR) {
+    current_consensus = c;
+    c = NULL; /* Prevent free. */
+
+    /* XXXXNM Microdescs: needs a non-ns variant. */
+    update_consensus_networkstatus_fetch_time(now);
+    dirvote_recalculate_timing(get_options(), now);
+    routerstatus_list_update_named_server_map();
+  }
 
   if (!from_cache) {
     write_str_to_file(consensus_fname, consensus, 0);
   }
 
-  if (directory_caches_dir_info(get_options()))
-    dirserv_set_cached_networkstatus_v3(consensus,
-                                        current_consensus->valid_after);
-
   if (ftime_definitely_before(now, current_consensus->valid_after)) {
     char tbuf[ISO_TIME_LEN+1];
     char dbuf[64];
@@ -1571,13 +1711,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags)
 void
 networkstatus_note_certs_arrived(void)
 {
-  if (consensus_waiting_for_certs) {
-    if (networkstatus_check_consensus_signature(
-                                    consensus_waiting_for_certs, 0)>=0) {
+  int i;
+  for (i=0; i<N_CONSENSUS_FLAVORS; ++i) {
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (!waiting->consensus)
+      continue;
+    if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
       if (!networkstatus_set_current_consensus(
-                                 consensus_waiting_for_certs_body,
+                                 waiting->body,
+                                 networkstatus_get_flavor_name(i),
                                  NSSET_WAS_WAITING_FOR_CERTS)) {
-        tor_free(consensus_waiting_for_certs_body);
+        tor_free(waiting->body);
       }
     }
   }
@@ -1663,10 +1807,8 @@ download_status_map_update_from_v2_networkstatus(void)
     v2_download_status_map = digestmap_new();
 
   dl_status = digestmap_new();
-  SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns,
-  {
-    SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs,
-    {
+  SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) {
+    SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) {
       const char *d = rs->descriptor_digest;
       download_status_t *s;
       if (digestmap_get(dl_status, d))
@@ -1675,8 +1817,8 @@ download_status_map_update_from_v2_networkstatus(void)
         s = tor_malloc_zero(sizeof(download_status_t));
       }
       digestmap_set(dl_status, d, s);
-    });
-  });
+    } SMARTLIST_FOREACH_END(rs);
+  } SMARTLIST_FOREACH_END(ns);
   digestmap_free(v2_download_status_map, _tor_free);
   v2_download_status_map = dl_status;
   networkstatus_v2_list_has_changed = 0;
@@ -1924,6 +2066,35 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name,
   return default_val;
 }
 
+/** Return the name of the consensus flavor <b>flav</b> as used to identify
+ * the flavor in directory documents. */
+const char *
+networkstatus_get_flavor_name(consensus_flavor_t flav)
+{
+  switch (flav) {
+    case FLAV_NS:
+      return "ns";
+    case FLAV_MICRODESC:
+      return "microdesc";
+    default:
+      tor_fragile_assert();
+      return "??";
+  }
+}
+
+/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>,
+ * or -1 if the flavor is not recongized. */
+int
+networkstatus_parse_flavor_name(const char *flavname)
+{
+  if (!strcmp(flavname, "ns"))
+    return FLAV_NS;
+  else if (!strcmp(flavname, "microdesc"))
+    return FLAV_MICRODESC;
+  else
+    return -1;
+}
+
 /** If <b>question</b> is a string beginning with "ns/" in a format the
  * control interface expects for a GETINFO question, set *<b>answer</b> to a
  * newly-allocated string containing networkstatus lines for the appropriate
@@ -1975,6 +2146,7 @@ getinfo_helper_networkstatus(control_connection_t *conn,
 void
 networkstatus_free_all(void)
 {
+  int i;
   if (networkstatus_v2_list) {
     SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns,
                       networkstatus_v2_free(ns));
@@ -1989,11 +2161,14 @@ networkstatus_free_all(void)
     networkstatus_vote_free(current_consensus);
     current_consensus = NULL;
   }
-  if (consensus_waiting_for_certs) {
-    networkstatus_vote_free(consensus_waiting_for_certs);
-    consensus_waiting_for_certs = NULL;
+  for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
+    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    if (waiting->consensus) {
+      networkstatus_vote_free(waiting->consensus);
+      waiting->consensus = NULL;
+    }
+    tor_free(waiting->body);
   }
-  tor_free(consensus_waiting_for_certs_body);
   if (named_server_map) {
     strmap_free(named_server_map, _tor_free);
   }

src/or/or.h

@@ -89,6 +89,7 @@
 #include "torgzip.h"
 #include "address.h"
 #include "compat_libevent.h"
+#include "ht.h"
 
 /* These signals are defined to help control_signal_act work.
  */
@@ -1170,7 +1171,8 @@ typedef struct dir_connection_t {
   enum {
     DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP,
     DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
-    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS
+    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS,
+    DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */
   } dir_spool_src : 3;
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
@@ -1281,6 +1283,7 @@ typedef struct cached_dir_t {
   size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
   size_t dir_z_len; /**< Length of <b>dir_z</b>. */
   time_t published; /**< When was this object published. */
+  digests_t digests; /**< Digests of this object (networkstatus only) */
   int refcnt; /**< Reference count for this cached_dir_t. */
 } cached_dir_t;
 
@@ -1557,6 +1560,52 @@ typedef struct routerstatus_t {
 
 } routerstatus_t;
 
+/** A microdescriptor is the smallest amount of information needed to build a
+ * circuit through a router.  They are generated by the directory authorities,
+ * using information from the uploaded routerinfo documents.  They are not
+ * self-signed, but are rather authenticated by having their hash in a signed
+ * networkstatus document. */
+typedef struct microdesc_t {
+  /** Hashtable node, used to look up the microdesc by its digest. */
+  HT_ENTRY(microdesc_t) node;
+
+  /* Cache information */
+
+  /**  When was this microdescriptor last listed in a consensus document?
+   * Once a microdesc has been unlisted long enough, we can drop it.
+   */
+  time_t last_listed;
+  /** Where is this microdescriptor currently stored? */
+  saved_location_t saved_location : 3;
+  /** If true, do not attempt to cache this microdescriptor on disk. */
+  unsigned int no_save : 1;
+  /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the
+   * microdescriptor in the cache. */
+  off_t off;
+
+  /* The string containing the microdesc. */
+
+  /** A pointer to the encoded body of the microdescriptor.  If the
+   * saved_location is SAVED_IN_CACHE, then the body is a pointer into an
+   * mmap'd region.  Otherwise, it is a malloc'd string.  The string might not
+   * be NUL-terminated; take the length from <b>bodylen</b>. */
+  char *body;
+  /** The length of the microdescriptor in <b>body</b>. */
+  size_t bodylen;
+  /** A SHA256-digest of the microdescriptor. */
+  char digest[DIGEST256_LEN];
+
+  /* Fields in the microdescriptor. */
+
+  /** As routerinfo_t.onion_pkey */
+  crypto_pk_env_t *onion_pkey;
+  /** As routerinfo_t.family */
+  smartlist_t *family;
+  /** Encoded exit policy summary */
+  char *exitsummary; /**< exit policy summary -
+                      * XXX this probably should not stay a string. */
+} microdesc_t;
+
 /** How many times will we try to download a router's descriptor before giving
  * up? */
 #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8
@@ -1599,6 +1648,11 @@ typedef struct networkstatus_v2_t {
                          * sorted by identity_digest. */
 } networkstatus_v2_t;
 
+typedef struct vote_microdesc_hash_t {
+  struct vote_microdesc_hash_t *next;
+  char *microdesc_hash_line;
+} vote_microdesc_hash_t;
+
 /** The claim about a single router, made in a vote. */
 typedef struct vote_routerstatus_t {
   routerstatus_t status; /**< Underlying 'status' object for this router.
@@ -1607,31 +1661,45 @@ typedef struct vote_routerstatus_t {
                    * networkstatus_t.known_flags. */
   char *version; /**< The version that the authority says this router is
                   * running. */
+  vote_microdesc_hash_t *microdesc;
 } vote_routerstatus_t;
 
+/** A signature of some document by an authority. */
+typedef struct document_signature_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
+  /** Declared SHA-1 digest of signing key used by this voter. */
+  char signing_key_digest[DIGEST_LEN];
+  /** Algorithm used to compute the digest of the document. */
+  digest_algorithm_t alg;
+  /** Signature of the signed thing. */
+  char *signature;
+  /** Length of <b>signature</b> */
+  int signature_len;
+  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
+                                   * the sig, and we know it's bad. */
+  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
+                                     * as good. */
+} document_signature_t;
+
 /** Information about a single voter in a vote or a consensus. */
 typedef struct networkstatus_voter_info_t {
+  /** Declared SHA-1 digest of this voter's identity key */
+  char identity_digest[DIGEST_LEN];
   char *nickname; /**< Nickname of this voter */
-  char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */
+  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
+   * consensuses, we treat legacy keys as additional signers. */
+  char legacy_id_digest[DIGEST_LEN];
   char *address; /**< Address of this voter, in string format. */
   uint32_t addr; /**< Address of this voter, in IPv4, in host order. */
   uint16_t dir_port; /**< Directory port of this voter */
   uint16_t or_port; /**< OR port of this voter */
   char *contact; /**< Contact information for this voter. */
   char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */
-  /** Digest of this voter's "legacy" identity key, if any.  In vote only; for
-   * consensuses, we treat legacy keys as additional signers. */
-  char legacy_id_digest[DIGEST_LEN];
 
   /* Nothing from here on is signed. */
-  char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key
-                                        * used by this voter. */
-  char *signature; /**< Signature from this voter. */
-  int signature_len; /**< Length of <b>signature</b> */
-  unsigned int bad_signature : 1; /**< Set to true if we've tried to verify
-                                   * the sig, and we know it's bad. */
-  unsigned int good_signature : 1; /**< Set to true if we've verified the sig
-                                     * as good. */
+  /** The signature of the document and the signature's status. */
+  smartlist_t *sigs;
 } networkstatus_voter_info_t;
 
 /** Enumerates the possible seriousness values of a networkstatus document. */
@@ -1641,10 +1709,25 @@ typedef enum {
   NS_TYPE_OPINION,
 } networkstatus_type_t;
 
+/** Enumerates recognized flavors of a consensus networkstatus document.  All
+ * flavors of a consensus are generated from the same set of votes, but they
+ * present different types information to different versions of Tor. */
+typedef enum {
+  FLAV_NS = 0,
+  FLAV_MICRODESC = 1,
+} consensus_flavor_t;
+
+/** Which consensus flavor do we actually want to use to build circuits? */
+#define USABLE_CONSENSUS_FLAVOR FLAV_NS
+
+/** How many different consensus flavors are there? */
+#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1)
+
 /** A common structure to hold a v3 network status vote, or a v3 network
  * status consensus. */
 typedef struct networkstatus_t {
-  networkstatus_type_t type; /**< Vote, consensus, or opinion? */
+  networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */
+  consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */
   time_t published; /**< Vote only: Time when vote was written. */
   time_t valid_after; /**< Time after which this vote or consensus applies. */
   time_t fresh_until; /**< Time before which this is the most recent vote or
@@ -1683,8 +1766,8 @@ typedef struct networkstatus_t {
 
   struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */
 
-  /** Digest of this document, as signed. */
-  char networkstatus_digest[DIGEST_LEN];
+  /** Digests of this document, as signed. */
+  digests_t digests;
 
   /** List of router statuses, sorted by identity digest.  For a vote,
    * the elements are vote_routerstatus_t; for a consensus, the elements
@@ -1696,14 +1779,15 @@ typedef struct networkstatus_t {
   digestmap_t *desc_digest_map;
 } networkstatus_t;
 
-/** A set of signatures for a networkstatus consensus.  All fields are as for
- * networkstatus_t. */
+/** A set of signatures for a networkstatus consensus.  Unless otherwise
+ * noted, all fields are as for networkstatus_t. */
 typedef struct ns_detached_signatures_t {
   time_t valid_after;
   time_t fresh_until;
   time_t valid_until;
-  char networkstatus_digest[DIGEST_LEN];
-  smartlist_t *signatures; /* list of networkstatus_voter_info_t */
+  strmap_t *digests; /**< Map from flavor name to digestset_t */
+  strmap_t *signatures; /**< Map from flavor name to list of
+                         * document_signature_t */
 } ns_detached_signatures_t;
 
 /** Allowable types of desc_store_t. */
@@ -3595,9 +3679,13 @@ void directory_initiate_command(const char *address, const tor_addr_t *addr,
                                 const char *payload, size_t payload_len,
                                 time_t if_modified_since);
 
+#define DSR_HEX       (1<<0)
+#define DSR_BASE64    (1<<1)
+#define DSR_DIGEST256 (1<<2)
+#define DSR_SORT_UNIQ (1<<3)
 int dir_split_resource_into_fingerprints(const char *resource,
-                                    smartlist_t *fp_out, int *compresseed_out,
-                                    int decode_hex, int sort_uniq);
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags);
 /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */
 typedef struct {
   char first[DIGEST_LEN];
@@ -3702,14 +3790,16 @@ int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now);
 void directory_set_dirty(void);
 cached_dir_t *dirserv_get_directory(void);
 cached_dir_t *dirserv_get_runningrouters(void);
-cached_dir_t *dirserv_get_consensus(void);
+cached_dir_t *dirserv_get_consensus(const char *flavor_name);
 void dirserv_set_cached_directory(const char *directory, time_t when,
                                   int is_running_routers);
 void dirserv_set_cached_networkstatus_v2(const char *directory,
                                          const char *identity,
                                          time_t published);
-void dirserv_set_cached_networkstatus_v3(const char *consensus,
-                                         time_t published);
+void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                                const char *flavor_name,
+                                                const digests_t *digests,
+                                                time_t published);
 void dirserv_clear_old_networkstatuses(time_t cutoff);
 void dirserv_clear_old_v1_info(time_t now);
 void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key);
@@ -3731,10 +3821,14 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
 int dirserv_would_reject_router(routerstatus_t *rs);
 int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff);
 int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
+int dirserv_have_any_microdesc(const smartlist_t *fps);
 size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
                                   int compressed);
+size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed);
+
 typedef enum {
-  NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT
+  NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT,
+  NS_V3_CONSENSUS_MICRODESC
 } routerstatus_format_type_t;
 int routerstatus_format_entry(char *buf, size_t buf_len,
                               routerstatus_t *rs, const char *platform,
@@ -3776,11 +3870,12 @@ char *networkstatus_compute_consensus(smartlist_t *votes,
                                       crypto_pk_env_t *identity_key,
                                       crypto_pk_env_t *signing_key,
                                       const char *legacy_identity_key_digest,
-                                      crypto_pk_env_t *legacy_signing_key);
+                                      crypto_pk_env_t *legacy_signing_key,
+                                      consensus_flavor_t flavor);
 int networkstatus_add_detached_signatures(networkstatus_t *target,
                                           ns_detached_signatures_t *sigs,
                                           const char **msg_out);
-char *networkstatus_get_detached_signatures(networkstatus_t *consensus);
+char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
 void ns_detached_signatures_free(ns_detached_signatures_t *s);
 
 /* cert manipulation */
@@ -3808,7 +3903,7 @@ int dirvote_add_signatures(const char *detached_signatures_body,
                            const char **msg_out);
 
 /* Item access */
-const char *dirvote_get_pending_consensus(void);
+const char *dirvote_get_pending_consensus(consensus_flavor_t flav);
 const char *dirvote_get_pending_detached_signatures(void);
 #define DGV_BY_ID 1
 #define DGV_INCLUDE_PENDING 2
@@ -3823,6 +3918,17 @@ networkstatus_t *
 dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
                                         authority_cert_t *cert);
 
+microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri);
+ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len,
+                                       const microdesc_t *md);
+int vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                          const vote_routerstatus_t *vrs,
+                                          int method,
+                                          digest_algorithm_t alg);
+document_signature_t *voter_get_sig_by_algorithm(
+                           const networkstatus_voter_info_t *voter,
+                           digest_algorithm_t alg);
+
 #ifdef DIRVOTE_PRIVATE
 char *format_networkstatus_vote(crypto_pk_env_t *private_key,
                                  networkstatus_t *v3_ns);
@@ -4031,6 +4137,31 @@ void do_hash_password(void);
 int tor_init(int argc, char **argv);
 #endif
 
+/********************************* microdesc.c *************************/
+
+typedef struct microdesc_cache_t microdesc_cache_t;
+
+microdesc_cache_t *get_microdesc_cache(void);
+
+smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save);
+smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                        smartlist_t *descriptors, saved_location_t where,
+                        int no_save);
+
+int microdesc_cache_rebuild(microdesc_cache_t *cache);
+int microdesc_cache_reload(microdesc_cache_t *cache);
+void microdesc_cache_clear(microdesc_cache_t *cache);
+
+microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
+                                                 const char *d);
+
+size_t microdesc_average_size(microdesc_cache_t *cache);
+
+void microdesc_free(microdesc_t *md);
+void microdesc_free_all(void);
+
 /********************************* networkstatus.c *********************/
 
 /** How old do we allow a v2 network-status to get before removing it
@@ -4068,9 +4199,9 @@ networkstatus_voter_info_t *networkstatus_get_voter_by_id(
                                        const char *identity);
 int networkstatus_check_consensus_signature(networkstatus_t *consensus,
                                             int warn);
-int networkstatus_check_voter_signature(networkstatus_t *consensus,
-                                        networkstatus_voter_info_t *voter,
-                                        authority_cert_t *cert);
+int networkstatus_check_document_signature(const networkstatus_t *consensus,
+                                           document_signature_t *sig,
+                                           const authority_cert_t *cert);
 char *networkstatus_get_cache_filename(const char *identity_digest);
 int router_set_networkstatus_v2(const char *s, time_t arrived_at,
                              v2_networkstatus_source_t source,
@@ -4107,7 +4238,10 @@ networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now);
 #define NSSET_WAS_WAITING_FOR_CERTS 2
 #define NSSET_DONT_DOWNLOAD_CERTS 4
 #define NSSET_ACCEPT_OBSOLETE 8
-int networkstatus_set_current_consensus(const char *consensus, unsigned flags);
+#define NSSET_REQUIRE_FLAVOR 16
+int networkstatus_set_current_consensus(const char *consensus,
+                                        const char *flavor,
+                                        unsigned flags);
 void networkstatus_note_certs_arrived(void);
 void routers_update_all_from_networkstatus(time_t now, int dir_version);
 void routerstatus_list_update_from_consensus_networkstatus(time_t now);
@@ -4123,6 +4257,10 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name,
                                 int32_t default_val);
 int getinfo_helper_networkstatus(control_connection_t *conn,
                                  const char *question, char **answer);
+const char *networkstatus_get_flavor_name(consensus_flavor_t flav);
+int networkstatus_parse_flavor_name(const char *flavname);
+void document_signature_free(document_signature_t *sig);
+document_signature_t *document_signature_dup(const document_signature_t *sig);
 void networkstatus_free_all(void);
 
 /********************************* ntmain.c ***************************/
@@ -4907,10 +5045,13 @@ int router_get_router_hash(const char *s, char *digest);
 int router_get_dir_hash(const char *s, char *digest);
 int router_get_runningrouters_hash(const char *s, char *digest);
 int router_get_networkstatus_v2_hash(const char *s, char *digest);
-int router_get_networkstatus_v3_hash(const char *s, char *digest);
+int router_get_networkstatus_v3_hash(const char *s, char *digest,
+                                     digest_algorithm_t algorithm);
+int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests);
 int router_get_extrainfo_hash(const char *s, char *digest);
 int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
+                                   size_t digest_len,
                                    crypto_pk_env_t *private_key);
 int router_parse_list_from_string(const char **s, const char *eos,
                                   smartlist_t *dest,
@@ -4950,6 +5091,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
 ns_detached_signatures_t *networkstatus_parse_detached_signatures(
                                           const char *s, const char *eos);
 
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+                                          int allow_annotations,
+                                          int copy_body);
+
 authority_cert_t *authority_cert_parse_from_string(const char *s,
                                                    const char **end_of_string);
 int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,

src/or/rendcommon.c

@@ -618,7 +618,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
     }
     if (router_append_dirobj_signature(desc_str + written,
                                        desc_len - written,
-                                       desc_digest, service_key) < 0) {
+                                       desc_digest, DIGEST_LEN,
+                                       service_key) < 0) {
       log_warn(LD_BUG, "Couldn't sign desc.");
       rend_encoded_v2_service_descriptor_free(enc);
       goto err;

src/or/router.c

@@ -1788,7 +1788,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
 
   note_crypto_pk_op(SIGN_RTR);
   if (router_append_dirobj_signature(s+written,maxlen-written,
-                                     digest,ident_key)<0) {
+                                     digest,DIGEST_LEN,ident_key)<0) {
     log_warn(LD_BUG, "Couldn't sign router descriptor");
     return -1;
   }
@@ -1980,7 +1980,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo,
   len += strlen(s+len);
   if (router_get_extrainfo_hash(s, digest)<0)
     return -1;
-  if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0)
+  if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN,
+                                     ident_key)<0)
     return -1;
 
   {

src/or/routerlist.c

@@ -448,17 +448,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now)
 
   list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
   if (status) {
-    SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter,
-      {
-        if (tor_digest_is_zero(voter->signing_key_digest))
-          continue; /* This authority never signed this consensus, so don't
-                     * go looking for a cert with key digest 0000000000. */
-        if (!cache &&
-            !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
-          continue; /* We are not a cache, and we don't know this authority.*/
-        cl = get_cert_list(voter->identity_digest);
+    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+                            voter) {
+      if (!smartlist_len(voter->sigs))
+        continue; /* This authority never signed this consensus, so don't
+                   * go looking for a cert with key digest 0000000000. */
+      if (!cache &&
+          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+        continue; /* We are not a cache, and we don't know this authority.*/
+      cl = get_cert_list(voter->identity_digest);
+      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
         cert = authority_cert_get_by_digests(voter->identity_digest,
-                                             voter->signing_key_digest);
+                                             sig->signing_key_digest);
         if (cert) {
           if (now < cert->expires)
             download_status_reset(&cl->dl_status);
@@ -469,37 +470,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now)
             !digestmap_get(pending, voter->identity_digest)) {
           log_notice(LD_DIR, "We're missing a certificate from authority "
                      "with signing key %s: launching request.",
-                     hex_str(voter->signing_key_digest, DIGEST_LEN));
-          smartlist_add(missing_digests, voter->identity_digest);
+                     hex_str(sig->signing_key_digest, DIGEST_LEN));
+          smartlist_add(missing_digests, sig->identity_digest);
         }
-      });
+      } SMARTLIST_FOREACH_END(sig);
+    } SMARTLIST_FOREACH_END(voter);
   }
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
-    {
-      int found = 0;
-      if (!(ds->type & V3_AUTHORITY))
-        continue;
-      if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest))
-        continue;
-      cl = get_cert_list(ds->v3_identity_digest);
-      SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
-        {
-          if (!ftime_definitely_after(now, cert->expires)) {
-            /* It's not expired, and we weren't looking for something to
-             * verify a consensus with.  Call it done. */
-            download_status_reset(&cl->dl_status);
-            found = 1;
-            break;
-          }
-        });
-      if (!found &&
-          download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) &&
-          !digestmap_get(pending, ds->v3_identity_digest)) {
-        log_notice(LD_DIR, "No current certificate known for authority %s; "
-                   "launching request.", ds->nickname);
-        smartlist_add(missing_digests, ds->v3_identity_digest);
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) {
+    int found = 0;
+    if (!(ds->type & V3_AUTHORITY))
+      continue;
+    if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest))
+      continue;
+    cl = get_cert_list(ds->v3_identity_digest);
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, {
+      if (!ftime_definitely_after(now, cert->expires)) {
+        /* It's not expired, and we weren't looking for something to
+         * verify a consensus with.  Call it done. */
+        download_status_reset(&cl->dl_status);
+        found = 1;
+        break;
       }
     });
+    if (!found &&
+        download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) &&
+        !digestmap_get(pending, ds->v3_identity_digest)) {
+      log_notice(LD_DIR, "No current certificate known for authority %s; "
+                 "launching request.", ds->nickname);
+        smartlist_add(missing_digests, ds->v3_identity_digest);
+    }
+  } SMARTLIST_FOREACH_END(ds);
 
   if (!smartlist_len(missing_digests)) {
     goto done;
@@ -3832,7 +3832,7 @@ list_pending_downloads(digestmap_t *result,
       const char *resource = TO_DIR_CONN(conn)->requested_resource;
       if (!strcmpstart(resource, prefix))
         dir_split_resource_into_fingerprints(resource + p_len,
-                                             tmp, NULL, 1, 0);
+                                             tmp, NULL, DSR_HEX);
     }
   });
   SMARTLIST_FOREACH(tmp, char *, d,

src/test/test_crypto.c

@@ -472,6 +472,17 @@ test_crypto_formats(void)
 
   test_assert(digest_from_base64(data3, "###") < 0);
 
+  /* Encoding SHA256 */
+  crypto_rand(data2, DIGEST256_LEN);
+  memset(data2, 100, 1024);
+  digest256_to_base64(data2, data1);
+  test_eq(BASE64_DIGEST256_LEN, strlen(data2));
+  test_eq(100, data2[BASE64_DIGEST256_LEN+2]);
+  memset(data3, 99, 1024);
+  test_eq(digest256_from_base64(data3, data2), 0);
+  test_memeq(data1, data3, DIGEST256_LEN);
+  test_eq(99, data3[DIGEST256_LEN+1]);
+
   /* Base32 tests */
   strlcpy(data1, "5chrs", 1024);
   /* bit pattern is:  [35 63 68 72 73] ->

src/test/test_dir.c

@@ -361,9 +361,9 @@ test_dir_versions(void)
   ;
 }
 
-/** Run unit tests for misc directory functions. */
+/** Run unit tests for directory fp_pair functions. */
 static void
-test_dir_util(void)
+test_dir_fp_pairs(void)
 {
   smartlist_t *sl = smartlist_create();
   fp_pair_t *pair;
@@ -390,6 +390,127 @@ test_dir_util(void)
   smartlist_free(sl);
 }
 
+static void
+test_dir_split_fps(void *testdata)
+{
+  smartlist_t *sl = smartlist_create();
+  char *mem_op_hex_tmp = NULL;
+  (void)testdata;
+
+  /* Some example hex fingerprints and their base64 equivalents */
+#define HEX1 "Fe0daff89127389bc67558691231234551193EEE"
+#define HEX2 "Deadbeef99999991111119999911111111f00ba4"
+#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d"
+#define HEX256_1 \
+    "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf"
+#define HEX256_2 \
+    "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc"
+#define HEX256_3 \
+    "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef"
+#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4"
+#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q"
+#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0"
+#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78"
+#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw"
+#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8"
+
+  /* no flags set */
+  dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0);
+  tt_int_op(smartlist_len(sl), ==, 3);
+  tt_str_op(smartlist_get(sl, 0), ==, "A");
+  tt_str_op(smartlist_get(sl, 1), ==, "C");
+  tt_str_op(smartlist_get(sl, 2), ==, "B");
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* uniq strings. */
+  dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ);
+  tt_int_op(smartlist_len(sl), ==, 3);
+  tt_str_op(smartlist_get(sl, 0), ==, "A");
+  tt_str_op(smartlist_get(sl, 1), ==, "B");
+  tt_str_op(smartlist_get(sl, 2), ==, "C");
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode hex. */
+  dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX);
+  tt_int_op(smartlist_len(sl), ==, 2);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* decode hex and drop weirdness. */
+  dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1,
+                                       sl, NULL, DSR_HEX);
+  tt_int_op(smartlist_len(sl), ==, 2);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode long hex */
+  dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3,
+                                       sl, NULL, DSR_HEX|DSR_DIGEST256);
+  tt_int_op(smartlist_len(sl), ==, 3);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2);
+  test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode hex and sort. */
+  dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2,
+                                       sl, NULL, DSR_HEX|DSR_SORT_UNIQ);
+  tt_int_op(smartlist_len(sl), ==, 3);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2);
+  test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode long hex and sort */
+  dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3
+                                       "+"HEX256_1,
+                                       sl, NULL,
+                                       DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ);
+  tt_int_op(smartlist_len(sl), ==, 3);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2);
+  test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode base64 */
+  dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64);
+  tt_int_op(smartlist_len(sl), ==, 2);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  /* Decode long base64 */
+  dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2,
+                                       sl, NULL, DSR_BASE64|DSR_DIGEST256);
+  tt_int_op(smartlist_len(sl), ==, 2);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1);
+  test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+  dir_split_resource_into_fingerprints(B64_256_1,
+                                       sl, NULL, DSR_BASE64|DSR_DIGEST256);
+  tt_int_op(smartlist_len(sl), ==, 1);
+  test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_clear(sl);
+
+ done:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  tor_free(mem_op_hex_tmp);
+}
+
 static void
 test_dir_measured_bw(void)
 {
@@ -559,6 +680,23 @@ generate_ri_from_rs(const vote_routerstatus_t *vrs)
   return r;
 }
 
+/** Helper: get a detached signatures document for one or two
+ * consensuses. */
+static char *
+get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2)
+{
+  char *r;
+  smartlist_t *sl;
+  tor_assert(ns && ns->flavor == FLAV_NS);
+  sl = smartlist_create();
+  smartlist_add(sl,ns);
+  if (ns2)
+    smartlist_add(sl,ns2);
+  r = networkstatus_get_detached_signatures(sl);
+  smartlist_free(sl);
+  return r;
+}
+
 /** Run unit tests for generating and parsing V3 consensus networkstatus
  * documents. */
 static void
@@ -571,7 +709,9 @@ test_dir_v3_networkstatus(void)
 
   time_t now = time(NULL);
   networkstatus_voter_info_t *voter;
-  networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL;
+  document_signature_t *sig;
+  networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL,
+    *con_md=NULL;
   vote_routerstatus_t *vrs;
   routerstatus_t *rs;
   char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp;
@@ -580,7 +720,9 @@ test_dir_v3_networkstatus(void)
   /* For generating the two other consensuses. */
   char *detached_text1=NULL, *detached_text2=NULL;
   char *consensus_text2=NULL, *consensus_text3=NULL;
-  networkstatus_t *con2=NULL, *con3=NULL;
+  char *consensus_text_md2=NULL, *consensus_text_md3=NULL;
+  char *consensus_text_md=NULL;
+  networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL;
   ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL;
 
   /* Parse certificates and keys. */
@@ -858,13 +1000,25 @@ test_dir_v3_networkstatus(void)
                                                    cert3->identity_key,
                                                    sign_skey_3,
                                                    "AAAAAAAAAAAAAAAAAAAA",
-                                                   sign_skey_leg1);
+                                                   sign_skey_leg1,
+                                                   FLAV_NS);
   test_assert(consensus_text);
   con = networkstatus_parse_vote_from_string(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
   test_assert(con);
   //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n",
   //           v1_text, v2_text, v3_text);
+  consensus_text_md = networkstatus_compute_consensus(votes, 3,
+                                                   cert3->identity_key,
+                                                   sign_skey_3,
+                                                   "AAAAAAAAAAAAAAAAAAAA",
+                                                   sign_skey_leg1,
+                                                   FLAV_MICRODESC);
+  test_assert(consensus_text_md);
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+                                                NS_TYPE_CONSENSUS);
+  test_assert(con_md);
+  test_eq(con_md->flavor, FLAV_MICRODESC);
 
   /* Check consensus contents. */
   test_assert(con->type == NS_TYPE_CONSENSUS);
@@ -939,26 +1093,23 @@ test_dir_v3_networkstatus(void)
   test_assert(rs->is_valid);
   test_assert(!rs->is_named);
   /* XXXX check version */
-  // x231
-  // x213
 
   /* Check signatures.  the first voter is a pseudo-entry with a legacy key.
    * The second one hasn't signed.  The fourth one has signed: validate it. */
   voter = smartlist_get(con->voters, 1);
-  test_assert(!voter->signature);
-  test_assert(!voter->good_signature);
-  test_assert(!voter->bad_signature);
+  test_eq(smartlist_len(voter->sigs), 0);
 
   voter = smartlist_get(con->voters, 3);
-  test_assert(voter->signature);
-  test_assert(!voter->good_signature);
-  test_assert(!voter->bad_signature);
-  test_assert(!networkstatus_check_voter_signature(con,
-                                               smartlist_get(con->voters, 3),
-                                               cert3));
-  test_assert(voter->signature);
-  test_assert(voter->good_signature);
-  test_assert(!voter->bad_signature);
+  test_eq(smartlist_len(voter->sigs), 1);
+  sig = smartlist_get(voter->sigs, 0);
+  test_assert(sig->signature);
+  test_assert(!sig->good_signature);
+  test_assert(!sig->bad_signature);
+
+  test_assert(!networkstatus_check_document_signature(con, sig, cert3));
+  test_assert(sig->signature);
+  test_assert(sig->good_signature);
+  test_assert(!sig->bad_signature);
 
   {
     const char *msg=NULL;
@@ -966,28 +1117,47 @@ test_dir_v3_networkstatus(void)
     smartlist_shuffle(votes);
     consensus_text2 = networkstatus_compute_consensus(votes, 3,
                                                       cert2->identity_key,
-                                                      sign_skey_2, NULL,NULL);
+                                                      sign_skey_2, NULL,NULL,
+                                                      FLAV_NS);
+    consensus_text_md2 = networkstatus_compute_consensus(votes, 3,
+                                                      cert2->identity_key,
+                                                      sign_skey_2, NULL,NULL,
+                                                      FLAV_MICRODESC);
     smartlist_shuffle(votes);
     consensus_text3 = networkstatus_compute_consensus(votes, 3,
                                                       cert1->identity_key,
-                                                      sign_skey_1, NULL,NULL);
+                                                      sign_skey_1, NULL,NULL,
+                                                      FLAV_NS);
+    consensus_text_md3 = networkstatus_compute_consensus(votes, 3,
+                                                      cert1->identity_key,
+                                                      sign_skey_1, NULL,NULL,
+                                                      FLAV_MICRODESC);
     test_assert(consensus_text2);
     test_assert(consensus_text3);
+    test_assert(consensus_text_md2);
+    test_assert(consensus_text_md3);
     con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
                                                 NS_TYPE_CONSENSUS);
     con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL,
                                                 NS_TYPE_CONSENSUS);
+    con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL,
+                                                NS_TYPE_CONSENSUS);
+    con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL,
+                                                NS_TYPE_CONSENSUS);
     test_assert(con2);
     test_assert(con3);
+    test_assert(con_md2);
+    test_assert(con_md3);
 
     /* All three should have the same digest. */
-    test_memeq(con->networkstatus_digest, con2->networkstatus_digest,
-               DIGEST_LEN);
-    test_memeq(con->networkstatus_digest, con3->networkstatus_digest,
-               DIGEST_LEN);
+    test_memeq(&con->digests, &con2->digests, sizeof(digests_t));
+    test_memeq(&con->digests, &con3->digests, sizeof(digests_t));
+
+    test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t));
+    test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t));
 
     /* Extract a detached signature from con3. */
-    detached_text1 = networkstatus_get_detached_signatures(con3);
+    detached_text1 = get_detached_sigs(con3, con_md3);
     tor_assert(detached_text1);
     /* Try to parse it. */
     dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL);
@@ -997,18 +1167,42 @@ test_dir_v3_networkstatus(void)
     test_eq(dsig1->valid_after, con3->valid_after);
     test_eq(dsig1->fresh_until, con3->fresh_until);
     test_eq(dsig1->valid_until, con3->valid_until);
-    test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest,
-               DIGEST_LEN);
-    test_eq(1, smartlist_len(dsig1->signatures));
-    voter = smartlist_get(dsig1->signatures, 0);
-    test_memeq(voter->identity_digest, cert1->cache_info.identity_digest,
-               DIGEST_LEN);
+    {
+      digests_t *dsig_digests = strmap_get(dsig1->digests, "ns");
+      test_assert(dsig_digests);
+      test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1],
+                 DIGEST_LEN);
+      dsig_digests = strmap_get(dsig1->digests, "microdesc");
+      test_assert(dsig_digests);
+      test_memeq(dsig_digests->d[DIGEST_SHA256],
+                 con_md3->digests.d[DIGEST_SHA256],
+                 DIGEST256_LEN);
+    }
+    {
+      smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns");
+      test_assert(dsig_signatures);
+      test_eq(1, smartlist_len(dsig_signatures));
+      sig = smartlist_get(dsig_signatures, 0);
+      test_memeq(sig->identity_digest, cert1->cache_info.identity_digest,
+                 DIGEST_LEN);
+      test_eq(sig->alg, DIGEST_SHA1);
+
+      dsig_signatures = strmap_get(dsig1->signatures, "microdesc");
+      test_assert(dsig_signatures);
+      test_eq(1, smartlist_len(dsig_signatures));
+      sig = smartlist_get(dsig_signatures, 0);
+      test_memeq(sig->identity_digest, cert1->cache_info.identity_digest,
+                 DIGEST_LEN);
+      test_eq(sig->alg, DIGEST_SHA256);
+    }
 
     /* Try adding it to con2. */
-    detached_text2 = networkstatus_get_detached_signatures(con2);
+    detached_text2 = get_detached_sigs(con2,con_md2);
     test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg));
     tor_free(detached_text2);
-    detached_text2 = networkstatus_get_detached_signatures(con2);
+    test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg));
+    tor_free(detached_text2);
+    detached_text2 = get_detached_sigs(con2,con_md2);
     //printf("\n<%s>\n", detached_text2);
     dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL);
     test_assert(dsig2);
@@ -1020,7 +1214,11 @@ test_dir_v3_networkstatus(void)
         printf("%s\n", hd);
       });
     */
-    test_eq(2, smartlist_len(dsig2->signatures));
+    test_eq(2,
+            smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns")));
+    test_eq(2,
+            smartlist_len((smartlist_t*)strmap_get(dsig2->signatures,
+                                                   "microdesc")));
 
     /* Try adding to con2 twice; verify that nothing changes. */
     test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg));
@@ -1028,13 +1226,14 @@ test_dir_v3_networkstatus(void)
     /* Add to con. */
     test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg));
     /* Check signatures */
-    test_assert(!networkstatus_check_voter_signature(con,
-                                               smartlist_get(con->voters, 1),
-                                               cert2));
-    test_assert(!networkstatus_check_voter_signature(con,
-                                               smartlist_get(con->voters, 2),
-                                               cert1));
-
+    voter = smartlist_get(con->voters, 1);
+    sig = smartlist_get(voter->sigs, 0);
+    test_assert(sig);
+    test_assert(!networkstatus_check_document_signature(con, sig, cert2));
+    voter = smartlist_get(con->voters, 2);
+    sig = smartlist_get(voter->sigs, 0);
+    test_assert(sig);
+    test_assert(!networkstatus_check_document_signature(con, sig, cert1));
   }
 
  done:
@@ -1043,6 +1242,7 @@ test_dir_v3_networkstatus(void)
   tor_free(v2_text);
   tor_free(v3_text);
   tor_free(consensus_text);
+  tor_free(consensus_text_md);
 
   if (vote)
     networkstatus_vote_free(vote);
@@ -1054,6 +1254,8 @@ test_dir_v3_networkstatus(void)
     networkstatus_vote_free(v3);
   if (con)
     networkstatus_vote_free(con);
+  if (con_md)
+    networkstatus_vote_free(con_md);
   if (sign_skey_1)
     crypto_free_pk_env(sign_skey_1);
   if (sign_skey_2)
@@ -1071,12 +1273,18 @@ test_dir_v3_networkstatus(void)
 
   tor_free(consensus_text2);
   tor_free(consensus_text3);
+  tor_free(consensus_text_md2);
+  tor_free(consensus_text_md3);
   tor_free(detached_text1);
   tor_free(detached_text2);
   if (con2)
     networkstatus_vote_free(con2);
   if (con3)
     networkstatus_vote_free(con3);
+  if (con_md2)
+    networkstatus_vote_free(con_md2);
+  if (con_md3)
+    networkstatus_vote_free(con_md3);
   if (dsig1)
     ns_detached_signatures_free(dsig1);
   if (dsig2)
@@ -1086,11 +1294,15 @@ test_dir_v3_networkstatus(void)
 #define DIR_LEGACY(name)                                                   \
   { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name }
 
+#define DIR(name)                               \
+  { #name, test_dir_##name, 0, NULL, NULL }
+
 struct testcase_t dir_tests[] = {
   DIR_LEGACY(nicknames),
   DIR_LEGACY(formats),
   DIR_LEGACY(versions),
-  DIR_LEGACY(util),
+  DIR_LEGACY(fp_pairs),
+  DIR(split_fps),
   DIR_LEGACY(measured_bw),
   DIR_LEGACY(param_voting),
   DIR_LEGACY(v3_networkstatus),