Code to generate, store, and parse microdescriptors and consensuses.

The consensus documents are not signed properly, not served, and not
exchanged yet.

src/common/util.c

@@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest)
   return tor_mem_is_zero(digest, DIGEST_LEN);
 }
 
+/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
+int
+tor_digest256_is_zero(const char *digest)
+{
+  return tor_mem_is_zero(digest, DIGEST256_LEN);
+}
+
 /* Helper: common code to check whether the result of a strtol or strtoul or
  * strtoll is correct. */
 #define CHECK_STRTOX_RESULT()                           \

src/common/util.h

@@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE;
 const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE;
 int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE;
 int tor_digest_is_zero(const char *digest) ATTR_PURE;
+int tor_digest256_is_zero(const char *digest) ATTR_PURE;
 char *esc_for_log(const char *string) ATTR_MALLOC;
 const char *escaped(const char *string);
 struct smartlist_t;

src/or/Makefile.am

@@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \
 	connection.c connection_edge.c connection_or.c control.c \
 	cpuworker.c directory.c dirserv.c dirvote.c \
 	dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \
+	microdesc.c \
 	networkstatus.c onion.c policies.c \
 	reasons.c relay.c rendcommon.c rendclient.c rendmid.c \
 	rendservice.c rephist.c router.c routerlist.c routerparse.c \

src/or/dirserv.c

@@ -1901,10 +1901,11 @@ routerstatus_format_entry(char *buf, size_t buf_len,
   tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
 
   r = tor_snprintf(buf, buf_len,
-                   "r %s %s %s %s %s %d %d\n",
+                   "r %s %s %s%s%s %s %d %d\n",
                    rs->nickname,
                    identity64,
-                   digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
                    published,
                    ipaddr,
                    (int)rs->or_port,
@@ -1918,7 +1919,7 @@ routerstatus_format_entry(char *buf, size_t buf_len,
    * this here, instead of in the caller. Then we could use the
    * networkstatus_type_t values, with an additional control port value
    * added -MP */
-  if (format == NS_V3_CONSENSUS)
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
     return 0;
 
   cp = buf + strlen(buf);
@@ -2434,6 +2435,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
   vote_timing_t timing;
   digestmap_t *omit_as_sybil = NULL;
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+  smartlist_t *microdescriptors = NULL;
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -2482,11 +2484,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
   omit_as_sybil = get_possible_sybil_list(routers);
 
   routerstatuses = smartlist_create();
+  microdescriptors = smartlist_create();
 
-  SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     if (ri->cache_info.published_on >= cutoff) {
       routerstatus_t *rs;
       vote_routerstatus_t *vrs;
+      microdesc_t *md;
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
@@ -2501,9 +2505,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
         rs->is_running = 0;
 
       vrs->version = version_from_platform(ri->platform);
+      md = dirvote_create_microdescriptor(ri);
+      if (md) {
+        char buf[128];
+        vote_microdesc_hash_t *h;
+        dirvote_format_microdesc_vote_line(buf, sizeof(buf), md);
+        h = tor_malloc(sizeof(vote_microdesc_hash_t));
+        h->microdesc_hash_line = tor_strdup(buf);
+        h->next = NULL;
+        vrs->microdesc = h;
+        md->last_listed = now;
+        smartlist_add(microdescriptors, md);
+      }
+
       smartlist_add(routerstatuses, vrs);
     }
-  });
+  } SMARTLIST_FOREACH_END(ri);
+
+  {
+    smartlist_t *added =
+      microdescs_add_list_to_cache(get_microdesc_cache(),
+                                   microdescriptors, SAVED_NOWHERE, 0);
+    smartlist_free(added);
+    smartlist_free(microdescriptors);
+  }
 
   smartlist_free(routers);
   digestmap_free(omit_as_sybil, NULL);

src/or/dirvote.c

@@ -24,14 +24,20 @@ static int dirvote_publish_consensus(void);
 static char *make_consensus_method_list(int low, int high, const char *sep);
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 7
+#define MAX_SUPPORTED_CONSENSUS_METHOD 8
 
 #define MIN_METHOD_FOR_PARAMS 7
 
+/** Lowest consensus method that generates microdescriptors */
+#define MIN_METHOD_FOR_MICRODESC 8
+
 /* =====
  * Voting
  * =====*/
 
+/* Overestimated. */
+#define MICRODESC_LINE_LEN 80
+
 /** Return a new string containing the string representation of the vote in
  * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>.
  * For v3 authorities. */
@@ -89,7 +95,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
 
   len = 8192;
   len += strlen(version_lines);
-  len += (RS_ENTRY_LEN)*smartlist_len(rl->routers);
+  len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers);
   len += v3_ns->cert->cache_info.signed_descriptor_len;
 
   status = tor_malloc(len);
@@ -158,15 +164,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
     outp += cert->cache_info.signed_descriptor_len;
   }
 
-  SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs,
-  {
+  SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *,
+                          vrs) {
+    vote_microdesc_hash_t *h;
     if (routerstatus_format_entry(outp, endp-outp, &vrs->status,
                                   vrs->version, NS_V3_VOTE) < 0) {
       log_warn(LD_BUG, "Unable to print router status.");
       goto err;
     }
     outp += strlen(outp);
-  });
+
+    for (h = vrs->microdesc; h; h = h->next) {
+      size_t mlen = strlen(h->microdesc_hash_line);
+      if (outp+mlen >= endp) {
+        log_warn(LD_BUG, "Can't fit microdesc line in vote.");
+      }
+      memcpy(outp, h->microdesc_hash_line, mlen+1);
+      outp += strlen(outp);
+    }
+  } SMARTLIST_FOREACH_END(vrs);
 
   {
     char signing_key_fingerprint[FINGERPRINT_LEN+1];
@@ -189,7 +205,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
     outp += strlen(outp);
   }
 
-  if (router_get_networkstatus_v3_hash(status, digest)<0)
+  if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0)
     goto err;
   note_crypto_pk_op(SIGN_DIR);
   if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN,
@@ -294,34 +310,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min)
 
 /** Given a sorted list of strings <b>lst</b>, return the member that appears
  * most.  Break ties in favor of later-occurring members. */
-static const char *
-get_most_frequent_member(smartlist_t *lst)
-{
-  const char *most_frequent = NULL;
-  int most_frequent_count = 0;
-
-  const char *cur = NULL;
-  int count = 0;
-
-  SMARTLIST_FOREACH(lst, const char *, s,
-  {
-    if (cur && !strcmp(s, cur)) {
-      ++count;
-    } else {
-      if (count >= most_frequent_count) {
-        most_frequent = cur;
-        most_frequent_count = count;
-      }
-      cur = s;
-      count = 1;
-    }
-  });
-  if (count >= most_frequent_count) {
-    most_frequent = cur;
-    most_frequent_count = count;
-  }
-  return most_frequent;
-}
+#define get_most_frequent_member(lst)           \
+  smartlist_get_most_frequent_string(lst)
 
 /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses
  * that come from the same routerinfo, with the same derived elements.
@@ -363,7 +353,8 @@ _compare_vote_rs(const void **_a, const void **_b)
  * in favor of smaller descriptor digest.
  */
 static vote_routerstatus_t *
-compute_routerstatus_consensus(smartlist_t *votes)
+compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
+                               char *microdesc_digest256_out)
 {
   vote_routerstatus_t *most = NULL, *cur = NULL;
   int most_n = 0, cur_n = 0;
@@ -399,6 +390,26 @@ compute_routerstatus_consensus(smartlist_t *votes)
   }
 
   tor_assert(most);
+
+  if (consensus_method >= MIN_METHOD_FOR_MICRODESC &&
+      microdesc_digest256_out) {
+    smartlist_t *digests = smartlist_create();
+    const char *best_microdesc_digest;
+    SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, {
+        char d[DIGEST256_LEN];
+        if (compare_vote_rs(rs, most))
+          continue;
+        if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method))
+          smartlist_add(digests, tor_memdup(d, sizeof(d)));
+    });
+    smartlist_sort_digests256(digests);
+    best_microdesc_digest = smartlist_get_most_frequent_digest256(digests);
+    if (best_microdesc_digest)
+      memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN);
+    SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp));
+    smartlist_free(digests);
+  }
+
   return most;
 }
 
@@ -615,16 +626,20 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                 crypto_pk_env_t *identity_key,
                                 crypto_pk_env_t *signing_key,
                                 const char *legacy_id_key_digest,
-                                crypto_pk_env_t *legacy_signing_key)
+                                crypto_pk_env_t *legacy_signing_key,
+                                consensus_flavor_t flavor)
 {
   smartlist_t *chunks;
   char *result = NULL;
   int consensus_method;
-
   time_t valid_after, fresh_until, valid_until;
   int vote_seconds, dist_seconds;
   char *client_versions = NULL, *server_versions = NULL;
   smartlist_t *flags;
+  const routerstatus_format_type_t rs_format =
+    flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
+
+  tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC);
   tor_assert(total_authorities >= smartlist_len(votes));
 
   if (!smartlist_len(votes)) {
@@ -955,6 +970,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
       int n_listing = 0;
       int i;
       char buf[256];
+      char microdesc_digest[DIGEST256_LEN];
 
       /* Of the next-to-be-considered digest in each voter, which is first? */
       SMARTLIST_FOREACH(votes, networkstatus_t *, v, {
@@ -1021,7 +1037,9 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
       /* Figure out the most popular opinion of what the most recent
        * routerinfo and its contents are. */
-      rs = compute_routerstatus_consensus(matching_descs);
+      memset(microdesc_digest, 0, sizeof(microdesc_digest));
+      rs = compute_routerstatus_consensus(matching_descs, consensus_method,
+                                          microdesc_digest);
       /* Copy bits of that into rs_out. */
       tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN));
       memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN);
@@ -1182,9 +1200,19 @@ networkstatus_compute_consensus(smartlist_t *votes,
       /* Okay!! Now we can write the descriptor... */
       /*     First line goes into "buf". */
       routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL,
-                                NS_V3_CONSENSUS);
+                                rs_format);
       smartlist_add(chunks, tor_strdup(buf));
-      /*     Second line is all flags.  The "\n" is missing. */
+      /*     Now an m line, if applicable. */
+      if (flavor == FLAV_MICRODESC &&
+          !tor_digest256_is_zero(microdesc_digest)) {
+        char m[BASE64_DIGEST256_LEN+1], *cp;
+        const size_t mlen = BASE64_DIGEST256_LEN+5;
+        digest256_to_base64(m, microdesc_digest);
+        cp = tor_malloc(mlen);
+        tor_snprintf(cp, mlen, "m %s\n", m);
+        smartlist_add(chunks, cp);
+      }
+      /*     Next line is all flags.  The "\n" is missing. */
       smartlist_add(chunks,
                     smartlist_join_strings(chosen_flags, " ", 0, NULL));
       /*     Now the version line. */
@@ -1206,7 +1234,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
       };
 
       /*     Now the exitpolicy summary line. */
-      if (rs_out.has_exitsummary) {
+      if (rs_out.has_exitsummary && flavor == FLAV_NS) {
         char buf[MAX_POLICY_LINE_LEN+1];
         int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary);
         if (r<0) {
@@ -2090,7 +2118,8 @@ dirvote_compute_consensus(void)
     consensus_body = networkstatus_compute_consensus(
         votes, n_voters,
         my_cert->identity_key,
-        get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign);
+        get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign,
+        FLAV_NS);
   }
   if (!consensus_body) {
     log_warn(LD_DIR, "Couldn't generate a consensus at all!");
@@ -2389,14 +2418,21 @@ dirvote_get_vote(const char *fp, int flags)
   return NULL;
 }
 
-int
-dirvote_create_microdescriptor(char *out, size_t outlen,
-                               const routerinfo_t *ri)
+/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>.
+ *
+ * XXX Right now, there is only one way to generate microdescriptors from
+ * router descriptors.  This may change in future consensus methods.  If so,
+ * we'll need an internal way to remember which method we used, and ask for a
+ * particular method.
+ **/
+microdesc_t *
+dirvote_create_microdescriptor(const routerinfo_t *ri)
 {
+  microdesc_t *result = NULL;
   char *key = NULL, *summary = NULL, *family = NULL;
+  char buf[1024];
   size_t keylen;
-  int result = -1;
-  char *start = out, *end = out+outlen;
+  char *out = buf, *end = buf+sizeof(buf);
 
   if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0)
     goto done;
@@ -2417,7 +2453,19 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
       goto done;
     out += strlen(out);
   }
-  result = out - start;
+  *out = '\0'; /* Make sure it's nul-terminated.  This should be a no-op */
+
+  {
+    smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1);
+    if (smartlist_len(lst) != 1) {
+      log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse.");
+      SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md));
+      smartlist_free(lst);
+      goto done;
+    }
+    result = smartlist_get(lst, 0);
+    smartlist_free(lst);
+  }
 
  done:
   tor_free(key);
@@ -2426,28 +2474,23 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
   return result;
 }
 
-/** Lowest consensus method that generates microdescriptors */
-#define MIN_CM_MICRODESC 7
 /** Cached space-separated string to hold */
 static char *microdesc_consensus_methods = NULL;
 
+/** DOCDOC */
 int
-dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
-                                         const char *microdesc,
-                                         size_t microdescriptor_len)
+dirvote_format_microdesc_vote_line(char *out, size_t out_len,
+                                   const microdesc_t *md)
 {
-  char d[DIGEST256_LEN];
   char d64[BASE64_DIGEST256_LEN];
   if (!microdesc_consensus_methods) {
     microdesc_consensus_methods =
-      make_consensus_method_list(MIN_CM_MICRODESC,
+      make_consensus_method_list(MIN_METHOD_FOR_MICRODESC,
                                  MAX_SUPPORTED_CONSENSUS_METHOD,
                                  ",");
     tor_assert(microdesc_consensus_methods);
   }
-  if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0)
-    return -1;
-  if (digest256_to_base64(d64, d)<0)
+  if (digest256_to_base64(d64, md->digest)<0)
     return -1;
 
   if (tor_snprintf(out, out_len, "m %s sha256=%s\n",
@@ -2457,3 +2500,44 @@ dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
   return strlen(out);
 }
 
+/** DOCDOC */
+int
+vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                      const vote_routerstatus_t *vrs,
+                                      int method)
+{
+  /* XXXX only returns the sha256 method. */
+  const vote_microdesc_hash_t *h;
+  char mstr[64];
+  size_t mlen;
+
+  tor_snprintf(mstr, sizeof(mstr), "%d", method);
+  mlen = strlen(mstr);
+
+  for (h = vrs->microdesc; h; h = h->next) {
+    const char *cp = h->microdesc_hash_line;
+    size_t num_len;
+    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
+     * the first part. */
+    while (1) {
+      num_len = strspn(cp, "1234567890");
+      if (num_len == mlen && !memcmp(mstr, cp, mlen)) {
+        /* This is the line. */
+        char buf[BASE64_DIGEST256_LEN+1];
+        /* XXXX ignores extraneous stuff if the digest is too long.  This
+         * seems harmless enough, right? */
+        cp = strstr(cp, " sha256=");
+        if (!cp)
+          return -1;
+        cp += strlen(" sha256=");
+        strlcpy(buf, cp, sizeof(buf));
+        return digest256_from_base64(digest256_out, buf);
+      }
+      if (num_len == 0 || cp[num_len] != ',')
+        break;
+      cp += num_len + 1;
+    }
+  }
+  return -1;
+}
+

src/or/microdesc.c

@@ -0,0 +1,267 @@
+/* Copyright (c) 2009, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+/** DOCDOC everything here. */
+
+#define MICRODESC_IN_CONSENSUS 1
+#define MICRODESC_IN_VOTE 2
+
+struct microdesc_cache_t {
+  HT_HEAD(microdesc_map, microdesc_t) map;
+
+  char *cache_fname;
+  char *journal_fname;
+  tor_mmap_t *cache_content;
+  size_t journal_len;
+};
+
+static INLINE unsigned int
+_microdesc_hash(microdesc_t *md)
+{
+  unsigned *d = (unsigned*)md->digest;
+#if SIZEOF_INT == 4
+  return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7];
+#else
+  return d[0] ^ d[1] ^ d[2] ^ d[3];
+#endif
+}
+
+static INLINE int
+_microdesc_eq(microdesc_t *a, microdesc_t *b)
+{
+  return !memcmp(a->digest, b->digest, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(microdesc_map, microdesc_t, node,
+             _microdesc_hash, _microdesc_eq);
+HT_GENERATE(microdesc_map, microdesc_t, node,
+             _microdesc_hash, _microdesc_eq, 0.6,
+             _tor_malloc, _tor_realloc, _tor_free);
+
+static int
+dump_microdescriptor(FILE *f, microdesc_t *md)
+{
+  /* XXXX drops unkown annotations. */
+  if (md->last_listed) {
+    char buf[ISO_TIME_LEN+1];
+    format_iso_time(buf, md->last_listed);
+    fprintf(f, "@last-listed %s\n", buf);
+  }
+
+  md->off = (off_t) ftell(f);
+  fwrite(md->body, 1, md->bodylen, f);
+  return 0;
+}
+
+static microdesc_cache_t *the_microdesc_cache = NULL;
+
+microdesc_cache_t *
+get_microdesc_cache(void)
+{
+  if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
+    microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
+    HT_INIT(microdesc_map, &cache->map);
+    cache->cache_fname = get_datadir_fname("cached-microdescs");
+    cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+    microdesc_cache_reload(cache);
+    the_microdesc_cache = cache;
+  }
+  return the_microdesc_cache;
+}
+
+/* There are three sources of microdescriptors:
+   1) Generated us while acting as a directory authority.
+   2) Loaded from the cache on disk.
+   3) Downloaded.
+*/
+
+/* Returns list of added microdesc_t. */
+smartlist_t *
+microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save)
+{
+  /*XXXX need an argument that sets last_listed as appropriate. */
+
+  smartlist_t *descriptors, *added;
+  const int allow_annotations = (where != SAVED_NOWHERE);
+  const int copy_body = (where != SAVED_IN_CACHE);
+
+  descriptors = microdescs_parse_from_string(s, eos,
+                                             allow_annotations,
+                                             copy_body);
+
+  added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
+  smartlist_free(descriptors);
+  return added;
+}
+
+/* Returns list of added microdesc_t. Frees any not added. */
+smartlist_t *
+microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                             smartlist_t *descriptors, saved_location_t where,
+                             int no_save)
+{
+  smartlist_t *added;
+  open_file_t *open_file = NULL;
+  FILE *f = NULL;
+  //  int n_added = 0;
+
+  if (where == SAVED_NOWHERE && !no_save) {
+    f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND,
+                                    0600, &open_file);
+    if (!f)
+      log_warn(LD_DIR, "Couldn't append to journal in %s",
+               cache->journal_fname);
+  }
+
+  added = smartlist_create();
+  SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+    microdesc_t *md2;
+    md2 = HT_FIND(microdesc_map, &cache->map, md);
+    if (md2) {
+      /* We already had this one. */
+      if (md2->last_listed < md->last_listed)
+        md2->last_listed = md->last_listed;
+      microdesc_free(md);
+      continue;
+    }
+
+    /* Okay, it's a new one. */
+    if (f) {
+      dump_microdescriptor(f, md);
+      md->saved_location = SAVED_IN_JOURNAL;
+    } else {
+      md->saved_location = where;
+    }
+
+    md->no_save = no_save;
+
+    HT_INSERT(microdesc_map, &cache->map, md);
+    smartlist_add(added, md);
+  } SMARTLIST_FOREACH_END(md);
+
+  finish_writing_to_file(open_file); /*XXX Check me.*/
+  return added;
+}
+
+void
+microdesc_cache_clear(microdesc_cache_t *cache)
+{
+  microdesc_t **entry, **next;
+  for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
+    next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
+    microdesc_free(*entry);
+  }
+  if (cache->cache_content) {
+    tor_munmap_file(cache->cache_content);
+    cache->cache_content = NULL;
+  }
+}
+
+int
+microdesc_cache_reload(microdesc_cache_t *cache)
+{
+  struct stat st;
+  char *journal_content;
+  smartlist_t *added;
+  tor_mmap_t *mm;
+  int total = 0;
+
+  microdesc_cache_clear(cache);
+
+  mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
+  if (mm) {
+    added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
+                                    SAVED_IN_CACHE, 0);
+    total += smartlist_len(added);
+    smartlist_free(added);
+  }
+
+  journal_content = read_file_to_str(cache->journal_fname,
+                                     RFTS_IGNORE_MISSING, &st);
+  if (journal_content) {
+    added = microdescs_add_to_cache(cache, journal_content,
+                                    journal_content+st.st_size,
+                                    SAVED_IN_JOURNAL, 0);
+    total += smartlist_len(added);
+    smartlist_free(added);
+    tor_free(journal_content);
+  }
+  log_notice(LD_DIR, "Reloaded microdescriptor cache.  Found %d descriptors.",
+             total);
+  return 0;
+}
+
+int
+microdesc_cache_rebuild(microdesc_cache_t *cache)
+{
+  open_file_t *open_file;
+  FILE *f;
+  microdesc_t **mdp;
+  smartlist_t *wrote;
+
+  f = start_writing_to_stdio_file(cache->cache_fname, OPEN_FLAGS_REPLACE,
+                                  0600, &open_file);
+  if (!f)
+    return -1;
+
+  wrote = smartlist_create();
+
+  HT_FOREACH(mdp, microdesc_map, &cache->map) {
+    microdesc_t *md = *mdp;
+    if (md->no_save)
+      continue;
+
+    dump_microdescriptor(f, md);
+    if (md->saved_location != SAVED_IN_CACHE) {
+      tor_free(md->body);
+      md->saved_location = SAVED_IN_CACHE;
+    }
+
+    smartlist_add(wrote, md);
+  }
+
+  finish_writing_to_file(open_file); /*XXX Check me.*/
+
+  if (cache->cache_content)
+    tor_munmap_file(cache->cache_content);
+  cache->cache_content = tor_mmap_file(cache->cache_fname);
+  if (!cache->cache_content && smartlist_len(wrote)) {
+    log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
+            cache->cache_fname);
+    return -1;
+  }
+  SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
+    if (md->no_save)
+      continue;
+    tor_assert(md->saved_location == SAVED_IN_CACHE);
+    md->body = (char*)cache->cache_content->data + md->off;
+    tor_assert(!memcmp(md->body, "onion-key", 9));
+  } SMARTLIST_FOREACH_END(wrote);
+
+  smartlist_free(wrote);
+
+  return 0;
+}
+
+void
+microdesc_free(microdesc_t *md)
+{
+  /* Must be removed from hash table! */
+  if (md->onion_pkey)
+    crypto_free_pk_env(md->onion_pkey);
+  if (md->body && md->saved_location != SAVED_IN_CACHE)
+    tor_free(md->body);
+
+  if (md->family) {
+    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
+    smartlist_free(md->family);
+  }
+  tor_free(md->exitsummary);
+
+  tor_free(md);
+}
+

src/or/networkstatus.c

@@ -242,8 +242,14 @@ router_reload_consensus_networkstatus(void)
 static void
 vote_routerstatus_free(vote_routerstatus_t *rs)
 {
+  vote_microdesc_hash_t *h, *next;
   tor_free(rs->version);
   tor_free(rs->status.exitsummary);
+  for (h = rs->microdesc; h; h = next) {
+    tor_free(h->microdesc_hash_line);
+    next = h->next;
+    tor_free(h);
+  }
   tor_free(rs);
 }
 

src/or/or.h

@@ -89,6 +89,7 @@
 #include "torgzip.h"
 #include "address.h"
 #include "compat_libevent.h"
+#include "ht.h"
 
 /* These signals are defined to help control_signal_act work.
  */
@@ -1558,12 +1559,29 @@ typedef struct routerstatus_t {
 } routerstatus_t;
 
 /**DOCDOC*/
-typedef struct microdescriptor_t {
+typedef struct microdesc_t {
+  HT_ENTRY(microdesc_t) node;
+
+  /* Cache information */
+
+  time_t last_listed;
+  saved_location_t saved_location : 3;
+  unsigned int no_save : 1;
+  off_t off;
+
+  /* The string containing the microdesc. */
+
+  char *body;
+  size_t bodylen;
+  char digest[DIGEST256_LEN];
+
+  /* Fields in the microdescriptor. */
+
   crypto_pk_env_t *onion_pkey;
   smartlist_t *family;
   char *exitsummary; /**< exit policy summary -
-                      * XXX weasel: this probably should not stay a string. */
-} microdescriptor_t;
+                      * XXX this probably should not stay a string. */
+} microdesc_t;
 
 /** How many times will we try to download a router's descriptor before giving
  * up? */
@@ -3748,7 +3766,8 @@ int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
 size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
                                   int compressed);
 typedef enum {
-  NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT
+  NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT,
+  NS_V3_CONSENSUS_MICRODESC
 } routerstatus_format_type_t;
 int routerstatus_format_entry(char *buf, size_t buf_len,
                               routerstatus_t *rs, const char *platform,
@@ -3784,13 +3803,20 @@ int dirserv_read_measured_bandwidths(const char *from_file,
 
 void dirvote_free_all(void);
 
+/** DOCDOC */
+typedef enum {
+  FLAV_NS,
+  FLAV_MICRODESC,
+} consensus_flavor_t;
+
 /* vote manipulation */
 char *networkstatus_compute_consensus(smartlist_t *votes,
                                       int total_authorities,
                                       crypto_pk_env_t *identity_key,
                                       crypto_pk_env_t *signing_key,
                                       const char *legacy_identity_key_digest,
-                                      crypto_pk_env_t *legacy_signing_key);
+                                      crypto_pk_env_t *legacy_signing_key,
+                                      consensus_flavor_t flavor);
 int networkstatus_add_detached_signatures(networkstatus_t *target,
                                           ns_detached_signatures_t *sigs,
                                           const char **msg_out);
@@ -3837,11 +3863,12 @@ networkstatus_t *
 dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
                                         authority_cert_t *cert);
 
-int dirvote_create_microdescriptor(char *out,
-                                   size_t outlen, const routerinfo_t *ri);
-int dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
-                                             const char *microdesc,
-                                             size_t microdescriptor_len);
+microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri);
+int dirvote_format_microdesc_vote_line(char *out, size_t out_len,
+                                       const microdesc_t *md);
+int vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                          const vote_routerstatus_t *vrs,
+                                          int method);
 
 #ifdef DIRVOTE_PRIVATE
 char *format_networkstatus_vote(crypto_pk_env_t *private_key,
@@ -4051,6 +4078,25 @@ void do_hash_password(void);
 int tor_init(int argc, char **argv);
 #endif
 
+/********************************* microdesc.c *************************/
+
+typedef struct microdesc_cache_t microdesc_cache_t;
+
+microdesc_cache_t *get_microdesc_cache(void);
+
+smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
+                        const char *s, const char *eos, saved_location_t where,
+                        int no_save);
+smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
+                        smartlist_t *descriptors, saved_location_t where,
+                        int no_save);
+
+int microdesc_cache_rebuild(microdesc_cache_t *cache);
+int microdesc_cache_reload(microdesc_cache_t *cache);
+void microdesc_cache_clear(microdesc_cache_t *cache);
+
+void microdesc_free(microdesc_t *md);
+
 /********************************* networkstatus.c *********************/
 
 /** How old do we allow a v2 network-status to get before removing it
@@ -4927,7 +4973,8 @@ int router_get_router_hash(const char *s, char *digest);
 int router_get_dir_hash(const char *s, char *digest);
 int router_get_runningrouters_hash(const char *s, char *digest);
 int router_get_networkstatus_v2_hash(const char *s, char *digest);
-int router_get_networkstatus_v3_hash(const char *s, char *digest);
+int router_get_networkstatus_v3_hash(const char *s, char *digest,
+                                     digest_algorithm_t algorithm);
 int router_get_extrainfo_hash(const char *s, char *digest);
 int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
@@ -4971,6 +5018,10 @@ networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
 ns_detached_signatures_t *networkstatus_parse_detached_signatures(
                                           const char *s, const char *eos);
 
+smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
+                                          int allow_annotations,
+                                          int copy_body);
+
 authority_cert_t *authority_cert_parse_from_string(const char *s,
                                                    const char **end_of_string);
 int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,

src/or/routerparse.c

@@ -111,6 +111,7 @@ typedef enum {
   K_LEGACY_DIR_KEY,
 
   A_PURPOSE,
+  A_LAST_LISTED,
   _A_UNKNOWN,
 
   R_RENDEZVOUS_SERVICE_DESCRIPTOR,
@@ -496,6 +497,14 @@ static token_rule_t networkstatus_detached_signature_token_table[] = {
   END_OF_TABLE
 };
 
+static token_rule_t microdesc_token_table[] = {
+  T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
+  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
+  A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
+  END_OF_TABLE
+};
+
 #undef T
 
 /* static function prototypes */
@@ -505,7 +514,8 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
 
 static int router_get_hash_impl(const char *s, char *digest,
                                 const char *start_str, const char *end_str,
-                                char end_char);
+                                char end_char,
+                                digest_algorithm_t alg);
 static void token_free(directory_token_t *tok);
 static smartlist_t *find_all_exitpolicy(smartlist_t *s);
 static directory_token_t *_find_by_keyword(smartlist_t *s,
@@ -586,7 +596,8 @@ int
 router_get_dir_hash(const char *s, char *digest)
 {
   return router_get_hash_impl(s,digest,
-                              "signed-directory","\ndirectory-signature",'\n');
+                              "signed-directory","\ndirectory-signature",'\n',
+                              DIGEST_SHA1);
 }
 
 /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in
@@ -596,7 +607,8 @@ int
 router_get_router_hash(const char *s, char *digest)
 {
   return router_get_hash_impl(s,digest,
-                              "router ","\nrouter-signature", '\n');
+                              "router ","\nrouter-signature", '\n',
+                              DIGEST_SHA1);
 }
 
 /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers
@@ -606,7 +618,8 @@ int
 router_get_runningrouters_hash(const char *s, char *digest)
 {
   return router_get_hash_impl(s,digest,
-                              "network-status","\ndirectory-signature", '\n');
+                              "network-status","\ndirectory-signature", '\n',
+                              DIGEST_SHA1);
 }
 
 /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status
@@ -616,17 +629,19 @@ router_get_networkstatus_v2_hash(const char *s, char *digest)
 {
   return router_get_hash_impl(s,digest,
                               "network-status-version","\ndirectory-signature",
-                              '\n');
+                              '\n',
+                              DIGEST_SHA1);
 }
 
 /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status
  * string in <b>s</b>.  Return 0 on success, -1 on failure. */
 int
-router_get_networkstatus_v3_hash(const char *s, char *digest)
+router_get_networkstatus_v3_hash(const char *s, char *digest,
+                                 digest_algorithm_t alg)
 {
   return router_get_hash_impl(s,digest,
                               "network-status-version","\ndirectory-signature",
-                              ' ');
+                              ' ', alg);
 }
 
 /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo
@@ -634,7 +649,8 @@ router_get_networkstatus_v3_hash(const char *s, char *digest)
 int
 router_get_extrainfo_hash(const char *s, char *digest)
 {
-  return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n');
+  return router_get_hash_impl(s,digest,"extra-info","\nrouter-signature",'\n',
+                              DIGEST_SHA1);
 }
 
 /** Helper: used to generate signatures for routers, directories and
@@ -643,6 +659,8 @@ router_get_extrainfo_hash(const char *s, char *digest)
  * surround it with -----BEGIN/END----- pairs, and write it to the
  * <b>buf_len</b>-byte buffer at <b>buf</b>.  Return 0 on success, -1 on
  * failure.
+ *
+ * DOCDOC alg
  */
 int
 router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
@@ -1691,7 +1709,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
     goto err;
   }
   if (router_get_hash_impl(s, digest, "dir-key-certificate-version",
-                           "\ndir-key-certification", '\n') < 0)
+                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
     goto err;
   tok = smartlist_get(tokens, 0);
   if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
@@ -2298,7 +2316,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hash(s, ns_digest)) {
+  if (router_get_networkstatus_v3_hash(s, ns_digest, DIGEST_SHA1)) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
@@ -3410,7 +3428,7 @@ find_all_exitpolicy(smartlist_t *s)
   return out;
 }
 
-/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first
+/** Compute the digest of the substring of <b>s</b> taken from the first
  * occurrence of <b>start_str</b> through the first instance of c after the
  * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in
  * <b>digest</b>; return 0 on success.
@@ -3420,7 +3438,8 @@ find_all_exitpolicy(smartlist_t *s)
 static int
 router_get_hash_impl(const char *s, char *digest,
                      const char *start_str,
-                     const char *end_str, char end_c)
+                     const char *end_str, char end_c,
+                     digest_algorithm_t alg)
 {
   char *start, *end;
   start = strstr(s, start_str);
@@ -3446,14 +3465,169 @@ router_get_hash_impl(const char *s, char *digest,
   }
   ++end;
 
-  if (crypto_digest(digest, start, end-start)) {
-    log_warn(LD_BUG,"couldn't compute digest");
-    return -1;
+  if (alg == DIGEST_SHA1) {
+    if (crypto_digest(digest, start, end-start)) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
+  } else {
+    if (crypto_digest256(digest, start, end-start, alg)) {
+      log_warn(LD_BUG,"couldn't compute digest");
+      return -1;
+    }
   }
 
   return 0;
 }
 
+/** DOCDOC Assuming that s starts with a microdesc, return the start of the
+ * *NEXT* one. */
+static const char *
+find_start_of_next_microdesc(const char *s, const char *eos)
+{
+  int started_with_annotations;
+  s = eat_whitespace_eos(s, eos);
+  if (!s)
+    return NULL;
+
+#define CHECK_LENGTH() STMT_BEGIN \
+    if (s+32 > eos)               \
+      return NULL;                \
+  STMT_END
+
+#define NEXT_LINE() STMT_BEGIN            \
+    s = memchr(s, '\n', eos-s);           \
+    if (!s || s+1 >= eos)                 \
+      return NULL;                        \
+    s++;                                  \
+  STMT_END
+
+  CHECK_LENGTH();
+
+  started_with_annotations = (*s == '@');
+
+  if (started_with_annotations) {
+    /* Start by advancing to the first non-annotation line. */
+    while (*s == '@')
+      NEXT_LINE();
+  }
+  CHECK_LENGTH();
+
+  /* Now we should be pointed at an onion-key line.  If we are, then skip
+   * it. */
+  if (!strcmpstart(s, "onion-key"))
+    NEXT_LINE();
+
+  /* Okay, now we're pointed at the first line of the microdescriptor which is
+     not an annotation or onion-key.  The next line that _is_ an annotation or
+     onion-key is the start of the next microdescriptor. */
+  while (s+32 < eos) {
+    if (*s == '@' || !strcmpstart(s, "onion-key"))
+      return s;
+    NEXT_LINE();
+  }
+  return NULL;
+
+#undef CHECK_LENGTH
+#undef NEXT_LINE
+}
+
+/**DOCDOC*/
+smartlist_t *
+microdescs_parse_from_string(const char *s, const char *eos,
+                             int allow_annotations, int copy_body)
+{
+  smartlist_t *tokens;
+  smartlist_t *result;
+  microdesc_t *md = NULL;
+  memarea_t *area;
+  const char *start = s;
+  const char *start_of_next_microdesc;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+
+  directory_token_t *tok;
+
+  if (!eos)
+    eos = s + strlen(s);
+
+  s = eat_whitespace_eos(s, eos);
+  area = memarea_new();
+  result = smartlist_create();
+  tokens = smartlist_create();
+
+  while (s < eos) {
+    start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
+    if (!start_of_next_microdesc)
+      start_of_next_microdesc = eos;
+
+    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                        microdesc_token_table, flags)) {
+      log_warn(LD_DIR, "Unparseable microdescriptor");
+      goto next;
+    }
+
+    md = tor_malloc_zero(sizeof(microdesc_t));
+    {
+      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
+                                  "onion-key");
+      tor_assert(cp);
+
+      md->bodylen = start_of_next_microdesc - cp;
+      if (copy_body)
+        md->body = tor_strndup(cp, md->bodylen);
+      else
+        md->body = (char*)cp;
+      md->off = cp - start;
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+      if (parse_iso_time(tok->args[0], &md->last_listed)) {
+        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+        goto next;
+      }
+    }
+
+    tok = find_by_keyword(tokens, K_ONION_KEY);
+    md->onion_pkey = tok->key;
+    tok->key = NULL;
+
+    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+      int i;
+      md->family = smartlist_create();
+      for (i=0;i<tok->n_args;++i) {
+        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
+          log_warn(LD_DIR, "Illegal nickname %s in family line",
+                   escaped(tok->args[i]));
+          goto next;
+        }
+        smartlist_add(md->family, tor_strdup(tok->args[i]));
+      }
+    }
+
+    if ((tok = find_opt_by_keyword(tokens, K_P))) {
+      md->exitsummary = tor_strdup(tok->args[0]);
+    }
+
+    crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+    smartlist_add(result, md);
+
+    md = NULL;
+  next:
+    if (md)
+      microdesc_free(md);
+
+    memarea_clear(area);
+    smartlist_clear(tokens);
+    s = start_of_next_microdesc;
+  }
+
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+
+  return result;
+}
+
 /** Parse the Tor version of the platform string <b>platform</b>,
  * and compare it to the version in <b>cutoff</b>. Return 1 if
  * the router is at least as new as the cutoff, else return 0.
@@ -3712,7 +3886,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
   /* Compute descriptor hash for later validation. */
   if (router_get_hash_impl(desc, desc_hash,
                            "rendezvous-service-descriptor ",
-                           "\nsignature", '\n') < 0) {
+                           "\nsignature", '\n', DIGEST_SHA1) < 0) {
     log_warn(LD_REND, "Couldn't compute descriptor hash.");
     goto err;
   }

src/test/test_dir.c

@@ -858,7 +858,8 @@ test_dir_v3_networkstatus(void)
                                                    cert3->identity_key,
                                                    sign_skey_3,
                                                    "AAAAAAAAAAAAAAAAAAAA",
-                                                   sign_skey_leg1);
+                                                   sign_skey_leg1,
+                                                   FLAV_NS);
   test_assert(consensus_text);
   con = networkstatus_parse_vote_from_string(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
@@ -966,11 +967,13 @@ test_dir_v3_networkstatus(void)
     smartlist_shuffle(votes);
     consensus_text2 = networkstatus_compute_consensus(votes, 3,
                                                       cert2->identity_key,
-                                                      sign_skey_2, NULL,NULL);
+                                                      sign_skey_2, NULL,NULL,
+                                                      FLAV_NS);
     smartlist_shuffle(votes);
     consensus_text3 = networkstatus_compute_consensus(votes, 3,
                                                       cert1->identity_key,
-                                                      sign_skey_1, NULL,NULL);
+                                                      sign_skey_1, NULL,NULL,
+                                                      FLAV_NS);
     test_assert(consensus_text2);
     test_assert(consensus_text3);
     con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,