Merge branch 'microdesc_dl_v2'

changes/microdesc_dl

@@ -0,0 +1,4 @@
+  o Major features:
+    - Caches now download and cache all the consensus flavors that
+      they know about.  This allows them to assess which microdescriptors
+      they need to fetch.

src/or/connection.c

@@ -3397,6 +3397,33 @@ connection_get_by_type_state_rendquery(int type, int state,
   return NULL;
 }
 
+/** Return a directory connection (if any one exists) that is fetching
+ * the item described by <b>state</b>/<b>resource</b> */
+dir_connection_t *
+connection_dir_get_by_purpose_and_resource(int purpose,
+                                           const char *resource)
+{
+  smartlist_t *conns = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    dir_connection_t *dirconn;
+    if (conn->type != CONN_TYPE_DIR || conn->marked_for_close ||
+        conn->purpose != purpose)
+      continue;
+    dirconn = TO_DIR_CONN(conn);
+    if (dirconn->requested_resource == NULL) {
+      if (resource == NULL)
+        return dirconn;
+    } else if (resource) {
+      if (0 == strcmp(resource, dirconn->requested_resource))
+        return dirconn;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  return NULL;
+}
+
+
 /** Return an open, non-marked connection of a given type and purpose, or NULL
  * if no such connection exists. */
 connection_t *

src/or/connection.h

@@ -124,6 +124,8 @@ connection_t *connection_get_by_type_addr_port_purpose(int type,
 connection_t *connection_get_by_type_state(int type, int state);
 connection_t *connection_get_by_type_state_rendquery(int type, int state,
                                                      const char *rendquery);
+dir_connection_t *connection_dir_get_by_purpose_and_resource(
+                                           int state, const char *resource);
 
 #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR)
 int connection_is_listener(connection_t *conn);

src/or/directory.c

@@ -15,6 +15,7 @@
 #include "dirvote.h"
 #include "geoip.h"
 #include "main.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "policies.h"
 #include "rendclient.h"
@@ -78,6 +79,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
                                            int router_purpose,
                                            int was_extrainfo,
                                            int was_descriptor_digests);
+static void dir_microdesc_download_failed(smartlist_t *failed,
+                                          int status_code);
 static void note_client_request(int purpose, int compressed, size_t bytes);
 static int client_likes_consensus(networkstatus_t *v, const char *want_url);
 
@@ -137,7 +140,8 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
       dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
       dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
       dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
+      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+      dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
     return 0;
   return 1;
 }
@@ -201,6 +205,8 @@ dir_conn_purpose_to_string(int purpose)
       return "hidden-service v2 descriptor fetch";
     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
       return "hidden-service v2 descriptor upload";
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return "microdescriptor fetch";
     }
 
   log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
@@ -355,15 +361,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
     case DIR_PURPOSE_FETCH_CERTIFICATE:
       type = V3_AUTHORITY;
       break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      type = V3_AUTHORITY;
+      break;
     default:
       log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
       return;
   }
 
   if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    networkstatus_t *v = networkstatus_get_latest_consensus();
-    if (v)
-      if_modified_since = v->valid_after + 180;
+    int flav = FLAV_NS;
+    networkstatus_t *v;
+    if (resource)
+      flav = networkstatus_parse_flavor_name(resource);
+
+    if (flav != -1) {
+      /* IF we have a parsed consensus of this type, we can do an
+       * if-modified-time based on it. */
+      v = networkstatus_get_latest_consensus_by_flavor(flav);
+      if (v)
+        if_modified_since = v->valid_after + 180;
+    } else {
+      /* Otherwise it might be a consensus we don't parse, but which we
+       * do cache.  Look at the cached copy, perhaps. */
+      cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns");
+      if (cd)
+        if_modified_since = cd->published + 180;
+    }
   }
 
   if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
@@ -395,7 +419,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
       if (prefer_authority || type == BRIDGE_AUTHORITY) {
         /* only ask authdirservers, and don't ask myself */
         rs = router_pick_trusteddirserver(type, pds_flags);
-        if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) {
+        if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                                        PDS_NO_EXISTING_MICRODESC_FETCH))) {
           /* We don't want to fetch from any authorities that we're currently
            * fetching server descriptors from, and we got no match.  Did we
            * get no match because all the authorities have connections
@@ -403,7 +428,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
            * return,) or because all the authorities are down or on fire or
            * unreachable or something (in which case we should go on with
            * our fallback code)? */
-          pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH;
+          pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH);
           rs = router_pick_trusteddirserver(type, pds_flags);
           if (rs) {
             log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
@@ -592,15 +618,19 @@ connection_dir_request_failed(dir_connection_t *conn)
     connection_dir_download_networkstatus_failed(conn, -1);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
              conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
-    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
+    log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
+             "directory server at '%s'; retrying",
              conn->_base.address);
     if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
       connection_dir_bridge_routerdesc_failed(conn);
     connection_dir_download_routerdesc_failed(conn);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    networkstatus_consensus_download_failed(0);
+    const char *flavname =
+      conn->requested_resource ? conn->requested_resource : "ns";
+    networkstatus_consensus_download_failed(0, flavname);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
+    log_info(LD_DIR, "Giving up on certificate fetch from directory server "
+             "at '%s'; retrying",
              conn->_base.address);
     connection_dir_download_cert_failed(conn, 0);
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
@@ -609,6 +639,10 @@ connection_dir_request_failed(dir_connection_t *conn)
   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
     log_info(LD_DIR, "Giving up downloading votes from '%s'",
              conn->_base.address);
+  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    log_info(LD_DIR, "Giving up on downloading microdescriptors from "
+             " directory server at '%s'; will retry", conn->_base.address);
+    connection_dir_download_routerdesc_failed(conn);
   }
 }
 
@@ -679,7 +713,8 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn)
   /* No need to relaunch descriptor downloads here: we already do it
    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
   tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+             conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
 
   (void) conn;
 }
@@ -955,12 +990,16 @@ _compare_strs(const void **a, const void **b)
  * This url depends on whether or not the server we go to
  * is sufficiently new to support conditional consensus downloading,
  * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
+ *
+ * If 'resource' is provided, it is the name of a consensus flavor to request.
  */
 static char *
-directory_get_consensus_url(int supports_conditional_consensus)
+directory_get_consensus_url(int supports_conditional_consensus,
+                            const char *resource)
 {
-  char *url;
-  size_t len;
+  char *url = NULL;
+  const char *hyphen = resource ? "-" : "";
+  const char *flavor = resource ? resource : "";
 
   if (supports_conditional_consensus) {
     char *authority_id_list;
@@ -982,16 +1021,15 @@ directory_get_consensus_url(int supports_conditional_consensus)
     authority_id_list = smartlist_join_strings(authority_digests,
                                                "+", 0, NULL);
 
-    len = strlen(authority_id_list)+64;
-    url = tor_malloc(len);
-    tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
-                 authority_id_list);
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
+                 hyphen, flavor, authority_id_list);
 
     SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
     smartlist_free(authority_digests);
     tor_free(authority_id_list);
   } else {
-    url = tor_strdup("/tor/status-vote/current/consensus.z");
+    tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
+                 hyphen, flavor);
   }
   return url;
 }
@@ -1072,10 +1110,11 @@ directory_send_command(dir_connection_t *conn,
       tor_snprintf(url, len, "/tor/status/%s", resource);
       break;
     case DIR_PURPOSE_FETCH_CONSENSUS:
-      tor_assert(!resource);
+      /* resource is optional.  If present, it's a flavor name */
       tor_assert(!payload);
       httpcommand = "GET";
-      url = directory_get_consensus_url(supports_conditional_consensus);
+      url = directory_get_consensus_url(supports_conditional_consensus,
+                                        resource);
       log_info(LD_DIR, "Downloading consensus from %s using %s",
                hoststring, url);
       break;
@@ -1115,6 +1154,11 @@ directory_send_command(dir_connection_t *conn,
       url = tor_malloc(len);
       tor_snprintf(url, len, "/tor/extra/%s", resource);
       break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      tor_assert(resource);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/micro/%s.z", resource);
+      break;
     case DIR_PURPOSE_UPLOAD_DIR:
       tor_assert(!resource);
       tor_assert(payload);
@@ -1390,6 +1434,9 @@ body_is_plausible(const char *body, size_t len, int purpose)
     return 1; /* empty bodies don't need decompression */
   if (len < 32)
     return 0;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    return (!strcmpstart(body,"onion-key"));
+  }
   if (purpose != DIR_PURPOSE_FETCH_RENDDESC) {
     if (!strcmpstart(body,"router") ||
         !strcmpstart(body,"signed-directory") ||
@@ -1466,7 +1513,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
   int plausible;
   int skewed=0;
   int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-                       conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+                       conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
   int was_compressed=0;
   time_t now = time(NULL);
 
@@ -1690,6 +1738,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
 
   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
     int r;
+    const char *flavname =
+      conn->requested_resource ? conn->requested_resource : "ns";
     if (status_code != 200) {
       int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
       log(severity, LD_DIR,
@@ -1698,22 +1748,24 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
            status_code, escaped(reason), conn->_base.address,
            conn->_base.port);
       tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(status_code);
+      networkstatus_consensus_download_failed(status_code, flavname);
       return -1;
     }
     log_info(LD_DIR,"Received consensus directory (size %d) from server "
              "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
-    if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
+    if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
       log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
-             "Unable to load consensus directory downloaded from "
+             "Unable to load %s consensus directory downloaded from "
              "server '%s:%d'. I'll try again soon.",
-             conn->_base.address, conn->_base.port);
+             flavname, conn->_base.address, conn->_base.port);
       tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(0);
+      networkstatus_consensus_download_failed(0, flavname);
       return -1;
     }
     /* launches router downloads as needed */
     routers_update_all_from_networkstatus(now, 3);
+    update_microdescs_from_networkstatus(now);
+    update_microdesc_downloads(now);
     directory_info_has_arrived(now, 0);
     log_info(LD_DIR, "Successfully loaded consensus.");
   }
@@ -1863,6 +1915,41 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
     if (directory_conn_is_self_reachability_test(conn))
       router_dirport_found_reachable();
   }
+  if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    smartlist_t *which = NULL;
+    log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+             "size %d) from server '%s:%d'",
+             status_code, (int)body_len, conn->_base.address, conn->_base.port);
+    tor_assert(conn->requested_resource &&
+               !strcmpstart(conn->requested_resource, "d/"));
+    which = smartlist_create();
+    dir_split_resource_into_fingerprints(conn->requested_resource+2,
+                                         which, NULL,
+                                         DSR_DIGEST256|DSR_BASE64);
+    if (status_code != 200) {
+      log_info(LD_DIR, "Received status code %d (%s) from server "
+               "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
+               "soon.",
+               status_code, escaped(reason), conn->_base.address,
+               (int)conn->_base.port, conn->requested_resource);
+      dir_microdesc_download_failed(which, status_code);
+      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+      smartlist_free(which);
+      tor_free(body); tor_free(headers); tor_free(reason);
+      return 0;
+    } else {
+      smartlist_t *mds;
+      mds = microdescs_add_to_cache(get_microdesc_cache(),
+                                    body, body+body_len, SAVED_NOWHERE, 0,
+                                    now, which);
+      if (smartlist_len(which)) {
+        /* Mark remaining ones as failed. */
+        dir_microdesc_download_failed(which, status_code);
+      }
+      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+      smartlist_free(which);
+    }
+  }
 
   if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_DIR) {
     switch (status_code) {
@@ -3589,6 +3676,36 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
 }
 
+/* DOCDOC NM */
+static void
+dir_microdesc_download_failed(smartlist_t *failed,
+                              int status_code)
+{
+  networkstatus_t *consensus
+    = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
+  routerstatus_t *rs;
+  download_status_t *dls;
+  time_t now = time(NULL);
+  int server = directory_fetches_from_authorities(get_options());
+
+  if (! consensus)
+    return;
+  SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
+    rs = router_get_consensus_status_by_descriptor_digest(consensus, d);
+    if (!rs)
+      continue;
+    dls = &rs->dl_status;
+    if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES)
+      continue;
+    {
+      char buf[BASE64_DIGEST256_LEN+1];
+      digest256_to_base64(buf, d);
+      download_status_increment_failure(dls, status_code, buf,
+                                        server, now);
+    }
+  } SMARTLIST_FOREACH_END(d);
+}
+
 /** Helper.  Compare two fp_pair_t objects, and return -1, 0, or 1 as
  * appropriate. */
 static int

src/or/dirserv.c

@@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters;
  * cached_dir_t. */
 static digestmap_t *cached_v2_networkstatus = NULL;
 
-/** Map from flavor name to the v3 consensuses that we're currently serving. */
+/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
+ * currently serving. */
 static strmap_t *cached_consensuses = NULL;
 
 /** Possibly replace the contents of <b>d</b> with the value of

src/or/main.c

@@ -851,10 +851,13 @@ directory_info_has_arrived(time_t now, int from_cache)
         "I learned some more directory information, but not enough to "
         "build a circuit: %s", get_dir_info_status_string());
     update_router_descriptor_downloads(now);
+    update_microdesc_downloads(now);
     return;
   } else {
-    if (directory_fetches_from_authorities(options))
+    if (directory_fetches_from_authorities(options)) {
       update_router_descriptor_downloads(now);
+      update_microdesc_downloads(now);
+    }
 
     /* if we have enough dir info, then update our guard status with
      * whatever we just learned. */
@@ -1062,6 +1065,7 @@ run_scheduled_events(time_t now)
   if (time_to_try_getting_descriptors < now) {
     update_router_descriptor_downloads(now);
     update_extrainfo_downloads(now);
+    update_microdesc_downloads(now);
     if (options->UseBridges)
       fetch_bridge_descriptors(now);
     if (router_have_minimum_dir_info())

src/or/microdesc.c

@@ -3,8 +3,12 @@
 
 #include "or.h"
 #include "config.h"
+#include "directory.h"
 #include "microdesc.h"
 #include "routerparse.h"
+#include "networkstatus.h"
+#include "routerlist.h"
+#include "dirserv.h"
 
 /** A data structure to hold a bunch of cached microdescriptors.  There are
  * two active files in the cache: a "cache file" that we mmap, and a "journal
@@ -23,6 +27,8 @@ struct microdesc_cache_t {
   tor_mmap_t *cache_content;
   /** Number of bytes used in the journal file. */
   size_t journal_len;
+  /** Number of bytes in descriptors removed as too old. */
+  size_t bytes_dropped;
 
   /** Total bytes of microdescriptor bodies we have added to this cache */
   uint64_t total_len_seen;
@@ -119,15 +125,19 @@ get_microdesc_cache(void)
  * ending at <b>eos</b>, and store them in <b>cache</b>.  If <b>no-save</b>,
  * mark them as non-writable to disk.  If <b>where</b> is SAVED_IN_CACHE,
  * leave their bodies as pointers to the mmap'd cache.  If where is
- * <b>SAVED_NOWHERE</b>, do not allow annotations.  Return a list of the added
- * microdescriptors.  */
+ * <b>SAVED_NOWHERE</b>, do not allow annotations.  If listed_at is positive,
+ * set the last_listed field of every microdesc to listed_at.  If
+ * requested_digests is non-null, then it contains a list of digests we mean
+ * to allow, so we should reject any non-requested microdesc with a different
+ * digest, and alter the list to contain only the digests of those microdescs we
+ * didn't find.
+ * Return a list of the added microdescriptors.  */
 smartlist_t *
 microdescs_add_to_cache(microdesc_cache_t *cache,
                         const char *s, const char *eos, saved_location_t where,
-                        int no_save)
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256)
 {
-  /*XXXX need an argument that sets last_listed as appropriate. */
-
   smartlist_t *descriptors, *added;
   const int allow_annotations = (where != SAVED_NOWHERE);
   const int copy_body = (where != SAVED_IN_CACHE);
@@ -135,6 +145,33 @@ microdescs_add_to_cache(microdesc_cache_t *cache,
   descriptors = microdescs_parse_from_string(s, eos,
                                              allow_annotations,
                                              copy_body);
+  if (listed_at > 0) {
+    SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
+                      md->last_listed = listed_at);
+  }
+  if (requested_digests256) {
+    digestmap_t *requested; /* XXXX actuqlly we should just use a
+                               digest256map */
+    requested = digestmap_new();
+    SMARTLIST_FOREACH(requested_digests256, const char *, cp,
+      digestmap_set(requested, cp, (void*)1));
+    SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
+      if (digestmap_get(requested, md->digest)) {
+        digestmap_set(requested, md->digest, (void*)2);
+      } else {
+        log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microcdesc");
+        microdesc_free(md);
+        SMARTLIST_DEL_CURRENT(descriptors, md);
+      }
+    } SMARTLIST_FOREACH_END(md);
+    SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) {
+      if (digestmap_get(requested, cp) == (void*)2) {
+        tor_free(cp);
+        SMARTLIST_DEL_CURRENT(requested_digests256, cp);
+      }
+    } SMARTLIST_FOREACH_END(cp);
+    digestmap_free(requested, NULL);
+  }
 
   added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
   smartlist_free(descriptors);
@@ -207,10 +244,9 @@ microdescs_add_list_to_cache(microdesc_cache_t *cache,
   {
     size_t old_content_len =
       cache->cache_content ? cache->cache_content->size : 0;
-    if (cache->journal_len > 16384 + old_content_len &&
-        cache->journal_len > old_content_len * 2) {
+    if ((cache->journal_len > 16384 + old_content_len &&
+         cache->journal_len > old_content_len / 2))
       microdesc_cache_rebuild(cache);
-    }
   }
 
   return added;
@@ -251,7 +287,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
-                                    SAVED_IN_CACHE, 0);
+                                    SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
       total += smartlist_len(added);
       smartlist_free(added);
@@ -263,7 +299,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   if (journal_content) {
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
-                                    SAVED_IN_JOURNAL, 0);
+                                    SAVED_IN_JOURNAL, 0, -1, NULL);
     if (added) {
       total += smartlist_len(added);
       smartlist_free(added);
@@ -275,6 +311,47 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   return 0;
 }
 
+/** DOCDOC */
+#define TOLERATE_MICRODESC_AGE (7*24*60*60)
+
+/** DOCDOC */
+void
+microdesc_cache_clean(microdesc_cache_t *cache)
+{
+  networkstatus_t *consensus;
+  time_t cutoff;
+  microdesc_t **mdp, *victim;
+  int dropped=0, kept=0;
+  size_t bytes_dropped = 0;
+  time_t now = time(NULL);
+
+  /* If we don't know a consensus, never believe last_listed values */
+  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+  if (consensus == NULL)
+    return;
+
+  cutoff = now - TOLERATE_MICRODESC_AGE;
+
+  for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
+    if ((*mdp)->last_listed < cutoff) {
+      ++dropped;
+      victim = *mdp;
+      mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
+      bytes_dropped += victim->bodylen;
+      microdesc_free(victim);
+    } else {
+      ++kept;
+      mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
+    }
+  }
+
+  if (dropped) {
+    log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.",
+               dropped,dropped+kept);
+    cache->bytes_dropped += bytes_dropped;
+  }
+}
+
 /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  * and update every microdesc_t in the cache with pointers to its new
  * location. */
@@ -290,6 +367,9 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
   int orig_size, new_size;
 
   log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
+
+  microdesc_cache_clean(cache);
+
   orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
   orig_size += (int)cache->journal_len;
 
@@ -344,6 +424,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache)
 
   write_str_to_file(cache->journal_fname, "", 1);
   cache->journal_len = 0;
+  cache->bytes_dropped = 0;
 
   new_size = (int)cache->cache_content->size;
   log_info(LD_DIR, "Done rebuilding microdesc cache. "
@@ -412,3 +493,90 @@ microdesc_average_size(microdesc_cache_t *cache)
   return (size_t)(cache->total_len_seen / cache->n_seen);
 }
 
+/** Return a smartlist of all the sha256 digest of the microdescriptors that
+ * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
+ * to internals of <b>ns</b>; you should not free the members of the resulting
+ * smartlist.  Omit all microdescriptors whose digest appear in <b>skip</b>. */
+smartlist_t *
+microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
+                                 int downloadable_only, digestmap_t *skip)
+{
+  smartlist_t *result = smartlist_create();
+  time_t now = time(NULL);
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
+      continue;
+    if (downloadable_only &&
+        !download_status_is_ready(&rs->dl_status, now,
+                                  MAX_MICRODESC_DOWNLOAD_FAILURES))
+      continue;
+    if (skip && digestmap_get(skip, rs->descriptor_digest))
+      continue;
+    /* XXXX Also skip if we're a noncache and wouldn't use this router.
+     * XXXX NM Microdesc
+     */
+    smartlist_add(result, rs->descriptor_digest);
+  } SMARTLIST_FOREACH_END(rs);
+  return result;
+}
+
+/** DOCDOC */
+void
+update_microdesc_downloads(time_t now)
+{
+  or_options_t *options = get_options();
+  networkstatus_t *consensus;
+  smartlist_t *missing;
+  digestmap_t *pending;
+
+  if (should_delay_dir_fetches(options))
+    return;
+  if (directory_too_idle_to_fetch_descriptors(options, now))
+    return;
+
+  consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+  if (!consensus)
+    return;
+
+  if (!directory_caches_dir_info(options)) {
+    /* Right now, only caches fetch microdescriptors.
+     * XXXX NM Microdescs */
+    return;
+  }
+
+  pending = digestmap_new();
+  list_pending_microdesc_downloads(pending);
+
+  missing = microdesc_list_missing_digest256(consensus,
+                                             get_microdesc_cache(),
+                                             1,
+                                             pending);
+  digestmap_free(pending, NULL);
+
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
+                              missing, NULL, now);
+
+  smartlist_free(missing);
+}
+
+/** DOCDOC */
+void
+update_microdescs_from_networkstatus(time_t now)
+{
+  microdesc_cache_t *cache = get_microdesc_cache();
+  microdesc_t *md;
+  networkstatus_t *ns =
+    networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
+
+  if (! ns)
+    return;
+
+  tor_assert(ns->flavor == FLAV_MICRODESC);
+
+  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
+    md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
+    if (md && ns->valid_after > md->last_listed)
+      md->last_listed = ns->valid_after;
+  } SMARTLIST_FOREACH_END(rs);
+}

src/or/microdesc.h

@@ -16,11 +16,13 @@ microdesc_cache_t *get_microdesc_cache(void);
 
 smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache,
                         const char *s, const char *eos, saved_location_t where,
-                        int no_save);
+                        int no_save, time_t listed_at,
+                        smartlist_t *requested_digests256);
 smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache,
                         smartlist_t *descriptors, saved_location_t where,
                         int no_save);
 
+void microdesc_cache_clean(microdesc_cache_t *cache);
 int microdesc_cache_rebuild(microdesc_cache_t *cache);
 int microdesc_cache_reload(microdesc_cache_t *cache);
 void microdesc_cache_clear(microdesc_cache_t *cache);
@@ -30,8 +32,16 @@ microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
 
 size_t microdesc_average_size(microdesc_cache_t *cache);
 
+smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
+                                              microdesc_cache_t *cache,
+                                              int downloadable_only,
+                                              digestmap_t *skip);
+
 void microdesc_free(microdesc_t *md);
 void microdesc_free_all(void);
 
+void update_microdesc_downloads(time_t now);
+void update_microdescs_from_networkstatus(time_t now);
+
 #endif
 

src/or/networkstatus.c

@@ -20,6 +20,7 @@
 #include "dirserv.h"
 #include "dirvote.h"
 #include "main.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "relay.h"
 #include "router.h"
@@ -44,8 +45,19 @@ static strmap_t *named_server_map = NULL;
  * as unnamed for some server in the consensus. */
 static strmap_t *unnamed_server_map = NULL;
 
-/** Most recently received and validated v3 consensus network status. */
-static networkstatus_t *current_consensus = NULL;
+/** Most recently received and validated v3 consensus network status,
+ * of whichever type we are using for our own circuits.  This will be the same
+ * as one of current_ns_consensus or current_md_consensus.
+ */
+#define current_consensus current_ns_consensus
+
+/** Most recently received and validated v3 "ns"-flavored consensus network
+ * status. */
+static networkstatus_t *current_ns_consensus = NULL;
+
+/** Most recently received and validated v3 "microdec"-flavored consensus
+ * network status. */
+static networkstatus_t *current_md_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
@@ -271,6 +283,7 @@ router_reload_consensus_networkstatus(void)
   update_certificate_downloads(time(NULL));
 
   routers_update_all_from_networkstatus(time(NULL), 3);
+  update_microdescs_from_networkstatus(time(NULL));
 
   return 0;
 }
@@ -967,21 +980,25 @@ networkstatus_get_v2_list(void)
 }
 
 /** Return the consensus view of the status of the router whose current
- * <i>descriptor</i> digest is <b>digest</b>, or NULL if no such router is
- * known. */
+ * <i>descriptor</i> digest in <b>consensus</b> is <b>digest</b>, or NULL if
+ * no such router is known. */
 routerstatus_t *
-router_get_consensus_status_by_descriptor_digest(const char *digest)
+router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus,
+                                                 const char *digest)
 {
-  if (!current_consensus) return NULL;
-  if (!current_consensus->desc_digest_map) {
-    digestmap_t * m = current_consensus->desc_digest_map = digestmap_new();
-    SMARTLIST_FOREACH(current_consensus->routerstatus_list,
+  if (!consensus)
+    consensus = current_consensus;
+  if (!consensus)
+    return NULL;
+  if (!consensus->desc_digest_map) {
+    digestmap_t *m = consensus->desc_digest_map = digestmap_new();
+    SMARTLIST_FOREACH(consensus->routerstatus_list,
                       routerstatus_t *, rs,
      {
        digestmap_set(m, rs->descriptor_digest, rs);
      });
   }
-  return digestmap_get(current_consensus->desc_digest_map, digest);
+  return digestmap_get(consensus->desc_digest_map, digest);
 }
 
 /** Given the digest of a router descriptor, return its current download
@@ -990,7 +1007,10 @@ download_status_t *
 router_get_dl_status_by_descriptor_digest(const char *d)
 {
   routerstatus_t *rs;
-  if ((rs = router_get_consensus_status_by_descriptor_digest(d)))
+  if (!current_ns_consensus)
+    return NULL;
+  if ((rs = router_get_consensus_status_by_descriptor_digest(
+                                                 current_ns_consensus, d)))
     return &rs->dl_status;
   if (v2_download_status_map)
     return digestmap_get(v2_download_status_map, d);
@@ -1199,6 +1219,25 @@ update_v2_networkstatus_cache_downloads(time_t now)
   }
 }
 
+/** DOCDOC */
+static int
+we_want_to_fetch_flavor(or_options_t *options, int flavor)
+{
+  if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
+    /* This flavor is crazy; we don't want it */
+    /*XXXX handle unrecognized flavors later */
+    return 0;
+  }
+  if (authdir_mode_v3(options) || directory_caches_dir_info(options)) {
+    /* We want to serve all flavors to others, regardless if we would use
+     * it ourselves. */
+    return 1;
+  }
+  /* Otherwise, we want the flavor only if we want to use it to build
+   * circuits. */
+  return (flavor == USABLE_CONSENSUS_FLAVOR);
+}
+
 /** How many times will we try to fetch a consensus before we give up? */
 #define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8
 /** How long will we hang onto a possibly live consensus for which we're
@@ -1211,48 +1250,65 @@ static void
 update_consensus_networkstatus_downloads(time_t now)
 {
   int i;
+  or_options_t *options = get_options();
+
   if (!networkstatus_get_live_consensus(now))
     time_to_download_next_consensus = now; /* No live consensus? Get one now!*/
   if (time_to_download_next_consensus > now)
     return; /* Wait until the current consensus is older. */
-  /* XXXXNM Microdescs: may need to download more types. */
-  if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now,
-                                CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
-    return; /* We failed downloading a consensus too recently. */
-  if (connection_get_by_type_purpose(CONN_TYPE_DIR,
-                                     DIR_PURPOSE_FETCH_CONSENSUS))
-    return; /* There's an in-progress download.*/
 
   for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
-    consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
+    /* XXXX need some way to download unknown flavors if we are caching. */
+    const char *resource;
+    consensus_waiting_for_certs_t *waiting;
+
+    if (! we_want_to_fetch_flavor(options, i))
+      continue;
+
+    resource = i==FLAV_NS ? NULL : networkstatus_get_flavor_name(i);
+
+    if (!download_status_is_ready(&consensus_dl_status[i], now,
+                                  CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
+      continue; /* We failed downloading a consensus too recently. */
+    if (connection_dir_get_by_purpose_and_resource(
+                                DIR_PURPOSE_FETCH_CONSENSUS, resource))
+      continue; /* There's an in-progress download.*/
+
+    waiting = &consensus_waiting_for_certs[i];
     if (waiting->consensus) {
       /* XXXX make sure this doesn't delay sane downloads. */
-      if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now)
-        return; /* We're still getting certs for this one. */
-      else {
+      if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) {
+        continue; /* We're still getting certs for this one. */
+      } else {
         if (!waiting->dl_failed) {
-          download_status_failed(&consensus_dl_status[FLAV_NS], 0);
+          download_status_failed(&consensus_dl_status[i], 0);
           waiting->dl_failed=1;
         }
       }
     }
-  }
 
-  log_info(LD_DIR, "Launching networkstatus consensus download.");
-  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
-                               ROUTER_PURPOSE_GENERAL, NULL,
-                               PDS_RETRY_IF_NO_SERVERS);
+    log_info(LD_DIR, "Launching %s networkstatus consensus download.",
+             networkstatus_get_flavor_name(i));
+
+    directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
+                                 ROUTER_PURPOSE_GENERAL, resource,
+                                 PDS_RETRY_IF_NO_SERVERS);
+  }
 }
 
 /** Called when an attempt to download a consensus fails: note that the
  * failure occurred, and possibly retry. */
 void
-networkstatus_consensus_download_failed(int status_code)
+networkstatus_consensus_download_failed(int status_code, const char *flavname)
 {
-  /* XXXXNM Microdescs: may need to handle more types. */
-  download_status_failed(&consensus_dl_status[FLAV_NS], status_code);
-  /* Retry immediately, if appropriate. */
-  update_consensus_networkstatus_downloads(time(NULL));
+  int flav = networkstatus_parse_flavor_name(flavname);
+  if (flav >= 0) {
+    tor_assert(flav < N_CONSENSUS_FLAVORS);
+    /* XXXX handle unrecognized flavors */
+    download_status_failed(&consensus_dl_status[flav], status_code);
+    /* Retry immediately, if appropriate. */
+    update_consensus_networkstatus_downloads(time(NULL));
+  }
 }
 
 /** How long do we (as a cache) wait after a consensus becomes non-fresh
@@ -1373,7 +1429,10 @@ update_certificate_downloads(time_t now)
                                     now);
   }
 
-  authority_certs_fetch_missing(current_consensus, now);
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_ns_consensus, now);
+  if (current_ns_consensus)
+    authority_certs_fetch_missing(current_md_consensus, now);
 }
 
 /** Return 1 if we have a consensus but we don't have enough certificates
@@ -1405,6 +1464,18 @@ networkstatus_get_latest_consensus(void)
   return current_consensus;
 }
 
+/** DOCDOC */
+networkstatus_t *
+networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  if (f == FLAV_NS)
+    return current_ns_consensus;
+  else if (f == FLAV_MICRODESC)
+    return current_md_consensus;
+  else
+    tor_assert(0);
+}
+
 /** Return the most recent consensus that we have downloaded, or NULL if it is
  * no longer live. */
 networkstatus_t *
@@ -1424,13 +1495,15 @@ networkstatus_get_live_consensus(time_t now)
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
  * consensuses. */
 networkstatus_t *
-networkstatus_get_reasonably_live_consensus(time_t now)
+networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
 {
 #define REASONABLY_LIVE_TIME (24*60*60)
-  if (current_consensus &&
-      current_consensus->valid_after <= now &&
-      now <= current_consensus->valid_until+REASONABLY_LIVE_TIME)
-    return current_consensus;
+  networkstatus_t *consensus =
+    networkstatus_get_latest_consensus_by_flavor(flavor);
+  if (consensus &&
+      consensus->valid_after <= now &&
+      now <= consensus->valid_until+REASONABLY_LIVE_TIME)
+    return consensus;
   else
     return NULL;
 }
@@ -1569,6 +1642,7 @@ networkstatus_set_current_consensus(const char *consensus,
   const digests_t *current_digests = NULL;
   consensus_waiting_for_certs_t *waiting = NULL;
   time_t current_valid_after = 0;
+  int free_consensus = 1;
 
   if (flav < 0) {
     /* XXXX we don't handle unrecognized flavors yet. */
@@ -1614,9 +1688,16 @@ networkstatus_set_current_consensus(const char *consensus,
   if (!strcmp(flavor, "ns")) {
     consensus_fname = get_datadir_fname("cached-consensus");
     unverified_fname = get_datadir_fname("unverified-consensus");
-    if (current_consensus) {
-      current_digests = &current_consensus->digests;
-      current_valid_after = current_consensus->valid_after;
+    if (current_ns_consensus) {
+      current_digests = &current_ns_consensus->digests;
+      current_valid_after = current_ns_consensus->valid_after;
+    }
+  } else if (!strcmp(flavor, "microdesc")) {
+    consensus_fname = get_datadir_fname("cached-microdesc-consensus");
+    unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
+    if (current_md_consensus) {
+      current_digests = &current_md_consensus->digests;
+      current_valid_after = current_md_consensus->valid_after;
     }
   } else {
     cached_dir_t *cur;
@@ -1702,11 +1783,21 @@ networkstatus_set_current_consensus(const char *consensus,
 
   if (flav == USABLE_CONSENSUS_FLAVOR) {
     notify_control_networkstatus_changed(current_consensus, c);
-
-    if (current_consensus) {
-      networkstatus_copy_old_consensus_info(c, current_consensus);
-      networkstatus_vote_free(current_consensus);
+  }
+  if (flav == FLAV_NS) {
+    if (current_ns_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+      networkstatus_vote_free(current_ns_consensus);
+    }
+    current_ns_consensus = c;
+    free_consensus = 0; /* avoid free */
+  } else if (flav == FLAV_MICRODESC) {
+    if (current_md_consensus) {
+      networkstatus_copy_old_consensus_info(c, current_md_consensus);
+      networkstatus_vote_free(current_md_consensus);
     }
+    current_md_consensus = c;
+    free_consensus = 0; /* avoid free */
   }
 
   waiting = &consensus_waiting_for_certs[flav];
@@ -1739,11 +1830,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (flav == USABLE_CONSENSUS_FLAVOR) {
-    current_consensus = c;
-    c = NULL; /* Prevent free. */
-
     /* XXXXNM Microdescs: needs a non-ns variant. */
     update_consensus_networkstatus_fetch_time(now);
+
     dirvote_recalculate_timing(options, now);
     routerstatus_list_update_named_server_map();
     cell_ewma_set_scale_factor(options, current_consensus);
@@ -1758,11 +1847,11 @@ networkstatus_set_current_consensus(const char *consensus,
     write_str_to_file(consensus_fname, consensus, 0);
   }
 
-  if (ftime_definitely_before(now, current_consensus->valid_after)) {
+  if (ftime_definitely_before(now, c->valid_after)) {
     char tbuf[ISO_TIME_LEN+1];
     char dbuf[64];
-    long delta = now - current_consensus->valid_after;
-    format_iso_time(tbuf, current_consensus->valid_after);
+    long delta = now - c->valid_after;
+    format_iso_time(tbuf, c->valid_after);
     format_time_interval(dbuf, sizeof(dbuf), delta);
     log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
              "consensus network status document (%s GMT).  Tor needs an "
@@ -1776,7 +1865,8 @@ networkstatus_set_current_consensus(const char *consensus,
 
   result = 0;
  done:
-  networkstatus_vote_free(c);
+  if (free_consensus)
+    networkstatus_vote_free(c);
   tor_free(consensus_fname);
   tor_free(unverified_fname);
   return result;
@@ -1812,7 +1902,8 @@ void
 routers_update_all_from_networkstatus(time_t now, int dir_version)
 {
   routerlist_t *rl = router_get_routerlist();
-  networkstatus_t *consensus = networkstatus_get_live_consensus(now);
+  networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,
+                                                                     FLAV_NS);
 
   if (networkstatus_v2_list_has_changed)
     download_status_map_update_from_v2_networkstatus();
@@ -2028,7 +2119,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
 void
 signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
 {
-  networkstatus_t *ns = current_consensus;
+  networkstatus_t *ns = current_ns_consensus;
   if (!ns)
     return;
 
@@ -2036,7 +2127,7 @@ signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
     char dummy[DIGEST_LEN];
     /* instantiates the digest map. */
     memset(dummy, 0, sizeof(dummy));
-    router_get_consensus_status_by_descriptor_digest(dummy);
+    router_get_consensus_status_by_descriptor_digest(ns, dummy);
   }
   SMARTLIST_FOREACH(descs, signed_descriptor_t *, d,
   {
@@ -2264,8 +2355,9 @@ networkstatus_free_all(void)
 
   digestmap_free(v2_download_status_map, _tor_free);
   v2_download_status_map = NULL;
-  networkstatus_vote_free(current_consensus);
-  current_consensus = NULL;
+  networkstatus_vote_free(current_ns_consensus);
+  networkstatus_vote_free(current_md_consensus);
+  current_md_consensus = current_ns_consensus = NULL;
 
   for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
     consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];

src/or/networkstatus.h

@@ -48,12 +48,14 @@ const smartlist_t *networkstatus_get_v2_list(void);
 download_status_t *router_get_dl_status_by_descriptor_digest(const char *d);
 routerstatus_t *router_get_consensus_status_by_id(const char *digest);
 routerstatus_t *router_get_consensus_status_by_descriptor_digest(
-                                                        const char *digest);
+                                   networkstatus_t *consensus,
+                                   const char *digest);
 routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname,
                                                        int warn_if_unnamed);
 const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
 int networkstatus_nickname_is_unnamed(const char *nickname);
-void networkstatus_consensus_download_failed(int status_code);
+void networkstatus_consensus_download_failed(int status_code,
+                                             const char *flavname);
 void update_consensus_networkstatus_fetch_time(time_t now);
 int should_delay_dir_fetches(or_options_t *options);
 void update_networkstatus_downloads(time_t now);
@@ -61,8 +63,11 @@ void update_certificate_downloads(time_t now);
 int consensus_is_waiting_for_certs(void);
 networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest);
 networkstatus_t *networkstatus_get_latest_consensus(void);
+networkstatus_t *networkstatus_get_latest_consensus_by_flavor(
+                                                  consensus_flavor_t f);
 networkstatus_t *networkstatus_get_live_consensus(time_t now);
-networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now);
+networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
+                                                             int flavor);
 #define NSSET_FROM_CACHE 1
 #define NSSET_WAS_WAITING_FOR_CERTS 2
 #define NSSET_DONT_DOWNLOAD_CERTS 4

src/or/or.h

@@ -391,7 +391,9 @@ typedef enum {
 /** A connection to a hidden service directory server: download a v2 rendezvous
  * descriptor. */
 #define DIR_PURPOSE_FETCH_RENDDESC_V2 18
-#define _DIR_PURPOSE_MAX 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+#define _DIR_PURPOSE_MAX 19
 
 /** True iff <b>p</b> is a purpose corresponding to uploading data to a
  * directory server. */
@@ -1191,8 +1193,13 @@ typedef struct edge_connection_t {
 typedef struct dir_connection_t {
   connection_t _base;
 
-  char *requested_resource; /**< Which 'resource' did we ask the directory
-                             * for? */
+ /** Which 'resource' did we ask the directory for? This is typically the part
+  * of the URL string that defines, relative to the directory conn purpose,
+  * what thing we want.  For example, in router descriptor downloads by
+  * descriptor digest, it contains "d/", then one ore more +-separated
+  * fingerprints.
+  **/
+  char *requested_resource;
   unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
 
   /* Used only for server sides of some dir connections, to implement
@@ -1569,8 +1576,9 @@ typedef struct routerstatus_t {
                                       * has. */
   char identity_digest[DIGEST_LEN]; /**< Digest of the router's identity
                                      * key. */
-  char descriptor_digest[DIGEST_LEN]; /**< Digest of the router's most recent
-                                       * descriptor. */
+  /** Digest of the router's most recent descriptor or microdescriptor.
+   * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
+  char descriptor_digest[DIGEST256_LEN];
   uint32_t addr; /**< IPv4 address for this router. */
   uint16_t or_port; /**< OR port for this router. */
   uint16_t dir_port; /**< Directory port for this router. */
@@ -1689,6 +1697,10 @@ typedef struct microdesc_t {
  * up? */
 #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8
 
+/** How many times will we try to download a microdescriptor before giving
+ * up? */
+#define MAX_MICRODESC_DOWNLOAD_FAILURES 8
+
 /** Contents of a v2 (non-consensus, non-vote) network status object. */
 typedef struct networkstatus_v2_t {
   /** When did we receive the network-status document? */
@@ -3516,6 +3528,8 @@ typedef struct trusted_dir_server_t {
  *  fetches to _any_ single directory server.]
  */
 #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
 #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16)
 
 /** Possible ways to weight routers when choosing one randomly.  See

src/or/routerlist.c

@@ -22,6 +22,7 @@
 #include "geoip.h"
 #include "hibernate.h"
 #include "main.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "policies.h"
 #include "reasons.h"
@@ -44,9 +45,6 @@ static routerstatus_t *router_pick_trusteddirserver_impl(
 static void mark_all_trusteddirservers_up(void);
 static int router_nickname_matches(routerinfo_t *router, const char *nickname);
 static void trusted_dir_server_free(trusted_dir_server_t *ds);
-static void launch_router_descriptor_downloads(smartlist_t *downloadable,
-                                               routerstatus_t *source,
-                                               time_t now);
 static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
 static void update_router_have_minimum_dir_info(void);
 static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc,
@@ -1047,7 +1045,8 @@ router_pick_trusteddirserver(authority_type_t type, int flags)
     /* If the reason that we got no server is that servers are "busy",
      * we must be excluding good servers because we already have serverdesc
      * fetches with them.  Do not mark down servers up because of this. */
-    tor_assert((flags & PDS_NO_EXISTING_SERVERDESC_FETCH));
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
     return NULL;
   }
 
@@ -1174,6 +1173,7 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags,
   const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
   const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS);
   const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
   int n_busy = 0;
 
   if (!trusted_dir_servers)
@@ -1212,6 +1212,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags,
           continue;
         }
       }
+      if (no_microdesc_fetching) {
+        if (connection_get_by_type_addr_port_purpose(
+             CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_MICRODESC)) {
+          ++n_busy;
+          continue;
+        }
+      }
 
       if (prefer_tunnel &&
           d->or_port &&
@@ -3874,6 +3881,7 @@ routerlist_retry_directory_downloads(time_t now)
   router_reset_descriptor_download_failures();
   update_networkstatus_downloads(now);
   update_router_descriptor_downloads(now);
+  update_microdesc_downloads(now);
 }
 
 /** Return 1 if all running sufficiently-stable routers will reject
@@ -4035,7 +4043,9 @@ any_trusted_dir_is_v1_authority(void)
 /** For every current directory connection whose purpose is <b>purpose</b>,
  * and where the resource being downloaded begins with <b>prefix</b>, split
  * rest of the resource into base16 fingerprints, decode them, and set the
- * corresponding elements of <b>result</b> to a nonzero value. */
+ * corresponding elements of <b>result</b> to a nonzero value.
+ * DOCDOC purpose==microdesc
+ */
 static void
 list_pending_downloads(digestmap_t *result,
                        int purpose, const char *prefix)
@@ -4043,20 +4053,23 @@ list_pending_downloads(digestmap_t *result,
   const size_t p_len = strlen(prefix);
   smartlist_t *tmp = smartlist_create();
   smartlist_t *conns = get_connection_array();
+  int flags = DSR_HEX;
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC)
+    flags = DSR_DIGEST256|DSR_BASE64;
 
   tor_assert(result);
 
-  SMARTLIST_FOREACH(conns, connection_t *, conn,
-  {
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
     if (conn->type == CONN_TYPE_DIR &&
         conn->purpose == purpose &&
         !conn->marked_for_close) {
       const char *resource = TO_DIR_CONN(conn)->requested_resource;
       if (!strcmpstart(resource, prefix))
         dir_split_resource_into_fingerprints(resource + p_len,
-                                             tmp, NULL, DSR_HEX);
+                                             tmp, NULL, flags);
     }
-  });
+  } SMARTLIST_FOREACH_END(conn);
+
   SMARTLIST_FOREACH(tmp, char *, d,
                     {
                       digestmap_set(result, d, (void*)1);
@@ -4076,10 +4089,18 @@ list_pending_descriptor_downloads(digestmap_t *result, int extrainfo)
   list_pending_downloads(result, purpose, "d/");
 }
 
-/** Launch downloads for all the descriptors whose digests are listed
- * as digests[i] for lo <= i < hi.  (Lo and hi may be out of range.)
- * If <b>source</b> is given, download from <b>source</b>; otherwise,
- * download from an appropriate random directory server.
+/** DOCDOC */
+/*XXXX NM should use digest256, if one comes into being. */
+void
+list_pending_microdesc_downloads(digestmap_t *result)
+{
+  list_pending_downloads(result, DIR_PURPOSE_FETCH_MICRODESC, "d/");
+}
+
+/** Launch downloads for all the descriptors whose digests or digests256
+ * are listed as digests[i] for lo <= i < hi.  (Lo and hi may be out of
+ * range.)  If <b>source</b> is given, download from <b>source</b>;
+ * otherwise, download from an appropriate random directory server.
  */
 static void
 initiate_descriptor_downloads(routerstatus_t *source,
@@ -4090,6 +4111,20 @@ initiate_descriptor_downloads(routerstatus_t *source,
   int i, n = hi-lo;
   char *resource, *cp;
   size_t r_len;
+
+  int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN;
+  char sep = '+';
+  int b64_256 = 0;
+
+  if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    /* Microdescriptors are downloaded by "-"-separated base64-encoded
+     * 256-bit digests. */
+    digest_len = DIGEST256_LEN;
+    enc_digest_len = BASE64_DIGEST256_LEN;
+    sep = '-';
+    b64_256 = 1;
+  }
+
   if (n <= 0)
     return;
   if (lo < 0)
@@ -4097,15 +4132,19 @@ initiate_descriptor_downloads(routerstatus_t *source,
   if (hi > smartlist_len(digests))
     hi = smartlist_len(digests);
 
-  r_len = 8 + (HEX_DIGEST_LEN+1)*n;
+  r_len = 8 + (enc_digest_len+1)*n;
   cp = resource = tor_malloc(r_len);
   memcpy(cp, "d/", 2);
   cp += 2;
   for (i = lo; i < hi; ++i) {
-    base16_encode(cp, r_len-(cp-resource),
-                  smartlist_get(digests,i), DIGEST_LEN);
-    cp += HEX_DIGEST_LEN;
-    *cp++ = '+';
+    if (b64_256) {
+      digest256_to_base64(cp, smartlist_get(digests, i));
+    } else {
+      base16_encode(cp, r_len-(cp-resource),
+                    smartlist_get(digests,i), digest_len);
+    }
+    cp += enc_digest_len;
+    *cp++ = sep;
   }
   memcpy(cp-1, ".z", 3);
 
@@ -4152,6 +4191,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options)
  *   So use 96 because it's a nice number.
  */
 #define MAX_DL_PER_REQUEST 96
+#define MAX_MICRODESC_DL_PER_REQUEST 92
 /** Don't split our requests so finely that we are requesting fewer than
  * this number per server. */
 #define MIN_DL_PER_REQUEST 4
@@ -4166,21 +4206,33 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options)
  * them until they have more, or until this amount of time has passed. */
 #define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60)
 
-/** Given a list of router descriptor digests in <b>downloadable</b>, decide
- * whether to delay fetching until we have more.  If we don't want to delay,
- * launch one or more requests to the appropriate directory authorities. */
-static void
-launch_router_descriptor_downloads(smartlist_t *downloadable,
-                                   routerstatus_t *source, time_t now)
+/** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of
+ * router descriptor digests or microdescriptor digest256s in
+ * <b>downloadable</b>, decide whether to delay fetching until we have more.
+ * If we don't want to delay, launch one or more requests to the appropriate
+ * directory authorities.
+ */
+void
+launch_descriptor_downloads(int purpose,
+                            smartlist_t *downloadable,
+                            routerstatus_t *source, time_t now)
 {
   int should_delay = 0, n_downloadable;
   or_options_t *options = get_options();
+  const char *descname;
+
+  tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ?
+    "routerdesc" : "microdesc";
 
   n_downloadable = smartlist_len(downloadable);
   if (!directory_fetches_dir_info_early(options)) {
     if (n_downloadable >= MAX_DL_TO_DELAY) {
       log_debug(LD_DIR,
-             "There are enough downloadable routerdescs to launch requests.");
+                "There are enough downloadable %ss to launch requests.",
+                descname);
       should_delay = 0;
     } else {
       should_delay = (last_routerdesc_download_attempted +
@@ -4188,13 +4240,15 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
       if (!should_delay && n_downloadable) {
         if (last_routerdesc_download_attempted) {
           log_info(LD_DIR,
-                   "There are not many downloadable routerdescs, but we've "
+                   "There are not many downloadable %ss, but we've "
                    "been waiting long enough (%d seconds). Downloading.",
+                   descname,
                    (int)(now-last_routerdesc_download_attempted));
         } else {
           log_info(LD_DIR,
-                 "There are not many downloadable routerdescs, but we haven't "
-                 "tried downloading descriptors recently. Downloading.");
+                   "There are not many downloadable %ss, but we haven't "
+                   "tried downloading descriptors recently. Downloading.",
+                   descname);
         }
       }
     }
@@ -4221,12 +4275,20 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
        * update_router_descriptor_downloads() later on, once the connections
        * have succeeded or failed.
        */
-      pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH;
+      pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ?
+        PDS_NO_EXISTING_MICRODESC_FETCH :
+        PDS_NO_EXISTING_SERVERDESC_FETCH;
+
     }
 
     n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
-    if (n_per_request > MAX_DL_PER_REQUEST)
-      n_per_request = MAX_DL_PER_REQUEST;
+    if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+      if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST)
+        n_per_request = MAX_MICRODESC_DL_PER_REQUEST;
+    } else {
+      if (n_per_request > MAX_DL_PER_REQUEST)
+        n_per_request = MAX_DL_PER_REQUEST;
+    }
     if (n_per_request < MIN_DL_PER_REQUEST)
       n_per_request = MIN_DL_PER_REQUEST;
 
@@ -4241,7 +4303,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable,
              req_plural, n_downloadable, rtr_plural, n_per_request);
     smartlist_sort_digests(downloadable);
     for (i=0; i < n_downloadable; i += n_per_request) {
-      initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC,
+      initiate_descriptor_downloads(source, purpose,
                                     downloadable, i, i+n_per_request,
                                     pds_flags);
     }
@@ -4514,7 +4576,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
            smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters,
            n_would_reject, n_wouldnt_use, n_inprogress);
 
-  launch_router_descriptor_downloads(downloadable, source, now);
+  launch_descriptor_downloads(DIR_PURPOSE_FETCH_SERVERDESC,
+                              downloadable, source, now);
 
   digestmap_free(map, NULL);
  done:
@@ -4539,10 +4602,13 @@ update_router_descriptor_downloads(time_t now)
   if (directory_fetches_dir_info_early(options)) {
     update_router_descriptor_cache_downloads_v2(now);
   }
+
   update_consensus_router_descriptor_downloads(now, 0,
-    networkstatus_get_reasonably_live_consensus(now));
+                  networkstatus_get_reasonably_live_consensus(now, FLAV_NS));
 
   /* XXXX021 we could be smarter here; see notes on bug 652. */
+  /* XXXX NM Microdescs: if we're not fetching microdescriptors, we need
+   * to make something else invoke this. */
   /* If we're a server that doesn't have a configured address, we rely on
    * directory fetches to learn when our address changes.  So if we haven't
    * tried to get any routerdescs in a long time, try a dummy fetch now. */
@@ -4713,7 +4779,7 @@ count_loading_descriptors_progress(void)
   int num_present = 0, num_usable=0;
   time_t now = time(NULL);
   const networkstatus_t *consensus =
-    networkstatus_get_reasonably_live_consensus(now);
+    networkstatus_get_reasonably_live_consensus(now, FLAV_NS);
   double fraction;
 
   if (!consensus)
@@ -4743,7 +4809,7 @@ update_router_have_minimum_dir_info(void)
   int res;
   or_options_t *options = get_options();
   const networkstatus_t *consensus =
-    networkstatus_get_reasonably_live_consensus(now);
+    networkstatus_get_reasonably_live_consensus(now, FLAV_NS);
 
   if (!consensus) {
     if (!networkstatus_get_latest_consensus())

src/or/routerlist.h

@@ -192,5 +192,11 @@ int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
 int hid_serv_acting_as_directory(void);
 int hid_serv_responsible_for_desc_id(const char *id);
 
+void list_pending_microdesc_downloads(digestmap_t *result);
+void launch_descriptor_downloads(int purpose,
+                                 smartlist_t *downloadable,
+                                 routerstatus_t *source,
+                                 time_t now);
+
 #endif
 

src/or/routerparse.c

@@ -1943,6 +1943,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
 
   if (!consensus_method)
     flav = FLAV_NS;
+  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 
   eos = find_start_of_next_routerstatus(*s);
 
@@ -1955,15 +1956,16 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     goto err;
   }
   tok = find_by_keyword(tokens, K_R);
-  tor_assert(tok->n_args >= 7);
+  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
   if (flav == FLAV_NS) {
     if (tok->n_args < 8) {
       log_warn(LD_DIR, "Too few arguments to r");
       goto err;
     }
-  } else {
-    offset = -1;
+  } else if (flav == FLAV_MICRODESC) {
+    offset = -1; /* There is no identity digest */
   }
+
   if (vote_rs) {
     rs = &vote_rs->status;
   } else {
@@ -2139,6 +2141,16 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         vote_rs->microdesc = line;
       }
     } SMARTLIST_FOREACH_END(t);
+  } else if (flav == FLAV_MICRODESC) {
+    tok = find_opt_by_keyword(tokens, K_M);
+    if (tok) {
+      tor_assert(tok->n_args);
+      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
+        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
+                 escaped(tok->args[0]));
+        goto err;
+      }
+    }
   }
 
   if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))