Merge branch 'split_routerlist_dirserv_v2'

src/app/config/config.c

@@ -87,7 +87,9 @@
 #else
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #endif
+#include "feature/dirauth/bwauth.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/guardfraction.h"
 #include "feature/relay/dns.h"
 #include "core/or/dos.h"
 #include "feature/client/entrynodes.h"
@@ -105,7 +107,7 @@
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
 #include "lib/sandbox/sandbox.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerset.h"
 #include "core/or/scheduler.h"
 #include "app/config/statefile.h"
@@ -141,6 +143,7 @@
 #include "lib/evloop/procmon.h"
 
 #include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/mode.h"
 
 #include "core/or/connection_st.h"

src/core/include.am

@@ -56,6 +56,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/client/entrynodes.c		\
 	src/feature/client/transports.c		\
 	src/feature/control/control.c		\
+	src/feature/control/fmt_serverstatus.c  \
 	src/feature/dirauth/keypin.c		\
 	src/feature/dircache/conscache.c	\
 	src/feature/dircache/consdiffmgr.c	\
@@ -80,13 +81,17 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/hs/hs_stats.c		\
 	src/feature/hs_common/replaycache.c	\
 	src/feature/hs_common/shared_random_client.c	\
+	src/feature/nodelist/authcert.c		\
+	src/feature/nodelist/dirlist.c		\
 	src/feature/nodelist/microdesc.c	\
 	src/feature/nodelist/networkstatus.c	\
 	src/feature/nodelist/nodelist.c		\
+	src/feature/nodelist/node_select.c	\
 	src/feature/nodelist/parsecommon.c	\
 	src/feature/nodelist/routerlist.c	\
 	src/feature/nodelist/routerparse.c	\
 	src/feature/nodelist/routerset.c	\
+	src/feature/nodelist/fmt_routerstatus.c	\
 	src/feature/nodelist/torcert.c		\
 	src/feature/relay/dns.c			\
 	src/feature/relay/ext_orport.c		\
@@ -100,6 +105,16 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/stats/geoip.c		\
 	src/feature/stats/rephist.c
 
+# These should eventually move into module_dirauth_sources, but for now
+# the separation is only in the code location.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirauth/bwauth.c		\
+	src/feature/dirauth/guardfraction.c	\
+	src/feature/dirauth/reachability.c	\
+	src/feature/dirauth/recommend_pkg.c	\
+	src/feature/dirauth/process_descs.c	\
+	src/feature/dirauth/voteflags.c
+
 if BUILD_NT_SERVICES
 LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
 endif
@@ -219,14 +234,21 @@ noinst_HEADERS +=					\
 	src/feature/client/transports.h			\
 	src/feature/control/control.h			\
 	src/feature/control/control_connection_st.h	\
+	src/feature/control/fmt_serverstatus.h		\
+	src/feature/dirauth/bwauth.h			\
 	src/feature/dirauth/dircollate.h		\
 	src/feature/dirauth/dirvote.h			\
+	src/feature/dirauth/guardfraction.h		\
 	src/feature/dirauth/keypin.h			\
 	src/feature/dirauth/mode.h			\
 	src/feature/dirauth/ns_detached_signatures_st.h	\
+	src/feature/dirauth/reachability.h		\
+	src/feature/dirauth/recommend_pkg.h		\
+	src/feature/dirauth/process_descs.h		\
 	src/feature/dirauth/shared_random.h		\
 	src/feature/dirauth/shared_random_state.h	\
 	src/feature/dirauth/vote_microdesc_hash_st.h	\
+	src/feature/dirauth/voteflags.h			\
 	src/feature/dircache/cached_dir_st.h		\
 	src/feature/dircache/conscache.h		\
 	src/feature/dircache/consdiffmgr.h		\
@@ -256,8 +278,10 @@ noinst_HEADERS +=					\
 	src/feature/hs/hsdir_index_st.h			\
 	src/feature/hs_common/replaycache.h		\
 	src/feature/hs_common/shared_random_client.h	\
+	src/feature/nodelist/authcert.h			\
 	src/feature/nodelist/authority_cert_st.h	\
 	src/feature/nodelist/desc_store_st.h		\
+	src/feature/nodelist/dirlist.h			\
 	src/feature/nodelist/document_signature_st.h	\
 	src/feature/nodelist/extrainfo_st.h		\
 	src/feature/nodelist/microdesc.h		\
@@ -268,12 +292,14 @@ noinst_HEADERS +=					\
 	src/feature/nodelist/networkstatus_voter_info_st.h	\
 	src/feature/nodelist/node_st.h			\
 	src/feature/nodelist/nodelist.h			\
+	src/feature/nodelist/node_select.h		\
 	src/feature/nodelist/parsecommon.h		\
 	src/feature/nodelist/routerinfo_st.h		\
 	src/feature/nodelist/routerlist.h		\
 	src/feature/nodelist/routerlist_st.h		\
 	src/feature/nodelist/routerparse.h		\
 	src/feature/nodelist/routerset.h		\
+	src/feature/nodelist/fmt_routerstatus.h		\
 	src/feature/nodelist/routerstatus_st.h		\
 	src/feature/nodelist/signed_descriptor_st.h	\
 	src/feature/nodelist/torcert.h			\

src/core/mainloop/main.c

@@ -74,6 +74,9 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircache/directory.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dirauth/process_descs.h"
 #include "feature/relay/dns.h"
 #include "feature/client/dnsserv.h"
 #include "core/or/dos.h"
@@ -101,6 +104,8 @@
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "core/or/scheduler.h"
@@ -3661,7 +3666,9 @@ tor_free_all(int postfork)
   routerlist_free_all();
   networkstatus_free_all();
   addressmap_free_all();
+  dirserv_free_fingerprint_list();
   dirserv_free_all();
+  dirserv_clear_measured_bw_cache();
   rend_cache_free_all();
   rend_service_authorization_free_all();
   rep_hist_free_all();

src/core/or/channeltls.c

@@ -53,7 +53,7 @@
 #include "core/or/relay.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
 #include "core/or/scheduler.h"
 #include "feature/nodelist/torcert.h"
 #include "feature/nodelist/networkstatus.h"

src/core/or/circuitbuild.c

@@ -60,6 +60,7 @@
 #include "feature/rend/rendcommon.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"
@@ -3007,4 +3008,3 @@ circuit_upgrade_circuits_from_guard_wait(void)
 
   smartlist_free(to_upgrade);
 }
-

src/core/or/connection_or.c

@@ -41,7 +41,7 @@
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirauth/reachability.h"
 #include "feature/client/entrynodes.h"
 #include "feature/stats/geoip.h"
 #include "core/mainloop/main.h"
@@ -56,6 +56,7 @@
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/ext_orport.h"
 #include "core/or/scheduler.h"

src/core/or/policies.c

@@ -20,7 +20,6 @@
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
 #include "app/config/config.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -39,6 +38,9 @@
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
+/** Maximum length of an exit policy summary. */
+#define MAX_EXITPOLICY_SUMMARY_LEN 1000
+
 /** Policy that addresses for incoming SOCKS connections must match. */
 static smartlist_t *socks_policy = NULL;
 /** Policy that addresses for incoming directory connections must match. */

src/core/or/reasons.c

@@ -17,7 +17,7 @@
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/or/reasons.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
 #include "lib/tls/tortls.h"
 
 /***************************** Edge (stream) reasons **********************/

src/feature/client/bridges.c

@@ -23,7 +23,9 @@
 #include "feature/nodelist/nodelist.h"
 #include "core/or/policies.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
+
 #include "feature/nodelist/routerset.h"
 #include "feature/client/transports.h"
 

src/feature/client/entrynodes.c

@@ -133,7 +133,7 @@
 #include "feature/nodelist/nodelist.h"
 #include "core/or/policies.h"
 #include "feature/relay/router.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"
 #include "feature/client/transports.h"

src/feature/control/control.c

@@ -53,6 +53,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "feature/control/control.h"
+#include "feature/control/fmt_serverstatus.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircache/directory.h"
@@ -77,6 +78,8 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/hs_common/shared_random_client.h"

src/feature/control/fmt_serverstatus.c

@@ -0,0 +1,103 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/control/fmt_serverstatus.h"
+
+#include "app/config/config.h"
+#include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/**
+ * Allocate and return a description of the status of the server <b>desc</b>,
+ * for use in a v1-style router-status line.  The server is listed
+ * as running iff <b>is_live</b> is true.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+static char *
+list_single_server_status(const routerinfo_t *desc, int is_live)
+{
+  char buf[MAX_NICKNAME_LEN+HEX_DIGEST_LEN+4]; /* !nickname=$hexdigest\0 */
+  char *cp;
+  const node_t *node;
+
+  tor_assert(desc);
+
+  cp = buf;
+  if (!is_live) {
+    *cp++ = '!';
+  }
+  node = node_get_by_id(desc->cache_info.identity_digest);
+  if (node && node->is_valid) {
+    strlcpy(cp, desc->nickname, sizeof(buf)-(cp-buf));
+    cp += strlen(cp);
+    *cp++ = '=';
+  }
+  *cp++ = '$';
+  base16_encode(cp, HEX_DIGEST_LEN+1, desc->cache_info.identity_digest,
+                DIGEST_LEN);
+  return tor_strdup(buf);
+}
+
+/** Based on the routerinfo_ts in <b>routers</b>, allocate the
+ * contents of a v1-style router-status line, and store it in
+ * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
+ *
+ * If for_controller is true, include the routers with very old descriptors.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
+ */
+int
+list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                      int for_controller)
+{
+  /* List of entries in a router-status style: An optional !, then an optional
+   * equals-suffixed nickname, then a dollar-prefixed hexdigest. */
+  smartlist_t *rs_entries;
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  const or_options_t *options = get_options();
+  /* We include v2 dir auths here too, because they need to answer
+   * controllers. Eventually we'll deprecate this whole function;
+   * see also networkstatus_getinfo_by_purpose(). */
+  int authdir = authdir_mode_publishes_statuses(options);
+  tor_assert(router_status_out);
+
+  rs_entries = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
+    tor_assert(node);
+    if (authdir) {
+      /* Update router status in routerinfo_t. */
+      dirserv_set_router_is_running(ri, now);
+    }
+    if (for_controller) {
+      char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
+      char *cp = name_buf;
+      if (!node->is_running)
+        *cp++ = '!';
+      router_get_verbose_nickname(cp, ri);
+      smartlist_add_strdup(rs_entries, name_buf);
+    } else if (ri->cache_info.published_on >= cutoff) {
+      smartlist_add(rs_entries, list_single_server_status(ri,
+                                                          node->is_running));
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(rs_entries, char *, cp, tor_free(cp));
+  smartlist_free(rs_entries);
+
+  return 0;
+}

src/feature/control/fmt_serverstatus.h

@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_serverstatus.h
+ * \brief Header file for fmt_serverstatus.c.
+ **/
+
+#ifndef TOR_FMT_SERVERSTATUS_H
+#define TOR_FMT_SERVERSTATUS_H
+
+int list_server_status_v1(smartlist_t *routers, char **router_status_out,
+                          int for_controller);
+
+#endif

src/feature/dirauth/bwauth.c

@@ -0,0 +1,453 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.c
+ * \brief Code to read and apply bandwidth authority data.
+ **/
+
+#define BWAUTH_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/bwauth.h"
+
+#include "app/config/config.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerparse.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/encoding/keyval.h"
+
+/** Total number of routers with measured bandwidth; this is set by
+ * dirserv_count_measured_bs() before the loop in
+ * dirserv_generate_networkstatus_vote_obj() and checked by
+ * dirserv_get_credible_bandwidth() and
+ * dirserv_compute_performance_thresholds() */
+static int routers_with_measured_bw = 0;
+
+/** Look through the routerlist, and using the measured bandwidth cache count
+ * how many measured bandwidths we know.  This is used to decide whether we
+ * ever trust advertised bandwidths for purposes of assigning flags. */
+void
+dirserv_count_measured_bws(const smartlist_t *routers)
+{
+  /* Initialize this first */
+  routers_with_measured_bw = 0;
+
+  /* Iterate over the routerlist and count measured bandwidths */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    /* Check if we know a measured bandwidth for this one */
+    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
+      ++routers_with_measured_bw;
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Return the last-computed result from dirserv_count_mesured_bws(). */
+int
+dirserv_get_last_n_measured_bws(void)
+{
+  return routers_with_measured_bw;
+}
+
+/** Measured bandwidth cache entry */
+typedef struct mbw_cache_entry_s {
+  long mbw_kb;
+  time_t as_of;
+} mbw_cache_entry_t;
+
+/** Measured bandwidth cache - keys are identity_digests, values are
+ * mbw_cache_entry_t *. */
+static digestmap_t *mbw_cache = NULL;
+
+/** Store a measured bandwidth cache entry when reading the measured
+ * bandwidths file. */
+STATIC void
+dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                          time_t as_of)
+{
+  mbw_cache_entry_t *e = NULL;
+
+  tor_assert(parsed_line);
+
+  /* Allocate a cache if we need */
+  if (!mbw_cache) mbw_cache = digestmap_new();
+
+  /* Check if we have an existing entry */
+  e = digestmap_get(mbw_cache, parsed_line->node_id);
+  /* If we do, we can re-use it */
+  if (e) {
+    /* Check that we really are newer, and update */
+    if (as_of > e->as_of) {
+      e->mbw_kb = parsed_line->bw_kb;
+      e->as_of = as_of;
+    }
+  } else {
+    /* We'll have to insert a new entry */
+    e = tor_malloc(sizeof(*e));
+    e->mbw_kb = parsed_line->bw_kb;
+    e->as_of = as_of;
+    digestmap_set(mbw_cache, parsed_line->node_id, e);
+  }
+}
+
+/** Clear and free the measured bandwidth cache */
+void
+dirserv_clear_measured_bw_cache(void)
+{
+  if (mbw_cache) {
+    /* Free the map and all entries */
+    digestmap_free(mbw_cache, tor_free_);
+    mbw_cache = NULL;
+  }
+}
+
+/** Scan the measured bandwidth cache and remove expired entries */
+STATIC void
+dirserv_expire_measured_bw_cache(time_t now)
+{
+
+  if (mbw_cache) {
+    /* Iterate through the cache and check each entry */
+    DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
+      if (now > e->as_of + MAX_MEASUREMENT_AGE) {
+        tor_free(e);
+        MAP_DEL_CURRENT(k);
+      }
+    } DIGESTMAP_FOREACH_END;
+
+    /* Check if we cleared the whole thing and free if so */
+    if (digestmap_size(mbw_cache) == 0) {
+      digestmap_free(mbw_cache, tor_free_);
+      mbw_cache = 0;
+    }
+  }
+}
+
+/** Query the cache by identity digest, return value indicates whether
+ * we found it. The bw_out and as_of_out pointers receive the cached
+ * bandwidth value and the time it was cached if not NULL. */
+int
+dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
+                                   time_t *as_of_out)
+{
+  mbw_cache_entry_t *v = NULL;
+  int rv = 0;
+
+  if (mbw_cache && node_id) {
+    v = digestmap_get(mbw_cache, node_id);
+    if (v) {
+      /* Found something */
+      rv = 1;
+      if (bw_kb_out) *bw_kb_out = v->mbw_kb;
+      if (as_of_out) *as_of_out = v->as_of;
+    }
+  }
+
+  return rv;
+}
+
+/** Predicate wrapper for dirserv_query_measured_bw_cache() */
+int
+dirserv_has_measured_bw(const char *node_id)
+{
+  return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
+}
+
+/** Get the current size of the measured bandwidth cache */
+int
+dirserv_get_measured_bw_cache_size(void)
+{
+  if (mbw_cache) return digestmap_size(mbw_cache);
+  else return 0;
+}
+
+/** Return the bandwidth we believe for assigning flags; prefer measured
+ * over advertised, and if we have above a threshold quantity of measured
+ * bandwidths, we don't want to ever give flags to unmeasured routers, so
+ * return 0. */
+uint32_t
+dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
+{
+  int threshold;
+  uint32_t bw_kb = 0;
+  long mbw_kb;
+
+  tor_assert(ri);
+  /* Check if we have a measured bandwidth, and check the threshold if not */
+  if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                       &mbw_kb, NULL))) {
+    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    if (routers_with_measured_bw > threshold) {
+      /* Return zero for unmeasured bandwidth if we are above threshold */
+      bw_kb = 0;
+    } else {
+      /* Return an advertised bandwidth otherwise */
+      bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
+    }
+  } else {
+    /* We have the measured bandwidth in mbw */
+    bw_kb = (uint32_t)mbw_kb;
+  }
+
+  return bw_kb;
+}
+
+/**
+ * Read the measured bandwidth list file, apply it to the list of
+ * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Returns -1 on error, 0 otherwise.
+ */
+int
+dirserv_read_measured_bandwidths(const char *from_file,
+                                 smartlist_t *routerstatuses,
+                                 smartlist_t *bw_file_headers)
+{
+  FILE *fp = tor_fopen_cloexec(from_file, "r");
+  int applied_lines = 0;
+  time_t file_time, now;
+  int ok;
+   /* This flag will be 1 only when the first successful bw measurement line
+   * has been encountered, so that measured_bw_line_parse don't give warnings
+   * if there are additional header lines, as introduced in Bandwidth List spec
+   * version 1.1.0 */
+  int line_is_after_headers = 0;
+  int rv = -1;
+  char *line = NULL;
+  size_t n = 0;
+
+  /* Initialise line, so that we can't possibly run off the end. */
+
+  if (fp == NULL) {
+    log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
+             from_file);
+    goto err;
+  }
+
+  /* If fgets fails, line is either unmodified, or indeterminate. */
+  if (tor_getline(&line,&n,fp) <= 0) {
+    log_warn(LD_DIRSERV, "Empty bandwidth file");
+    goto err;
+  }
+
+  if (!strlen(line) || line[strlen(line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
+             escaped(line));
+    goto err;
+  }
+
+  line[strlen(line)-1] = '\0';
+  file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
+             escaped(line));
+    goto err;
+  }
+
+  now = time(NULL);
+  if ((now - file_time) > MAX_MEASUREMENT_AGE) {
+    log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
+             (unsigned)(time(NULL) - file_time));
+    goto err;
+  }
+
+  /* If timestamp was correct and bw_file_headers is not NULL,
+   * add timestamp to bw_file_headers */
+  if (bw_file_headers)
+    smartlist_add_asprintf(bw_file_headers, "timestamp=%lu",
+                           (unsigned long)file_time);
+
+  if (routerstatuses)
+    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
+
+  while (!feof(fp)) {
+    measured_bw_line_t parsed_line;
+    if (tor_getline(&line, &n, fp) >= 0) {
+      if (measured_bw_line_parse(&parsed_line, line,
+                                 line_is_after_headers) != -1) {
+        /* This condition will be true when the first complete valid bw line
+         * has been encountered, which means the end of the header lines. */
+        line_is_after_headers = 1;
+        /* Also cache the line for dirserv_get_bandwidth_for_router() */
+        dirserv_cache_measured_bw(&parsed_line, file_time);
+        if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
+          applied_lines++;
+      /* if the terminator is found, it is the end of header lines, set the
+       * flag but do not store anything */
+      } else if (strcmp(line, BW_FILE_HEADERS_TERMINATOR) == 0) {
+        line_is_after_headers = 1;
+      /* if the line was not a correct relay line nor the terminator and
+       * the end of the header lines has not been detected yet
+       * and it is key_value and bw_file_headers did not reach the maximum
+       * number of headers,
+       * then assume this line is a header and add it to bw_file_headers */
+      } else if (bw_file_headers &&
+              (line_is_after_headers == 0) &&
+              string_is_key_value(LOG_DEBUG, line) &&
+              !strchr(line, ' ') &&
+              (smartlist_len(bw_file_headers)
+               < MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
+        line[strlen(line)-1] = '\0';
+        smartlist_add_strdup(bw_file_headers, line);
+      };
+    }
+  }
+
+  /* Now would be a nice time to clean the cache, too */
+  dirserv_expire_measured_bw_cache(now);
+
+  log_info(LD_DIRSERV,
+           "Bandwidth measurement file successfully read. "
+           "Applied %d measurements.", applied_lines);
+  rv = 0;
+
+ err:
+  if (line) {
+    // we need to raw_free this buffer because we got it from tor_getdelim()
+    raw_free(line);
+  }
+  if (fp)
+    fclose(fp);
+  return rv;
+}
+
+/**
+ * Helper function to parse out a line in the measured bandwidth file
+ * into a measured_bw_line_t output structure.
+ *
+ * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
+ * bw line, return -1 and warn, since we are after the headers and we should
+ * only parse bw lines. Return 0 otherwise.
+ *
+ * If <b>line_is_after_headers</b> is false then it means that we are not past
+ * the header block yet. If we encounter an incomplete bw line, return -1 but
+ * don't warn since there could be additional header lines coming. If we
+ * encounter a proper bw line, return 0 (and we got past the headers).
+ */
+STATIC int
+measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
+                       int line_is_after_headers)
+{
+  char *line = tor_strdup(orig_line);
+  char *cp = line;
+  int got_bw = 0;
+  int got_node_id = 0;
+  char *strtok_state; /* lame sauce d'jour */
+
+  if (strlen(line) == 0) {
+    log_warn(LD_DIRSERV, "Empty line in bandwidth file");
+    tor_free(line);
+    return -1;
+  }
+
+  /* Remove end of line character, so that is not part of the token */
+  if (line[strlen(line) - 1] == '\n') {
+    line[strlen(line) - 1] = '\0';
+  }
+
+  cp = tor_strtok_r(cp, " \t", &strtok_state);
+
+  if (!cp) {
+    log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  if (orig_line[strlen(orig_line)-1] != '\n') {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+
+  do {
+    if (strcmpstart(cp, "bw=") == 0) {
+      int parse_ok = 0;
+      char *endptr;
+      if (got_bw) {
+        log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("bw=");
+
+      out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr);
+      if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
+        log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      got_bw=1;
+    } else if (strcmpstart(cp, "node_id=$") == 0) {
+      if (got_node_id) {
+        log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      cp+=strlen("node_id=$");
+
+      if (strlen(cp) != HEX_DIGEST_LEN ||
+          base16_decode(out->node_id, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s",
+                 escaped(orig_line));
+        tor_free(line);
+        return -1;
+      }
+      strlcpy(out->node_hex, cp, sizeof(out->node_hex));
+      got_node_id=1;
+    }
+  } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state)));
+
+  if (got_bw && got_node_id) {
+    tor_free(line);
+    return 0;
+  } else if (line_is_after_headers == 0) {
+    /* There could be additional header lines, therefore do not give warnings
+     * but returns -1 since it's not a complete bw line. */
+    log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  } else {
+    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
+  }
+}
+
+/**
+ * Helper function to apply a parsed measurement line to a list
+ * of bandwidth statuses. Returns true if a line is found,
+ * false otherwise.
+ */
+STATIC int
+measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                       smartlist_t *routerstatuses)
+{
+  vote_routerstatus_t *rs = NULL;
+  if (!routerstatuses)
+    return 0;
+
+  rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (rs) {
+    rs->has_measured_bw = 1;
+    rs->measured_bw_kb = (uint32_t)parsed_line->bw_kb;
+  } else {
+    log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list",
+             parsed_line->node_hex);
+  }
+
+  return rs != NULL;
+}

src/feature/dirauth/bwauth.h

@@ -0,0 +1,58 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.h
+ * \brief Header file for bwauth.c
+ **/
+
+#ifndef TOR_BWAUTH_H
+#define TOR_BWAUTH_H
+
+/** Maximum allowable length of bandwidth headers in a bandwidth file */
+#define MAX_BW_FILE_HEADER_COUNT_IN_VOTE 50
+
+/** Terminatore that separates bandwidth file headers from bandwidth file
+ * relay lines */
+#define BW_FILE_HEADERS_TERMINATOR "=====\n"
+
+int dirserv_read_measured_bandwidths(const char *from_file,
+                                     smartlist_t *routerstatuses,
+                                     smartlist_t *bw_file_headers);
+
+int dirserv_query_measured_bw_cache_kb(const char *node_id,
+                                       long *bw_out,
+                                       time_t *as_of_out);
+void dirserv_clear_measured_bw_cache(void);
+int dirserv_has_measured_bw(const char *node_id);
+int dirserv_get_measured_bw_cache_size(void);
+void dirserv_count_measured_bws(const smartlist_t *routers);
+int dirserv_get_last_n_measured_bws(void);
+
+uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
+
+#ifdef BWAUTH_PRIVATE
+typedef struct measured_bw_line_t {
+  char node_id[DIGEST_LEN];
+  char node_hex[MAX_HEX_NICKNAME_LEN+1];
+  long int bw_kb;
+} measured_bw_line_t;
+
+/* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
+#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
+
+STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line,
+                                  int line_is_after_headers);
+
+STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
+                           smartlist_t *routerstatuses);
+
+STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
+                               time_t as_of);
+STATIC void dirserv_expire_measured_bw_cache(time_t now);
+#endif /* defined(BWAUTH_PRIVATE) */
+
+#endif

src/feature/dirauth/dirvote.c

@@ -7,8 +7,12 @@
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "feature/dirauth/dircollate.h"
+#include "feature/dirauth/recommend_pkg.h"
+#include "feature/dirauth/voteflags.h"
 #include "feature/dircache/directory.h"
+#include "feature/dirauth/bwauth.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/guardfraction.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -19,8 +23,11 @@
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
+#include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/client/entrynodes.h" /* needed for guardfraction methods */
 #include "feature/nodelist/torcert.h"
 #include "feature/dircommon/voting_schedule.h"

src/feature/dirauth/dirvote.h

@@ -115,6 +115,10 @@ int dirvote_add_signatures(const char *detached_signatures_body,
                            const char *source,
                            const char **msg_out);
 
+struct config_line_t;
+char *format_recommended_version_list(const struct config_line_t *line,
+                                      int warn);
+
 #else /* HAVE_MODULE_DIRAUTH */
 
 static inline time_t
@@ -192,10 +196,6 @@ const cached_dir_t *dirvote_get_vote(const char *fp, int flags);
  * API used _only_ by the dirauth subsystem.
  */
 
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
 networkstatus_t *
 dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                                         authority_cert_t *cert);

src/feature/dirauth/guardfraction.c

@@ -0,0 +1,333 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bwauth.c
+ * \brief Code to read and apply guard fraction data.
+ **/
+
+#define GUARDFRACTION_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerparse.h"
+
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+
+/** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
+ *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
+ *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
+ *  information to it.
+ *
+ *  Return 1 if we applied the information and 0 if we couldn't find a
+ *  matching guard.
+ *
+ * Requires that <b>vote_routerstatuses</b> be sorted.
+ */
+static int
+guardfraction_line_apply(const char *guard_id,
+                      uint32_t guardfraction_percentage,
+                      smartlist_t *vote_routerstatuses)
+{
+  vote_routerstatus_t *vrs = NULL;
+
+  tor_assert(vote_routerstatuses);
+
+  vrs = smartlist_bsearch(vote_routerstatuses, guard_id,
+                         compare_digest_to_vote_routerstatus_entry);
+
+  if (!vrs) {
+    return 0;
+  }
+
+  vrs->status.has_guardfraction = 1;
+  vrs->status.guardfraction_percentage = guardfraction_percentage;
+
+  return 1;
+}
+
+/* Given a guard line from a guardfraction file, parse it and register
+ * its information to <b>vote_routerstatuses</b>.
+ *
+ * Return:
+ * * 1 if the line was proper and its information got registered.
+ * * 0 if the line was proper but no currently active guard was found
+ *     to register the guardfraction information to.
+ * * -1 if the line could not be parsed and set <b>err_msg</b> to a
+      newly allocated string containing the error message.
+ */
+static int
+guardfraction_file_parse_guard_line(const char *guard_line,
+                                    smartlist_t *vote_routerstatuses,
+                                    char **err_msg)
+{
+  char guard_id[DIGEST_LEN];
+  uint32_t guardfraction;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+
+  smartlist_t *sl = smartlist_new();
+  int retval = -1;
+
+  tor_assert(err_msg);
+
+  /* guard_line should contain something like this:
+     <hex digest> <guardfraction> <appearances> */
+  smartlist_split_string(sl, guard_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 3) {
+    tor_asprintf(err_msg, "bad line '%s'", guard_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  if (strlen(inputs_tmp) != HEX_DIGEST_LEN ||
+      base16_decode(guard_id, DIGEST_LEN,
+                    inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) {
+    tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  /* Guardfraction is an integer in [0, 100]. */
+  guardfraction =
+    (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp);
+    goto done;
+  }
+
+  /* If routerstatuses were provided, apply this info to actual routers. */
+  if (vote_routerstatuses) {
+    retval = guardfraction_line_apply(guard_id, guardfraction,
+                                      vote_routerstatuses);
+  } else {
+    retval = 0; /* If we got this far, line was correctly formatted. */
+  }
+
+ done:
+
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/** Given an inputs line from a guardfraction file, parse it and
+ *  register its information to <b>total_consensuses</b> and
+ *  <b>total_days</b>.
+ *
+ *  Return 0 if it parsed well. Return -1 if there was an error, and
+ *  set <b>err_msg</b> to a newly allocated string containing the
+ *  error message.
+ */
+static int
+guardfraction_file_parse_inputs_line(const char *inputs_line,
+                                     int *total_consensuses,
+                                     int *total_days,
+                                     char **err_msg)
+{
+  int retval = -1;
+  char *inputs_tmp = NULL;
+  int num_ok = 1;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(err_msg);
+
+  /* Second line is inputs information:
+   *   n-inputs <total_consensuses> <total_days>. */
+  smartlist_split_string(sl, inputs_line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+  if (smartlist_len(sl) < 2) {
+    tor_asprintf(err_msg, "incomplete line '%s'", inputs_line);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 0);
+  *total_consensuses =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp);
+    goto done;
+  }
+
+  inputs_tmp = smartlist_get(sl, 1);
+  *total_days =
+    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
+  if (!num_ok) {
+    tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp);
+    goto done;
+  }
+
+  retval = 0;
+
+ done:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+
+  return retval;
+}
+
+/* Maximum age of a guardfraction file that we are willing to accept. */
+#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */
+
+/** Static strings of guardfraction files. */
+#define GUARDFRACTION_DATE_STR "written-at"
+#define GUARDFRACTION_INPUTS "n-inputs"
+#define GUARDFRACTION_GUARD "guard-seen"
+#define GUARDFRACTION_VERSION "guardfraction-file-version"
+
+/** Given a guardfraction file in a string, parse it and register the
+ *  guardfraction information to the provided vote routerstatuses.
+ *
+ *  This is the rough format of the guardfraction file:
+ *
+ *      guardfraction-file-version 1
+ *      written-at <date and time>
+ *      n-inputs <number of consesuses parsed> <number of days considered>
+ *
+ *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 3> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 4> <guardfraction percentage> <consensus appearances>
+ *      guard-seen <fpr 5> <guardfraction percentage> <consensus appearances>
+ *      ...
+ *
+ *  Return -1 if the parsing failed and 0 if it went smoothly. Parsing
+ *  should tolerate errors in all lines but the written-at header.
+ */
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses)
+{
+  config_line_t *front=NULL, *line;
+  int ret_tmp;
+  int retval = -1;
+  int current_line_n = 0; /* line counter for better log messages */
+
+  /* Guardfraction info to be parsed */
+  int total_consensuses = 0;
+  int total_days = 0;
+
+  /* Stats */
+  int guards_read_n = 0;
+  int guards_applied_n = 0;
+
+  /* Parse file and split it in lines */
+  ret_tmp = config_get_lines(guardfraction_file_str, &front, 0);
+  if (ret_tmp < 0) {
+    log_warn(LD_CONFIG, "Error reading from guardfraction file");
+    goto done;
+  }
+
+  /* Sort routerstatuses (needed later when applying guardfraction info) */
+  if (vote_routerstatuses)
+    smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries);
+
+  for (line = front; line; line=line->next) {
+    current_line_n++;
+
+    if (!strcmp(line->key, GUARDFRACTION_VERSION)) {
+      int num_ok = 1;
+      unsigned int version;
+
+      version =
+        (unsigned int) tor_parse_long(line->value,
+                                      10, 0, INT_MAX, &num_ok, NULL);
+
+      if (!num_ok || version != 1) {
+        log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version);
+        goto done;
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) {
+      time_t file_written_at;
+      time_t now = time(NULL);
+
+      /* First line is 'written-at <date>' */
+      if (parse_iso_time(line->value, &file_written_at) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+      if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'",
+                 current_line_n, line->value);
+        goto done; /* don't tolerate failure here. */
+      }
+    } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) {
+      char *err_msg = NULL;
+
+      if (guardfraction_file_parse_inputs_line(line->value,
+                                               &total_consensuses,
+                                               &total_days,
+                                               &err_msg) < 0) {
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+    } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) {
+      char *err_msg = NULL;
+
+      ret_tmp = guardfraction_file_parse_guard_line(line->value,
+                                                    vote_routerstatuses,
+                                                    &err_msg);
+      if (ret_tmp < 0) { /* failed while parsing the guard line */
+        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
+                 current_line_n, err_msg);
+        tor_free(err_msg);
+        continue;
+      }
+
+      /* Successfully parsed guard line. Check if it was applied properly. */
+      guards_read_n++;
+      if (ret_tmp > 0) {
+        guards_applied_n++;
+      }
+    } else {
+      log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)",
+               current_line_n, line->key, line->value);
+    }
+  }
+
+  retval = 0;
+
+  log_info(LD_CONFIG,
+           "Successfully parsed guardfraction file with %d consensuses over "
+           "%d days. Parsed %d nodes and applied %d of them%s.",
+           total_consensuses, total_days, guards_read_n, guards_applied_n,
+           vote_routerstatuses ? "" : " (no routerstatus provided)" );
+
+ done:
+  config_free_lines(front);
+
+  if (retval < 0) {
+    return retval;
+  } else {
+    return guards_read_n;
+  }
+}
+
+/** Read a guardfraction file at <b>fname</b> and load all its
+ *  information to <b>vote_routerstatuses</b>. */
+int
+dirserv_read_guardfraction_file(const char *fname,
+                             smartlist_t *vote_routerstatuses)
+{
+  char *guardfraction_file_str;
+
+  /* Read file to a string */
+  guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!guardfraction_file_str) {
+      log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname);
+      return -1;
+  }
+
+  return dirserv_read_guardfraction_file_from_str(guardfraction_file_str,
+                                               vote_routerstatuses);
+}

src/feature/dirauth/guardfraction.h

@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file guardfraction.h
+ * \brief Header file for guardfraction.c
+ **/
+
+#ifndef TOR_GUARDFRACTION_H
+#define TOR_GUARDFRACTION_H
+
+#ifdef GUARDFRACTION_PRIVATE
+STATIC int
+dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
+                                      smartlist_t *vote_routerstatuses);
+#endif /* defined(DIRSERV_PRIVATE) */
+
+int dirserv_read_guardfraction_file(const char *fname,
+                                 smartlist_t *vote_routerstatuses);
+
+#endif

src/feature/dirauth/process_descs.c

@@ -0,0 +1,835 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_descs.c
+ * \brief Make decisions about uploaded descriptors
+ *
+ * Authorities use the code in this module to decide what to do with just-
+ * uploaded descriptors, and to manage the fingerprint file that helps
+ * them make those decisions.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/dircache/directory.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerparse.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+
+#include "core/or/tor_version_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+#include "lib/encoding/confline.h"
+
+/** How far in the future do we allow a router to get? (seconds) */
+#define ROUTER_ALLOW_SKEW (60*60*12)
+
+static void directory_remove_invalid(void);
+struct authdir_config_t;
+static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
+                                                const char **msg);
+static uint32_t
+dirserv_get_status_impl(const char *fp, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg,
+                        int severity);
+
+/*                 1  Historically used to indicate Named */
+#define FP_INVALID 2  /**< Believed invalid. */
+#define FP_REJECT  4  /**< We will not publish this router. */
+/*                 8  Historically used to avoid using this as a dir. */
+#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
+/*                 32 Historically used to indicade Unnamed */
+
+/** Target of status_by_digest map. */
+typedef uint32_t router_status_t;
+
+static void add_fingerprint_to_dir(const char *fp,
+                                   struct authdir_config_t *list,
+                                   router_status_t add_status);
+
+/** List of nickname-\>identity fingerprint mappings for all the routers
+ * that we name.  Used to prevent router impersonation. */
+typedef struct authdir_config_t {
+  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
+  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
+} authdir_config_t;
+
+/** Should be static; exposed for testing. */
+static authdir_config_t *fingerprint_list = NULL;
+
+/** Allocate and return a new, empty, authdir_config_t. */
+static authdir_config_t *
+authdir_config_new(void)
+{
+  authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
+  list->fp_by_name = strmap_new();
+  list->status_by_digest = digestmap_new();
+  return list;
+}
+
+/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
+ * <b>list</b>, or-ing the currently set status flags with
+ * <b>add_status</b>.
+ */
+/* static */ void
+add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
+                       router_status_t add_status)
+{
+  char *fingerprint;
+  char d[DIGEST_LEN];
+  router_status_t *status;
+  tor_assert(fp);
+  tor_assert(list);
+
+  fingerprint = tor_strdup(fp);
+  tor_strstrip(fingerprint, " ");
+  if (base16_decode(d, DIGEST_LEN,
+                    fingerprint, strlen(fingerprint)) != DIGEST_LEN) {
+    log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
+             escaped(fp));
+    tor_free(fingerprint);
+    return;
+  }
+
+  status = digestmap_get(list->status_by_digest, d);
+  if (!status) {
+    status = tor_malloc_zero(sizeof(router_status_t));
+    digestmap_set(list->status_by_digest, d, status);
+  }
+
+  tor_free(fingerprint);
+  *status |= add_status;
+  return;
+}
+
+/** Add the fingerprint for this OR to the global list of recognized
+ * identity key fingerprints. */
+int
+dirserv_add_own_fingerprint(crypto_pk_t *pk)
+{
+  char fp[FINGERPRINT_LEN+1];
+  if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
+    log_err(LD_BUG, "Error computing fingerprint");
+    return -1;
+  }
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+  add_fingerprint_to_dir(fp, fingerprint_list, 0);
+  return 0;
+}
+
+/** Load the nickname-\>fingerprint mappings stored in the approved-routers
+ * file.  The file format is line-based, with each non-blank holding one
+ * nickname, some space, and a fingerprint for that nickname.  On success,
+ * replace the current fingerprint list with the new list and return 0.  On
+ * failure, leave the current fingerprint list untouched, and return -1. */
+int
+dirserv_load_fingerprint_file(void)
+{
+  char *fname;
+  char *cf;
+  char *nickname, *fingerprint;
+  authdir_config_t *fingerprint_list_new;
+  int result;
+  config_line_t *front=NULL, *list;
+
+  fname = get_datadir_fname("approved-routers");
+  log_info(LD_GENERAL,
+           "Reloading approved fingerprints from \"%s\"...", fname);
+
+  cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (!cf) {
+    log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname);
+    tor_free(fname);
+    return 0;
+  }
+  tor_free(fname);
+
+  result = config_get_lines(cf, &front, 0);
+  tor_free(cf);
+  if (result < 0) {
+    log_warn(LD_CONFIG, "Error reading from fingerprint file");
+    return -1;
+  }
+
+  fingerprint_list_new = authdir_config_new();
+
+  for (list=front; list; list=list->next) {
+    char digest_tmp[DIGEST_LEN];
+    router_status_t add_status = 0;
+    nickname = list->key; fingerprint = list->value;
+    tor_strstrip(fingerprint, " "); /* remove spaces */
+    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
+        base16_decode(digest_tmp, sizeof(digest_tmp),
+                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
+      log_notice(LD_CONFIG,
+                 "Invalid fingerprint (nickname '%s', "
+                 "fingerprint %s). Skipping.",
+                 nickname, fingerprint);
+      continue;
+    }
+    if (!strcasecmp(nickname, "!reject")) {
+        add_status = FP_REJECT;
+    } else if (!strcasecmp(nickname, "!badexit")) {
+        add_status = FP_BADEXIT;
+    } else if (!strcasecmp(nickname, "!invalid")) {
+        add_status = FP_INVALID;
+    }
+    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
+  }
+
+  config_free_lines(front);
+  dirserv_free_fingerprint_list();
+  fingerprint_list = fingerprint_list_new;
+  /* Delete any routers whose fingerprints we no longer recognize */
+  directory_remove_invalid();
+  return 0;
+}
+
+/* If this is set, then we don't allow routers that have advertised an Ed25519
+ * identity to stop doing so.  This is going to be essential for good identity
+ * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could
+ * just sign fake descriptors missing the Ed25519 key.  But we won't actually
+ * be able to prevent that kind of thing until we're confident that there isn't
+ * actually a legit reason to downgrade to 0.2.5.  Now we are not recommending
+ * 0.2.5 anymore so there is no reason to keep the #undef.
+ */
+
+#define DISABLE_DISABLING_ED25519
+
+/** Check whether <b>router</b> has a nickname/identity key combination that
+ * we recognize from the fingerprint list, or an IP we automatically act on
+ * according to our configuration.  Return the appropriate router status.
+ *
+ * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
+ * *<b>msg</b> to an explanation of why. */
+uint32_t
+dirserv_router_get_status(const routerinfo_t *router, const char **msg,
+                          int severity)
+{
+  char d[DIGEST_LEN];
+  const int key_pinning = get_options()->AuthDirPinKeys;
+
+  if (crypto_pk_get_digest(router->identity_pkey, d)) {
+    log_warn(LD_BUG,"Error computing fingerprint");
+    if (msg)
+      *msg = "Bug: Error computing fingerprint";
+    return FP_REJECT;
+  }
+
+  /* Check for the more usual versions to reject a router first. */
+  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
+                                             router->addr, router->or_port,
+                                             router->platform, msg, severity);
+  if (r)
+    return r;
+
+  /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
+   * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
+   * But just in case a relay doesn't provide or lies about its version, or
+   * doesn't include an ntor key in its descriptor, check that it exists,
+   * and is non-zero (clients check that it's non-zero before using it). */
+  if (!routerinfo_has_curve25519_onion_key(router)) {
+    log_fn(severity, LD_DIR,
+           "Descriptor from router %s is missing an ntor curve25519 onion "
+           "key.", router_describe(router));
+    if (msg)
+      *msg = "Missing ntor curve25519 onion key. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  if (router->cache_info.signing_key_cert) {
+    /* This has an ed25519 identity key. */
+    if (KEYPIN_MISMATCH ==
+        keypin_check((const uint8_t*)router->cache_info.identity_digest,
+                   router->cache_info.signing_key_cert->signing_key.pubkey)) {
+      log_fn(severity, LD_DIR,
+             "Descriptor from router %s has an Ed25519 key, "
+               "but the <rsa,ed25519> keys don't match what they were before.",
+               router_describe(router));
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key or RSA identity key has changed.";
+        }
+        return FP_REJECT;
+      }
+    }
+  } else {
+    /* No ed25519 key */
+    if (KEYPIN_MISMATCH == keypin_check_lone_rsa(
+                        (const uint8_t*)router->cache_info.identity_digest)) {
+      log_fn(severity, LD_DIR,
+               "Descriptor from router %s has no Ed25519 key, "
+               "when we previously knew an Ed25519 for it. Ignoring for now, "
+               "since Ed25519 keys are fairly new.",
+               router_describe(router));
+#ifdef DISABLE_DISABLING_ED25519
+      if (key_pinning) {
+        if (msg) {
+          *msg = "Ed25519 identity key has disappeared.";
+        }
+        return FP_REJECT;
+      }
+#endif /* defined(DISABLE_DISABLING_ED25519) */
+    }
+  }
+
+  return 0;
+}
+
+/** Return true if there is no point in downloading the router described by
+ * <b>rs</b> because this directory would reject it. */
+int
+dirserv_would_reject_router(const routerstatus_t *rs)
+{
+  uint32_t res;
+
+  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
+                                rs->addr, rs->or_port,
+                                NULL, NULL, LOG_DEBUG);
+
+  return (res & FP_REJECT) != 0;
+}
+
+/** Helper: As dirserv_router_get_status, but takes the router fingerprint
+ * (hex, no spaces), nickname, address (used for logging only), IP address, OR
+ * port and platform (logging only) as arguments.
+ *
+ * Log messages at 'severity'. (There's not much point in
+ * logging that we're rejecting servers we'll not download.)
+ */
+static uint32_t
+dirserv_get_status_impl(const char *id_digest, const char *nickname,
+                        uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg, int severity)
+{
+  uint32_t result = 0;
+  router_status_t *status_by_digest;
+
+  if (!fingerprint_list)
+    fingerprint_list = authdir_config_new();
+
+  log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.",
+            strmap_size(fingerprint_list->fp_by_name),
+            digestmap_size(fingerprint_list->status_by_digest));
+
+  if (platform) {
+    tor_version_t ver_tmp;
+    if (tor_version_parse_platform(platform, &ver_tmp, 1) < 0) {
+      if (msg) {
+        *msg = "Malformed platform string.";
+      }
+      return FP_REJECT;
+    }
+  }
+
+  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
+   * missing some important security fixes too. Disable them. */
+  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
+    if (msg)
+      *msg = "Tor version is insecure or unsupported. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
+   * keep their consensus up to date so they make bad guards.
+   * The simple fix is to just drop them from the network. */
+  if (platform &&
+      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
+      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
+    if (msg)
+      *msg = "Tor version contains bug 20499. Please upgrade!";
+    return FP_REJECT;
+  }
+
+  status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
+                                   id_digest);
+  if (status_by_digest)
+    result |= *status_by_digest;
+
+  if (result & FP_REJECT) {
+    if (msg)
+      *msg = "Fingerprint is marked rejected -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your fingerprint(s)?";
+    return FP_REJECT;
+  } else if (result & FP_INVALID) {
+    if (msg)
+      *msg = "Fingerprint is marked invalid";
+  }
+
+  if (authdir_policy_badexit_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Marking '%s' as bad exit because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_BADEXIT;
+  }
+
+  if (!authdir_policy_permits_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'",
+               nickname, fmt_addr32(addr));
+    if (msg)
+      *msg = "Suspicious relay address range -- if you think this is a "
+             "mistake please set a valid email address in ContactInfo and "
+             "send an email to bad-relays@lists.torproject.org mentioning "
+             "your address(es) and fingerprint(s)?";
+    return FP_REJECT;
+  }
+  if (!authdir_policy_valid_address(addr, or_port)) {
+    log_fn(severity, LD_DIRSERV,
+           "Not marking '%s' valid because of address '%s'",
+               nickname, fmt_addr32(addr));
+    result |= FP_INVALID;
+  }
+
+  return result;
+}
+
+/** Clear the current fingerprint list. */
+void
+dirserv_free_fingerprint_list(void)
+{
+  if (!fingerprint_list)
+    return;
+
+  strmap_free(fingerprint_list->fp_by_name, tor_free_);
+  digestmap_free(fingerprint_list->status_by_digest, tor_free_);
+  tor_free(fingerprint_list);
+}
+
+/*
+ *    Descriptor list
+ */
+
+/** Return -1 if <b>ri</b> has a private or otherwise bad address,
+ * unless we're configured to not care. Return 0 if all ok. */
+static int
+dirserv_router_has_valid_address(routerinfo_t *ri)
+{
+  tor_addr_t addr;
+  if (get_options()->DirAllowPrivateAddresses)
+    return 0; /* whatever it is, we're fine with it */
+  tor_addr_from_ipv4h(&addr, ri->addr);
+
+  if (tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IP address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
+  return 0;
+}
+
+/** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
+ * set its is_valid,running fields and return 0.  Otherwise, return -1.
+ *
+ * If the router is rejected, set *<b>msg</b> to an explanation of why.
+ *
+ * If <b>complain</b> then explain at log-level 'notice' why we refused
+ * a descriptor; else explain at log-level 'info'.
+ */
+int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain, int *valid_out)
+{
+  /* Okay.  Now check whether the fingerprint is recognized. */
+  time_t now;
+  int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
+  uint32_t status = dirserv_router_get_status(ri, msg, severity);
+  tor_assert(msg);
+  if (status & FP_REJECT)
+    return -1; /* msg is already set. */
+
+  /* Is there too much clock skew? */
+  now = time(NULL);
+  if (ri->cache_info.published_on > now+ROUTER_ALLOW_SKEW) {
+    log_fn(severity, LD_DIRSERV, "Publication time for %s is too "
+           "far (%d minutes) in the future; possible clock skew. Not adding "
+           "(%s)",
+           router_describe(ri),
+           (int)((ri->cache_info.published_on-now)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Your clock is set too far in the future, or your "
+      "timezone is not correct.";
+    return -1;
+  }
+  if (ri->cache_info.published_on < now-ROUTER_MAX_AGE_TO_PUBLISH) {
+    log_fn(severity, LD_DIRSERV,
+           "Publication time for %s is too far "
+           "(%d minutes) in the past. Not adding (%s)",
+           router_describe(ri),
+           (int)((now-ri->cache_info.published_on)/60),
+           esc_router_info(ri));
+    *msg = "Rejected: Server is expired, or your clock is too far in the past,"
+      " or your timezone is not correct.";
+    return -1;
+  }
+  if (dirserv_router_has_valid_address(ri) < 0) {
+    log_fn(severity, LD_DIRSERV,
+           "Router %s has invalid address. Not adding (%s).",
+           router_describe(ri),
+           esc_router_info(ri));
+    *msg = "Rejected: Address is a private address.";
+    return -1;
+  }
+
+  *valid_out = ! (status & FP_INVALID);
+
+  return 0;
+}
+
+/** Update the relevant flags of <b>node</b> based on our opinion as a
+ * directory authority in <b>authstatus</b>, as returned by
+ * dirserv_router_get_status or equivalent.  */
+void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
+  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
+}
+
+/** True iff <b>a</b> is more severe than <b>b</b>. */
+static int
+WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
+{
+  return a < b;
+}
+
+/** As for dirserv_add_descriptor(), but accepts multiple documents, and
+ * returns the most severe error that occurred for any one of them. */
+was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  was_router_added_t r, r_tmp;
+  const char *msg_out;
+  smartlist_t *list;
+  const char *s;
+  int n_parsed = 0;
+  time_t now = time(NULL);
+  char annotation_buf[ROUTER_ANNOTATION_BUF_LEN];
+  char time_buf[ISO_TIME_LEN+1];
+  int general = purpose == ROUTER_PURPOSE_GENERAL;
+  tor_assert(msg);
+
+  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+
+  format_iso_time(time_buf, now);
+  if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
+                   "@uploaded-at %s\n"
+                   "@source %s\n"
+                   "%s%s%s", time_buf, escaped(source),
+                   !general ? "@purpose " : "",
+                   !general ? router_purpose_to_string(purpose) : "",
+                   !general ? "\n" : "")<0) {
+    *msg = "Couldn't format annotations";
+    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
+     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
+    return -1;
+  }
+
+  s = desc;
+  list = smartlist_new();
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+                                     annotation_buf, NULL)) {
+    SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
+        msg_out = NULL;
+        tor_assert(ri->purpose == purpose);
+        r_tmp = dirserv_add_descriptor(ri, &msg_out, source);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_clear(list);
+
+  s = desc;
+  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+                                     NULL, NULL)) {
+    SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
+        msg_out = NULL;
+
+        r_tmp = dirserv_add_extrainfo(ei, &msg_out);
+        if (WRA_MORE_SEVERE(r_tmp, r)) {
+          r = r_tmp;
+          *msg = msg_out;
+        }
+      });
+  }
+  n_parsed += smartlist_len(list);
+  smartlist_free(list);
+
+  if (! *msg) {
+    if (!n_parsed) {
+      *msg = "No descriptors found in your POST.";
+      if (WRA_WAS_ADDED(r))
+        r = ROUTER_IS_ALREADY_KNOWN;
+    } else {
+      *msg = "(no message)";
+    }
+  }
+
+  return r;
+}
+
+/** Examine the parsed server descriptor in <b>ri</b> and maybe insert it into
+ * the list of server descriptors. Set *<b>msg</b> to a message that should be
+ * passed back to the origin of this descriptor, or NULL if there is no such
+ * message. Use <b>source</b> to produce better log messages.
+ *
+ * If <b>ri</b> is not added to the list of server descriptors, free it.
+ * That means the caller must not access <b>ri</b> after this function
+ * returns, since it might have been freed.
+ *
+ * Return the status of the operation.
+ *
+ * This function is only called when fresh descriptors are posted, not when
+ * we re-load the cache.
+ */
+was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
+{
+  was_router_added_t r;
+  routerinfo_t *ri_old;
+  char *desc, *nickname;
+  const size_t desclen = ri->cache_info.signed_descriptor_len +
+      ri->cache_info.annotations_len;
+  const int key_pinning = get_options()->AuthDirPinKeys;
+  *msg = NULL;
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ri->cache_info.signed_descriptor_len > MAX_DESCRIPTOR_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish a router descriptor '%s'"
+               " (source: %s) with size %d. Either this is an attack, or the "
+               "MAX_DESCRIPTOR_UPLOAD_SIZE (%d) constant is too low.",
+               ri->nickname, source, (int)ri->cache_info.signed_descriptor_len,
+               MAX_DESCRIPTOR_UPLOAD_SIZE);
+    *msg = "Router descriptor was too large.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Check whether this descriptor is semantically identical to the last one
+   * from this server.  (We do this here and not in router_add_to_routerlist
+   * because we want to be able to accept the newest router descriptor that
+   * another authority has, so we all converge on the same one.) */
+  ri_old = router_get_mutable_by_digest(ri->cache_info.identity_digest);
+  if (ri_old && ri_old->cache_info.published_on < ri->cache_info.published_on
+      && router_differences_are_cosmetic(ri_old, ri)
+      && !router_is_me(ri)) {
+    log_info(LD_DIRSERV,
+             "Not replacing descriptor from %s (source: %s); "
+             "differences are cosmetic.",
+             router_describe(ri), source);
+    *msg = "Not replacing router descriptor; no information has changed since "
+      "the last one with this identity.";
+    r = ROUTER_IS_ALREADY_KNOWN;
+    goto fail;
+  }
+
+  /* Do keypinning again ... this time, to add the pin if appropriate */
+  int keypin_status;
+  if (ri->cache_info.signing_key_cert) {
+    ed25519_public_key_t *pkey = &ri->cache_info.signing_key_cert->signing_key;
+    /* First let's validate this pubkey before pinning it */
+    if (ed25519_validate_pubkey(pkey) < 0) {
+      log_warn(LD_DIRSERV, "Received bad key from %s (source %s)",
+               router_describe(ri), source);
+      routerinfo_free(ri);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+
+    /* Now pin it! */
+    keypin_status = keypin_check_and_add(
+      (const uint8_t*)ri->cache_info.identity_digest,
+      pkey->pubkey, ! key_pinning);
+  } else {
+    keypin_status = keypin_check_lone_rsa(
+      (const uint8_t*)ri->cache_info.identity_digest);
+#ifndef DISABLE_DISABLING_ED25519
+    if (keypin_status == KEYPIN_MISMATCH)
+      keypin_status = KEYPIN_NOT_FOUND;
+#endif
+  }
+  if (keypin_status == KEYPIN_MISMATCH && key_pinning) {
+    log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
+             "its key did not match an older RSA/Ed25519 keypair",
+             router_describe(ri), source);
+    *msg = "Looks like your keypair has changed? This authority previously "
+      "recorded a different RSA identity for this Ed25519 identity (or vice "
+      "versa.) Did you replace or copy some of your key files, but not "
+      "the others? You should either restore the expected keypair, or "
+      "delete your keys and restart Tor to start your relay with a new "
+      "identity.";
+    r = ROUTER_AUTHDIR_REJECTS;
+    goto fail;
+  }
+
+  /* Make a copy of desc, since router_add_to_routerlist might free
+   * ri and its associated signed_descriptor_t. */
+  desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen);
+  nickname = tor_strdup(ri->nickname);
+
+  /* Tell if we're about to need to launch a test if we add this. */
+  ri->needs_retest_if_added =
+    dirserv_should_launch_reachability_test(ri, ri_old);
+
+  r = router_add_to_routerlist(ri, msg, 0, 0);
+  if (!WRA_WAS_ADDED(r)) {
+    /* unless the routerinfo was fine, just out-of-date */
+    log_info(LD_DIRSERV,
+             "Did not add descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg ? *msg : "(no message)");
+  } else {
+    smartlist_t *changed;
+
+    changed = smartlist_new();
+    smartlist_add(changed, ri);
+    routerlist_descriptors_added(changed, 0);
+    smartlist_free(changed);
+    if (!*msg) {
+      *msg =  "Descriptor accepted";
+    }
+    log_info(LD_DIRSERV,
+             "Added descriptor from '%s' (source: %s): %s.",
+             nickname, source, *msg);
+  }
+  tor_free(desc);
+  tor_free(nickname);
+  return r;
+ fail:
+  {
+    const char *desc_digest = ri->cache_info.signed_descriptor_digest;
+    download_status_t *dls =
+      router_get_dl_status_by_descriptor_digest(desc_digest);
+    if (dls) {
+      log_info(LD_GENERAL, "Marking router with descriptor %s as rejected, "
+               "and therefore undownloadable",
+               hex_str(desc_digest, DIGEST_LEN));
+      download_status_mark_impossible(dls);
+    }
+    routerinfo_free(ri);
+  }
+  return r;
+}
+
+/** As dirserv_add_descriptor, but for an extrainfo_t <b>ei</b>. */
+static was_router_added_t
+dirserv_add_extrainfo(extrainfo_t *ei, const char **msg)
+{
+  routerinfo_t *ri;
+  int r;
+  was_router_added_t rv;
+  tor_assert(msg);
+  *msg = NULL;
+
+  /* Needs to be mutable so routerinfo_incompatible_with_extrainfo
+   * can mess with some of the flags in ri->cache_info. */
+  ri = router_get_mutable_by_digest(ei->cache_info.identity_digest);
+  if (!ri) {
+    *msg = "No corresponding router descriptor for extra-info descriptor";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  /* If it's too big, refuse it now. Otherwise we'll cache it all over the
+   * network and it'll clog everything up. */
+  if (ei->cache_info.signed_descriptor_len > MAX_EXTRAINFO_UPLOAD_SIZE) {
+    log_notice(LD_DIR, "Somebody attempted to publish an extrainfo "
+               "with size %d. Either this is an attack, or the "
+               "MAX_EXTRAINFO_UPLOAD_SIZE (%d) constant is too low.",
+               (int)ei->cache_info.signed_descriptor_len,
+               MAX_EXTRAINFO_UPLOAD_SIZE);
+    *msg = "Extrainfo document was too large";
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+
+  if ((r = routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                  &ri->cache_info, msg))) {
+    if (r<0) {
+      extrainfo_free(ei);
+      return ROUTER_IS_ALREADY_KNOWN;
+    }
+    rv = ROUTER_BAD_EI;
+    goto fail;
+  }
+  router_add_extrainfo_to_routerlist(ei, msg, 0, 0);
+  return ROUTER_ADDED_SUCCESSFULLY;
+ fail:
+  {
+    const char *d = ei->cache_info.signed_descriptor_digest;
+    signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d);
+    if (sd) {
+      log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as "
+               "rejected, and therefore undownloadable",
+               hex_str((char*)d,DIGEST_LEN));
+      download_status_mark_impossible(&sd->ei_dl_status);
+    }
+    extrainfo_free(ei);
+  }
+  return rv;
+}
+
+/** Remove all descriptors whose nicknames or fingerprints no longer
+ * are allowed by our fingerprint list. (Descriptors that used to be
+ * good can become bad when we reload the fingerprint list.)
+ */
+static void
+directory_remove_invalid(void)
+{
+  routerlist_t *rl = router_get_routerlist();
+  smartlist_t *nodes = smartlist_new();
+  smartlist_add_all(nodes, nodelist_get_list());
+
+  SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) {
+    const char *msg = NULL;
+    const char *description;
+    routerinfo_t *ent = node->ri;
+    uint32_t r;
+    if (!ent)
+      continue;
+    r = dirserv_router_get_status(ent, &msg, LOG_INFO);
+    description = router_describe(ent);
+    if (r & FP_REJECT) {
+      log_info(LD_DIRSERV, "Router %s is now rejected: %s",
+               description, msg?msg:"");
+      routerlist_remove(rl, ent, 0, time(NULL));
+      continue;
+    }
+    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
+      log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
+               (r&FP_INVALID) ? "in" : "");
+      node->is_valid = (r&FP_INVALID)?0:1;
+    }
+    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
+      log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
+               (r & FP_BADEXIT) ? "bad" : "good");
+      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  routerlist_assert_ok(rl);
+  smartlist_free(nodes);
+}

src/feature/dirauth/process_descs.h

@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_descs.h
+ * \brief Header file for process_descs.c.
+ **/
+
+#ifndef TOR_RECV_UPLOADS_H
+#define TOR_RECV_UPLOADS_H
+
+int dirserv_load_fingerprint_file(void);
+void dirserv_free_fingerprint_list(void);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk);
+
+enum was_router_added_t dirserv_add_multiple_descriptors(
+                                     const char *desc, uint8_t purpose,
+                                     const char *source,
+                                     const char **msg);
+enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
+                                               const char **msg,
+                                               const char *source);
+
+int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                                   int complain,
+                                   int *valid_out);
+uint32_t dirserv_router_get_status(const routerinfo_t *router,
+                                   const char **msg,
+                                   int severity);
+void dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                      uint32_t authstatus);
+
+int dirserv_would_reject_router(const routerstatus_t *rs);
+
+#endif

src/feature/dirauth/reachability.c

@@ -0,0 +1,205 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file reachability.c
+ * \brief Router reachability testing; run by authorities to tell who is
+ * running.
+ */
+
+#include "core/or/or.h"
+#include "feature/dirauth/reachability.h"
+
+#include "app/config/config.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/command.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/relay/router.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+/** Called when a TLS handshake has completed successfully with a
+ * router listening at <b>address</b>:<b>or_port</b>, and has yielded
+ * a certificate with digest <b>digest_rcvd</b>.
+ *
+ * Inform the reachability checker that we could get to this relay.
+ */
+void
+dirserv_orconn_tls_done(const tor_addr_t *addr,
+                        uint16_t or_port,
+                        const char *digest_rcvd,
+                        const ed25519_public_key_t *ed_id_rcvd)
+{
+  node_t *node = NULL;
+  tor_addr_port_t orport;
+  routerinfo_t *ri = NULL;
+  time_t now = time(NULL);
+  tor_assert(addr);
+  tor_assert(digest_rcvd);
+
+  node = node_get_mutable_by_id(digest_rcvd);
+  if (node == NULL || node->ri == NULL)
+    return;
+
+  ri = node->ri;
+
+  if (get_options()->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      ri->cache_info.signing_key_cert) {
+    /* We allow the node to have an ed25519 key if we haven't been told one in
+     * the routerinfo, but if we *HAVE* been told one in the routerinfo, it
+     * needs to match. */
+    const ed25519_public_key_t *expected_id =
+      &ri->cache_info.signing_key_cert->signing_key;
+    tor_assert(!ed25519_public_key_is_zero(expected_id));
+    if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) {
+      log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s "
+               "did not present expected Ed25519 ID.",
+               fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN));
+      return; /* Don't mark it as reachable. */
+    }
+  }
+
+  tor_addr_copy(&orport.addr, addr);
+  orport.port = or_port;
+  if (router_has_orport(ri, &orport)) {
+    /* Found the right router.  */
+    if (!authdir_mode_bridge(get_options()) ||
+        ri->purpose == ROUTER_PURPOSE_BRIDGE) {
+      char addrstr[TOR_ADDR_BUF_LEN];
+      /* This is a bridge or we're not a bridge authority --
+         mark it as reachable.  */
+      log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
+               router_describe(ri),
+               tor_addr_to_str(addrstr, addr, sizeof(addrstr), 1),
+               ri->or_port);
+      if (tor_addr_family(addr) == AF_INET) {
+        rep_hist_note_router_reachable(digest_rcvd, addr, or_port, now);
+        node->last_reachable = now;
+      } else if (tor_addr_family(addr) == AF_INET6) {
+        /* No rephist for IPv6.  */
+        node->last_reachable6 = now;
+      }
+    }
+  }
+}
+
+/** Called when we, as an authority, receive a new router descriptor either as
+ * an upload or a download.  Used to decide whether to relaunch reachability
+ * testing for the server. */
+int
+dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                        const routerinfo_t *ri_old)
+{
+  if (!authdir_mode_handles_descs(get_options(), ri->purpose))
+    return 0;
+  if (!ri_old) {
+    /* New router: Launch an immediate reachability test, so we will have an
+     * opinion soon in case we're generating a consensus soon */
+    return 1;
+  }
+  if (ri_old->is_hibernating && !ri->is_hibernating) {
+    /* It just came out of hibernation; launch a reachability test */
+    return 1;
+  }
+  if (! routers_have_same_or_addrs(ri, ri_old)) {
+    /* Address or port changed; launch a reachability test */
+    return 1;
+  }
+  return 0;
+}
+
+/** Helper function for dirserv_test_reachability(). Start a TLS
+ * connection to <b>router</b>, and annotate it with when we started
+ * the test. */
+void
+dirserv_single_reachability_test(time_t now, routerinfo_t *router)
+{
+  const or_options_t *options = get_options();
+  channel_t *chan = NULL;
+  const node_t *node = NULL;
+  tor_addr_t router_addr;
+  const ed25519_public_key_t *ed_id_key;
+  (void) now;
+
+  tor_assert(router);
+  node = node_get_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (options->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      router->cache_info.signing_key_cert) {
+    ed_id_key = &router->cache_info.signing_key_cert->signing_key;
+  } else {
+    ed_id_key = NULL;
+  }
+
+  /* IPv4. */
+  log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
+            router->nickname, fmt_addr32(router->addr), router->or_port);
+  tor_addr_from_ipv4h(&router_addr, router->addr);
+  chan = channel_tls_connect(&router_addr, router->or_port,
+                             router->cache_info.identity_digest,
+                             ed_id_key);
+  if (chan) command_setup_channel(chan);
+
+  /* Possible IPv6. */
+  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&router->ipv6_addr)) {
+    char addrstr[TOR_ADDR_BUF_LEN];
+    log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
+              router->nickname,
+              tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
+              router->ipv6_orport);
+    chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
+                               router->cache_info.identity_digest,
+                               ed_id_key);
+    if (chan) command_setup_channel(chan);
+  }
+}
+
+/** Auth dir server only: load balance such that we only
+ * try a few connections per call.
+ *
+ * The load balancing is such that if we get called once every ten
+ * seconds, we will cycle through all the tests in
+ * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes).
+ */
+void
+dirserv_test_reachability(time_t now)
+{
+  /* XXX decide what to do here; see or-talk thread "purging old router
+   * information, revocation." -NM
+   * We can't afford to mess with this in 0.1.2.x. The reason is that
+   * if we stop doing reachability tests on some of routerlist, then
+   * we'll for-sure think they're down, which may have unexpected
+   * effects in other parts of the code. It doesn't hurt much to do
+   * the testing, and directory authorities are easy to upgrade. Let's
+   * wait til 0.2.0. -RD */
+//  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  routerlist_t *rl = router_get_routerlist();
+  static char ctr = 0;
+  int bridge_auth = authdir_mode_bridge(get_options());
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) {
+    const char *id_digest = router->cache_info.identity_digest;
+    if (router_is_me(router))
+      continue;
+    if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue; /* bridge authorities only test reachability on bridges */
+//    if (router->cache_info.published_on > cutoff)
+//      continue;
+    if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) {
+      dirserv_single_reachability_test(now, router);
+    }
+  } SMARTLIST_FOREACH_END(router);
+  ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
+}

src/feature/dirauth/reachability.h

@@ -0,0 +1,36 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file reachability.h
+ * \brief Header file for reachability.c.
+ **/
+
+#ifndef TOR_REACHABILITY_H
+#define TOR_REACHABILITY_H
+
+/** What fraction (1 over this number) of the relay ID space do we
+ * (as a directory authority) launch connections to at each reachability
+ * test? */
+#define REACHABILITY_MODULO_PER_TEST 128
+
+/** How often (in seconds) do we launch reachability tests? */
+#define REACHABILITY_TEST_INTERVAL 10
+
+/** How many seconds apart are the reachability tests for a given relay? */
+#define REACHABILITY_TEST_CYCLE_PERIOD \
+  (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
+
+void dirserv_orconn_tls_done(const tor_addr_t *addr,
+                             uint16_t or_port,
+                             const char *digest_rcvd,
+                             const struct ed25519_public_key_t *ed_id_rcvd);
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+#endif

src/feature/dirauth/recommend_pkg.c

@@ -0,0 +1,90 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file recommend_pkg.c
+ * \brief Code related to the recommended-packages subsystem.
+ *
+ * Currently unused.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/recommend_pkg.h"
+
+/** Return true iff <b>line</b> is a valid RecommendedPackages line.
+ */
+/*
+  The grammar is:
+
+    "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL
+
+      PACKAGENAME = NONSPACE
+      VERSION = NONSPACE
+      URL = NONSPACE
+      DIGESTS = DIGEST | DIGESTS SP DIGEST
+      DIGEST = DIGESTTYPE "=" DIGESTVAL
+
+      NONSPACE = one or more non-space printing characters
+
+      DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters.
+
+      SP = " "
+      NL = a newline
+
+ */
+int
+validate_recommended_package_line(const char *line)
+{
+  const char *cp = line;
+
+#define WORD()                                  \
+  do {                                          \
+    if (*cp == ' ')                             \
+      return 0;                                 \
+    cp = strchr(cp, ' ');                       \
+    if (!cp)                                    \
+      return 0;                                 \
+  } while (0)
+
+  WORD(); /* skip packagename */
+  ++cp;
+  WORD(); /* skip version */
+  ++cp;
+  WORD(); /* Skip URL */
+  ++cp;
+
+  /* Skip digesttype=digestval + */
+  int n_entries = 0;
+  while (1) {
+    const char *start_of_word = cp;
+    const char *end_of_word = strchr(cp, ' ');
+    if (! end_of_word)
+      end_of_word = cp + strlen(cp);
+
+    if (start_of_word == end_of_word)
+      return 0;
+
+    const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word);
+
+    if (!eq)
+      return 0;
+    if (eq == start_of_word)
+      return 0;
+    if (eq == end_of_word - 1)
+      return 0;
+    if (memchr(eq+1, '=', end_of_word - (eq+1)))
+      return 0;
+
+    ++n_entries;
+    if (0 == *end_of_word)
+      break;
+
+    cp = end_of_word + 1;
+  }
+
+  /* If we reach this point, we have at least 1 entry. */
+  tor_assert(n_entries > 0);
+  return 1;
+}

src/feature/dirauth/recommend_pkg.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file recommend_pkg.h
+ * \brief Header file for recommend_pkg.c
+ **/
+
+#ifndef TOR_RECOMMEND_PKG_H
+#define TOR_RECOMMEND_PKG_H
+
+int validate_recommended_package_line(const char *line);
+
+#endif

src/feature/dirauth/shared_random.c

@@ -96,7 +96,7 @@
 #include "feature/nodelist/networkstatus.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dirauth/shared_random_state.h"
 #include "feature/dircommon/voting_schedule.h"
@@ -1288,4 +1288,3 @@ set_num_srv_agreements(int32_t value)
 }
 
 #endif /* defined(TOR_UNIT_TESTS) */
-

src/feature/dirauth/voteflags.c

@@ -0,0 +1,644 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voteflags.c
+ * \brief Authority code for deciding the performance thresholds for flags,
+ *   and assigning flags to routers.
+ **/
+
+#define VOTEFLAGS_PRIVATE
+#include "core/or/or.h"
+#include "feature/dirauth/voteflags.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/main.h"
+#include "core/or/policies.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+#include "lib/container/order.h"
+
+/** If a router's uptime is at least this value, then it is always
+ * considered stable, regardless of the rest of the network. This
+ * way we resist attacks where an attacker doubles the size of the
+ * network using allegedly high-uptime nodes, displacing all the
+ * current guards. */
+#define UPTIME_TO_GUARANTEE_STABLE (3600*24*30)
+/** If a router's MTBF is at least this value, then it is always stable.
+ * See above.  (Corresponds to about 7 days for current decay rates.) */
+#define MTBF_TO_GUARANTEE_STABLE (60*60*24*5)
+/** Similarly, every node with at least this much weighted time known can be
+ * considered familiar enough to be a guard.  Corresponds to about 20 days for
+ * current decay rates.
+ */
+#define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60)
+/** Similarly, every node with sufficient WFU is around enough to be a guard.
+ */
+#define WFU_TO_GUARANTEE_GUARD (0.98)
+
+/* Thresholds for server performance: set by
+ * dirserv_compute_performance_thresholds, and used by
+ * generate_v2_networkstatus */
+
+/** Any router with an uptime of at least this value is stable. */
+static uint32_t stable_uptime = 0; /* start at a safe value */
+/** Any router with an mtbf of at least this value is stable. */
+static double stable_mtbf = 0.0;
+/** If true, we have measured enough mtbf info to look at stable_mtbf rather
+ * than stable_uptime. */
+static int enough_mtbf_info = 0;
+/** Any router with a weighted fractional uptime of at least this much might
+ * be good as a guard. */
+static double guard_wfu = 0.0;
+/** Don't call a router a guard unless we've known about it for at least this
+ * many seconds. */
+static long guard_tk = 0;
+/** Any router with a bandwidth at least this high is "Fast" */
+static uint32_t fast_bandwidth_kb = 0;
+/** If exits can be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_including_exits_kb = 0;
+/** If exits can't be guards, then all guards must have a bandwidth this
+ * high. */
+static uint32_t guard_bandwidth_excluding_exits_kb = 0;
+
+/** Helper: estimate the uptime of a router given its stated uptime and the
+ * amount of time since it last stated its stated uptime. */
+static inline long
+real_uptime(const routerinfo_t *router, time_t now)
+{
+  if (now < router->cache_info.published_on)
+    return router->uptime;
+  else
+    return router->uptime + (now - router->cache_info.published_on);
+}
+
+/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
+ * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
+ * If <b>need_capacity</b> is non-zero, we require a minimum advertised
+ * bandwidth.
+ */
+static int
+dirserv_thinks_router_is_unreliable(time_t now,
+                                    routerinfo_t *router,
+                                    int need_uptime, int need_capacity)
+{
+  if (need_uptime) {
+    if (!enough_mtbf_info) {
+      /* XXXX We should change the rule from
+       * "use uptime if we don't have mtbf data" to "don't advertise Stable on
+       * v3 if we don't have enough mtbf data."  Or maybe not, since if we ever
+       * hit a point where we need to reset a lot of authorities at once,
+       * none of them would be in a position to declare Stable.
+       */
+      long uptime = real_uptime(router, now);
+      if ((unsigned)uptime < stable_uptime &&
+          (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE)
+        return 1;
+    } else {
+      double mtbf =
+        rep_hist_get_stability(router->cache_info.identity_digest, now);
+      if (mtbf < stable_mtbf &&
+          mtbf < MTBF_TO_GUARANTEE_STABLE)
+        return 1;
+    }
+  }
+  if (need_capacity) {
+    uint32_t bw_kb = dirserv_get_credible_bandwidth_kb(router);
+    if (bw_kb < fast_bandwidth_kb)
+      return 1;
+  }
+  return 0;
+}
+
+/** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid,
+ * not hibernating, having observed bw greater 0, and not too old. Else
+ * return 0.
+ */
+static int
+router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
+{
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  if (ri->cache_info.published_on < cutoff) {
+    return 0;
+  }
+  if (!node->is_running || !node->is_valid || ri->is_hibernating) {
+    return 0;
+  }
+  /* Only require bandwidth capacity in non-test networks, or
+   * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
+  if (!ri->bandwidthcapacity) {
+    if (get_options()->TestingTorNetwork) {
+      if (get_options()->TestingMinExitFlagThreshold > 0) {
+        /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
+         * then require bandwidthcapacity */
+        return 0;
+      }
+    } else {
+      /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */
+      return 0;
+    }
+  }
+  return 1;
+}
+
+/** Return true iff <b>router</b> should be assigned the "HSDir" flag.
+ *
+ * Right now this means it advertises support for it, it has a high uptime,
+ * it's a directory cache, it has the Stable and Fast flags, and it's currently
+ * considered Running.
+ *
+ * This function needs to be called after router-\>is_running has
+ * been set.
+ */
+static int
+dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
+                                const node_t *node, time_t now)
+{
+
+  long uptime;
+
+  /* If we haven't been running for at least
+   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
+   * have accurate data telling us a relay has been up for at least
+   * that long. We also want to allow a bit of slack: Reachability
+   * tests aren't instant. If we haven't been running long enough,
+   * trust the relay. */
+
+  if (get_uptime() >
+      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
+    uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
+                 real_uptime(router, now));
+  else
+    uptime = real_uptime(router, now);
+
+  return (router->wants_to_be_hs_dir &&
+          router->supports_tunnelled_dir_requests &&
+          node->is_stable && node->is_fast &&
+          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
+          router_is_active(router, node, now));
+}
+
+/** Don't consider routers with less bandwidth than this when computing
+ * thresholds. */
+#define ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB 4
+
+/** Helper for dirserv_compute_performance_thresholds(): Decide whether to
+ * include a router in our calculations, and return true iff we should; the
+ * require_mbw parameter is passed in by
+ * dirserv_compute_performance_thresholds() and controls whether we ever
+ * count routers with only advertised bandwidths */
+static int
+router_counts_toward_thresholds(const node_t *node, time_t now,
+                                const digestmap_t *omit_as_sybil,
+                                int require_mbw)
+{
+  /* Have measured bw? */
+  int have_mbw =
+    dirserv_has_measured_bw(node->identity);
+  uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
+  const or_options_t *options = get_options();
+
+  if (options->TestingTorNetwork) {
+    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
+  }
+
+  return node->ri && router_is_active(node->ri, node, now) &&
+    !digestmap_get(omit_as_sybil, node->identity) &&
+    (dirserv_get_credible_bandwidth_kb(node->ri) >= min_bw_kb) &&
+    (have_mbw || !require_mbw);
+}
+
+/** Look through the routerlist, the Mean Time Between Failure history, and
+ * the Weighted Fractional Uptime history, and use them to set thresholds for
+ * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
+ * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth,
+ * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
+ *
+ * Also, set the is_exit flag of each router appropriately. */
+void
+dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
+{
+  int n_active, n_active_nonexit, n_familiar;
+  uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
+  long *tks;
+  double *mtbfs, *wfus;
+  smartlist_t *nodelist;
+  time_t now = time(NULL);
+  const or_options_t *options = get_options();
+
+  /* Require mbw? */
+  int require_mbw =
+    (dirserv_get_last_n_measured_bws() >
+     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
+
+  /* initialize these all here, in case there are no routers */
+  stable_uptime = 0;
+  stable_mtbf = 0;
+  fast_bandwidth_kb = 0;
+  guard_bandwidth_including_exits_kb = 0;
+  guard_bandwidth_excluding_exits_kb = 0;
+  guard_tk = 0;
+  guard_wfu = 0;
+
+  nodelist_assert_ok();
+  nodelist = nodelist_get_list();
+
+  /* Initialize arrays that will hold values for each router.  We'll
+   * sort them and use that to compute thresholds. */
+  n_active = n_active_nonexit = 0;
+  /* Uptime for every active router. */
+  uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active router. */
+  bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Bandwidth for every active non-exit router. */
+  bandwidths_excluding_exits_kb =
+    tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
+  /* Weighted mean time between failure for each active router. */
+  mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double));
+  /* Time-known for each active router. */
+  tks = tor_calloc(smartlist_len(nodelist), sizeof(long));
+  /* Weighted fractional uptime for each active router. */
+  wfus = tor_calloc(smartlist_len(nodelist), sizeof(double));
+
+  /* Now, fill in the arrays. */
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+    if (options->BridgeAuthoritativeDir &&
+        node->ri &&
+        node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
+      continue;
+
+    routerinfo_t *ri = node->ri;
+    if (ri) {
+      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
+                       exit_policy_is_general_exit(ri->exit_policy));
+    }
+
+    if (router_counts_toward_thresholds(node, now, omit_as_sybil,
+                                        require_mbw)) {
+      const char *id = node->identity;
+      uint32_t bw_kb;
+
+      /* resolve spurious clang shallow analysis null pointer errors */
+      tor_assert(ri);
+
+      uptimes[n_active] = (uint32_t)real_uptime(ri, now);
+      mtbfs[n_active] = rep_hist_get_stability(id, now);
+      tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
+      bandwidths_kb[n_active] = bw_kb = dirserv_get_credible_bandwidth_kb(ri);
+      if (!node->is_exit || node->is_bad_exit) {
+        bandwidths_excluding_exits_kb[n_active_nonexit] = bw_kb;
+        ++n_active_nonexit;
+      }
+      ++n_active;
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Now, compute thresholds. */
+  if (n_active) {
+    /* The median uptime is stable. */
+    stable_uptime = median_uint32(uptimes, n_active);
+    /* The median mtbf is stable, if we have enough mtbf info */
+    stable_mtbf = median_double(mtbfs, n_active);
+    /* The 12.5th percentile bandwidth is fast. */
+    fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8);
+    /* (Now bandwidths is sorted.) */
+    if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000))
+      fast_bandwidth_kb = bandwidths_kb[n_active/4];
+    guard_bandwidth_including_exits_kb =
+      third_quartile_uint32(bandwidths_kb, n_active);
+    guard_tk = find_nth_long(tks, n_active, n_active/8);
+  }
+
+  if (guard_tk > TIME_KNOWN_TO_GUARANTEE_FAMILIAR)
+    guard_tk = TIME_KNOWN_TO_GUARANTEE_FAMILIAR;
+
+  {
+    /* We can vote on a parameter for the minimum and maximum. */
+#define ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG 4
+    int32_t min_fast_kb, max_fast_kb, min_fast, max_fast;
+    min_fast = networkstatus_get_param(NULL, "FastFlagMinThreshold",
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
+      INT32_MAX);
+    if (options->TestingTorNetwork) {
+      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
+    }
+    max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
+                                       INT32_MAX, min_fast, INT32_MAX);
+    min_fast_kb = min_fast / 1000;
+    max_fast_kb = max_fast / 1000;
+
+    if (fast_bandwidth_kb < (uint32_t)min_fast_kb)
+      fast_bandwidth_kb = min_fast_kb;
+    if (fast_bandwidth_kb > (uint32_t)max_fast_kb)
+      fast_bandwidth_kb = max_fast_kb;
+  }
+  /* Protect sufficiently fast nodes from being pushed out of the set
+   * of Fast nodes. */
+  if (options->AuthDirFastGuarantee &&
+      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
+    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
+
+  /* Now that we have a time-known that 7/8 routers are known longer than,
+   * fill wfus with the wfu of every such "familiar" router. */
+  n_familiar = 0;
+
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+      if (router_counts_toward_thresholds(node, now,
+                                          omit_as_sybil, require_mbw)) {
+        routerinfo_t *ri = node->ri;
+        const char *id = ri->cache_info.identity_digest;
+        long tk = rep_hist_get_weighted_time_known(id, now);
+        if (tk < guard_tk)
+          continue;
+        wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now);
+      }
+  } SMARTLIST_FOREACH_END(node);
+  if (n_familiar)
+    guard_wfu = median_double(wfus, n_familiar);
+  if (guard_wfu > WFU_TO_GUARANTEE_GUARD)
+    guard_wfu = WFU_TO_GUARANTEE_GUARD;
+
+  enough_mtbf_info = rep_hist_have_measured_enough_stability();
+
+  if (n_active_nonexit) {
+    guard_bandwidth_excluding_exits_kb =
+      find_nth_uint32(bandwidths_excluding_exits_kb,
+                      n_active_nonexit, n_active_nonexit*3/4);
+  }
+
+  log_info(LD_DIRSERV,
+      "Cutoffs: For Stable, %lu sec uptime, %lu sec MTBF. "
+      "For Fast: %lu kilobytes/sec. "
+      "For Guard: WFU %.03f%%, time-known %lu sec, "
+      "and bandwidth %lu or %lu kilobytes/sec. "
+      "We%s have enough stability data.",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb,
+      (unsigned long)guard_bandwidth_excluding_exits_kb,
+      enough_mtbf_info ? "" : " don't");
+
+  tor_free(uptimes);
+  tor_free(mtbfs);
+  tor_free(bandwidths_kb);
+  tor_free(bandwidths_excluding_exits_kb);
+  tor_free(tks);
+  tor_free(wfus);
+}
+
+/* Use dirserv_compute_performance_thresholds() to compute the thresholds
+ * for the status flags, specifically for bridges.
+ *
+ * This is only called by a Bridge Authority from
+ * networkstatus_getinfo_by_purpose().
+ */
+void
+dirserv_compute_bridge_flag_thresholds(void)
+{
+  digestmap_t *omit_as_sybil = digestmap_new();
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+  digestmap_free(omit_as_sybil, NULL);
+}
+
+/** Give a statement of our current performance thresholds for inclusion
+ * in a vote document. */
+char *
+dirserv_get_flag_thresholds_line(void)
+{
+  char *result=NULL;
+  const int measured_threshold =
+    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+  const int enough_measured_bw =
+    dirserv_get_last_n_measured_bws() > measured_threshold;
+
+  tor_asprintf(&result,
+      "stable-uptime=%lu stable-mtbf=%lu "
+      "fast-speed=%lu "
+      "guard-wfu=%.03f%% guard-tk=%lu "
+      "guard-bw-inc-exits=%lu guard-bw-exc-exits=%lu "
+      "enough-mtbf=%d ignoring-advertised-bws=%d",
+      (unsigned long)stable_uptime,
+      (unsigned long)stable_mtbf,
+      (unsigned long)fast_bandwidth_kb*1000,
+      guard_wfu*100,
+      (unsigned long)guard_tk,
+      (unsigned long)guard_bandwidth_including_exits_kb*1000,
+      (unsigned long)guard_bandwidth_excluding_exits_kb*1000,
+      enough_mtbf_info ? 1 : 0,
+      enough_measured_bw ? 1 : 0);
+
+  return result;
+}
+
+/* DOCDOC running_long_enough_to_decide_unreachable */
+int
+running_long_enough_to_decide_unreachable(void)
+{
+  return time_of_process_start
+    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
+}
+
+/** Each server needs to have passed a reachability test no more
+ * than this number of seconds ago, or it is listed as down in
+ * the directory. */
+#define REACHABLE_TIMEOUT (45*60)
+
+/** If we tested a router and found it reachable _at least this long_ after it
+ * declared itself hibernating, it is probably done hibernating and we just
+ * missed a descriptor from it. */
+#define HIBERNATION_PUBLICATION_SKEW (60*60)
+
+/** Treat a router as alive if
+ *    - It's me, and I'm not hibernating.
+ * or - We've found it reachable recently. */
+void
+dirserv_set_router_is_running(routerinfo_t *router, time_t now)
+{
+  /*XXXX This function is a mess.  Separate out the part that calculates
+    whether it's reachable and the part that tells rephist that the router was
+    unreachable.
+   */
+  int answer;
+  const or_options_t *options = get_options();
+  node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
+  tor_assert(node);
+
+  if (router_is_me(router)) {
+    /* We always know if we are shutting down or hibernating ourselves. */
+    answer = ! we_are_hibernating();
+  } else if (router->is_hibernating &&
+             (router->cache_info.published_on +
+              HIBERNATION_PUBLICATION_SKEW) > node->last_reachable) {
+    /* A hibernating router is down unless we (somehow) had contact with it
+     * since it declared itself to be hibernating. */
+    answer = 0;
+  } else if (options->AssumeReachable) {
+    /* If AssumeReachable, everybody is up unless they say they are down! */
+    answer = 1;
+  } else {
+    /* Otherwise, a router counts as up if we found all announced OR
+       ports reachable in the last REACHABLE_TIMEOUT seconds.
+
+       XXX prop186 For now there's always one IPv4 and at most one
+       IPv6 OR port.
+
+       If we're not on IPv6, don't consider reachability of potential
+       IPv6 OR port since that'd kill all dual stack relays until a
+       majority of the dir auths have IPv6 connectivity. */
+    answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
+              (options->AuthDirHasIPv6Connectivity != 1 ||
+               tor_addr_is_null(&router->ipv6_addr) ||
+               now < node->last_reachable6 + REACHABLE_TIMEOUT));
+  }
+
+  if (!answer && running_long_enough_to_decide_unreachable()) {
+    /* Not considered reachable. tell rephist about that.
+
+       Because we launch a reachability test for each router every
+       REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably
+       been down since at least that time after we last successfully reached
+       it.
+
+       XXX ipv6
+     */
+    time_t when = now;
+    if (node->last_reachable &&
+        node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now)
+      when = node->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD;
+    rep_hist_note_router_unreachable(router->cache_info.identity_digest, when);
+  }
+
+  node->is_running = answer;
+}
+
+/** Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that ri-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 node_t *node,
+                                 routerinfo_t *ri,
+                                 time_t now,
+                                 int listbadexits)
+{
+  const or_options_t *options = get_options();
+  uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
+
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Already set by compute_performance_thresholds. */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable =
+    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  rs->is_fast = node->is_fast =
+    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  rs->is_flagged_running = node->is_running; /* computed above */
+
+  rs->is_valid = node->is_valid;
+
+  if (node->is_fast && node->is_stable &&
+      ri->supports_tunnelled_dir_requests &&
+      ((options->AuthDirGuardBWGuarantee &&
+        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
+       routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
+                          guard_bandwidth_excluding_exits_kb))) {
+    long tk = rep_hist_get_weighted_time_known(
+                                      node->identity, now);
+    double wfu = rep_hist_get_weighted_fractional_uptime(
+                                      node->identity, now);
+    rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0;
+  } else {
+    rs->is_possible_guard = 0;
+  }
+
+  rs->is_bad_exit = listbadexits && node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir =
+    dirserv_thinks_router_is_hs_dir(ri, node, now);
+
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+  if (options->AuthDirHasIPv6Connectivity == 1 &&
+      !tor_addr_is_null(&ri->ipv6_addr) &&
+      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
+    /* We're configured as having IPv6 connectivity. There's an IPv6
+       OR port and it's reachable so copy it to the routerstatus.  */
+    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+    rs->ipv6_orport = ri->ipv6_orport;
+  } else {
+    tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
+    rs->ipv6_orport = 0;
+  }
+
+  if (options->TestingTorNetwork) {
+    dirserv_set_routerstatus_testing(rs);
+  }
+}
+
+/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and
+ * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags,
+ * respectively. But don't set the corresponding node flags.
+ * Should only be called if TestingTorNetwork is set. */
+STATIC void
+dirserv_set_routerstatus_testing(routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(options->TestingTorNetwork);
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
+                                      rs, 0)) {
+    rs->is_exit = 1;
+  } else if (options->TestingDirAuthVoteExitIsStrict) {
+    rs->is_exit = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard,
+                                      rs, 0)) {
+    rs->is_possible_guard = 1;
+  } else if (options->TestingDirAuthVoteGuardIsStrict) {
+    rs->is_possible_guard = 0;
+  }
+
+  if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir,
+                                      rs, 0)) {
+    rs->is_hs_dir = 1;
+  } else if (options->TestingDirAuthVoteHSDirIsStrict) {
+    rs->is_hs_dir = 0;
+  }
+}

src/feature/dirauth/voteflags.h

@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voteflags.h
+ * \brief Header file for voteflags.c
+ **/
+
+#ifndef TOR_VOTEFLAGS_H
+#define TOR_VOTEFLAGS_H
+
+void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
+char *dirserv_get_flag_thresholds_line(void);
+void dirserv_compute_bridge_flag_thresholds(void);
+int running_long_enough_to_decide_unreachable(void);
+
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      node_t *node,
+                                      routerinfo_t *ri, time_t now,
+                                      int listbadexits);
+
+void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+
+#ifdef VOTEFLAGS_PRIVATE
+STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
+#endif
+
+#endif

src/feature/dircache/directory.c

@@ -22,6 +22,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircache/directory.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/process_descs.h"
 #include "feature/client/entrynodes.h"
 #include "feature/dircommon/fp_pair.h"
 #include "feature/stats/geoip.h"
@@ -40,7 +41,11 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
+
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"
 #include "lib/encoding/confline.h"

src/feature/dircache/dirserv.h

@@ -16,46 +16,6 @@ struct ed25519_public_key_t;
 
 #include "lib/testsupport/testsupport.h"
 
-/** An enum to describe what format we're generating a routerstatus line in.
- */
-typedef enum {
-  /** For use in a v2 opinion */
-  NS_V2,
-  /** For use in a consensus networkstatus document (ns flavor) */
-  NS_V3_CONSENSUS,
-  /** For use in a vote networkstatus document */
-  NS_V3_VOTE,
-  /** For passing to the controlport in response to a GETINFO request */
-  NS_CONTROL_PORT,
-  /** For use in a consensus networkstatus document (microdesc flavor) */
-  NS_V3_CONSENSUS_MICRODESC
-} routerstatus_format_type_t;
-
-/** What fraction (1 over this number) of the relay ID space do we
- * (as a directory authority) launch connections to at each reachability
- * test? */
-#define REACHABILITY_MODULO_PER_TEST 128
-
-/** How often (in seconds) do we launch reachability tests? */
-#define REACHABILITY_TEST_INTERVAL 10
-
-/** How many seconds apart are the reachability tests for a given relay? */
-#define REACHABILITY_TEST_CYCLE_PERIOD \
-  (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
-
-/** Maximum length of an exit policy summary. */
-#define MAX_EXITPOLICY_SUMMARY_LEN 1000
-
-/** Maximum allowable length of a version line in a networkstatus. */
-#define MAX_V_LINE_LEN 128
-
-/** Maximum allowable length of bandwidth headers in a bandwidth file */
-#define MAX_BW_FILE_HEADER_COUNT_IN_VOTE 50
-
-/** Terminatore that separates bandwidth file headers from bandwidth file
- * relay lines */
-#define BW_FILE_HEADERS_TERMINATOR "=====\n"
-
 /** Ways to convert a spoolable_resource_t to a bunch of bytes. */
 typedef enum dir_spool_source_t {
     DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
@@ -111,32 +71,8 @@ typedef struct spooled_resource_t {
   off_t cached_dir_offset;
 } spooled_resource_t;
 
-#ifdef DIRSERV_PRIVATE
-typedef struct measured_bw_line_t {
-  char node_id[DIGEST_LEN];
-  char node_hex[MAX_HEX_NICKNAME_LEN+1];
-  long int bw_kb;
-} measured_bw_line_t;
-#endif /* defined(DIRSERV_PRIVATE) */
-
 int connection_dirserv_flushed_some(dir_connection_t *conn);
 
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
-int dirserv_load_fingerprint_file(void);
-void dirserv_free_fingerprint_list(void);
-enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
-                                     const char *source,
-                                     const char **msg);
-enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
-                                               const char **msg,
-                                               const char *source);
-void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
-int list_server_status_v1(smartlist_t *routers, char **router_status_out,
-                          int for_controller);
-char *dirserv_get_flag_thresholds_line(void);
-void dirserv_compute_bridge_flag_thresholds(void);
-
 int directory_fetches_from_authorities(const or_options_t *options);
 int directory_fetches_dir_info_early(const or_options_t *options);
 int directory_fetches_dir_info_later(const or_options_t *options);
@@ -159,76 +95,10 @@ int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
                                  const char **msg_out);
 int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
                             const char **msg);
-void dirserv_orconn_tls_done(const tor_addr_t *addr,
-                             uint16_t or_port,
-                             const char *digest_rcvd,
-                             const struct ed25519_public_key_t *ed_id_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
-int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
-                                   int complain,
-                                   int *valid_out);
-uint32_t dirserv_router_get_status(const routerinfo_t *router,
-                                   const char **msg,
-                                   int severity);
-void dirserv_set_node_flags_from_authoritative_status(node_t *node,
-                                                      uint32_t authstatus);
 
-int dirserv_would_reject_router(const routerstatus_t *rs);
-char *routerstatus_format_entry(
-                              const routerstatus_t *rs,
-                              const char *version,
-                              const char *protocols,
-                              routerstatus_format_type_t format,
-                              int consensus_method,
-                              const vote_routerstatus_t *vrs);
 void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
 cached_dir_t *new_cached_dir(char *s, time_t published);
-struct config_line_t;
-char *format_recommended_version_list(const struct config_line_t *line,
-                                      int warn);
-int validate_recommended_package_line(const char *line);
-int dirserv_query_measured_bw_cache_kb(const char *node_id,
-                                       long *bw_out,
-                                       time_t *as_of_out);
-void dirserv_clear_measured_bw_cache(void);
-int dirserv_has_measured_bw(const char *node_id);
-int dirserv_get_measured_bw_cache_size(void);
-void dirserv_count_measured_bws(const smartlist_t *routers);
-int running_long_enough_to_decide_unreachable(void);
-void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
-
-#ifdef DIRSERV_PRIVATE
-
-STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-
-/* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
-#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
-
-STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line,
-                                  int line_is_after_headers);
-
-STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
-                           smartlist_t *routerstatuses);
-
-STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
-                               time_t as_of);
-STATIC void dirserv_expire_measured_bw_cache(time_t now);
-
-STATIC int
-dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
-                                      smartlist_t *vote_routerstatuses);
-#endif /* defined(DIRSERV_PRIVATE) */
-
-int dirserv_read_measured_bandwidths(const char *from_file,
-                                     smartlist_t *routerstatuses,
-                                     smartlist_t *bw_file_headers);
-
-int dirserv_read_guardfraction_file(const char *fname,
-                                 smartlist_t *vote_routerstatuses);
 
 spooled_resource_t *spooled_resource_new(dir_spool_source_t source,
                                          const uint8_t *digest,

src/feature/hs/hs_service.c

@@ -27,7 +27,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "app/config/statefile.h"
 

src/feature/nodelist/authcert.c

@@ -0,0 +1,1205 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.c
+ * \brief Code to maintain directory authorities' certificates.
+ *
+ * Authority certificates are signed with authority identity keys; they
+ * are used to authenticate shorter-term authority signing keys. We
+ * fetch them when we find a consensus or a vote that has been signed
+ * with a signing key we don't recognize.  We cache them on disk and
+ * load them on startup.  Authority operators generate them with the
+ * "tor-gencert" utility.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/main.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/dircache/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerparse.h"
+#include "feature/relay/router.h"
+
+#include "core/or/connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/document_signature_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/networkstatus_voter_info_st.h"
+#include "feature/nodelist/node_st.h"
+
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+#define DSMAP_FOREACH(map, keyvar, valvar) \
+  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
+                    valvar)
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+
+/* Forward declaration for cert_list_t */
+typedef struct cert_list_t cert_list_t;
+
+static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
+                                              const char *digest);
+static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                                const char *digest,
+                                                time_t now);
+static void list_pending_fpsk_downloads(fp_pair_map_t *result);
+
+/** List of certificates for a single authority, and download status for
+ * latest certificate.
+ */
+struct cert_list_t {
+  /*
+   * The keys of download status map are cert->signing_key_digest for pending
+   * downloads by (identity digest/signing key digest) pair; functions such
+   * as authority_cert_get_by_digest() already assume these are unique.
+   */
+  struct digest_ds_map_t *dl_status_map;
+  /* There is also a dlstatus for the download by identity key only */
+  download_status_t dl_status_by_id;
+  smartlist_t *certs;
+};
+/** Map from v3 identity key digest to cert_list_t. */
+static digestmap_t *trusted_dir_certs = NULL;
+
+/** True iff any key certificate in at least one member of
+ * <b>trusted_dir_certs</b> has changed since we last flushed the
+ * certificates to disk. */
+static int trusted_dir_servers_certs_changed = 0;
+
+/** Initialise schedule, want_authority, and increment_on in the download
+ * status dlstatus, then call download_status_reset() on it.
+ * It is safe to call this function or download_status_reset() multiple times
+ * on a new dlstatus. But it should *not* be called after a dlstatus has been
+ * used to count download attempts or failures. */
+static void
+download_status_cert_init(download_status_t *dlstatus)
+{
+  dlstatus->schedule = DL_SCHED_CONSENSUS;
+  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
+  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
+  dlstatus->last_backoff_position = 0;
+  dlstatus->last_delay_used = 0;
+
+  /* Use the new schedule to set next_attempt_at */
+  download_status_reset(dlstatus);
+}
+
+/** Reset the download status of a specified element in a dsmap */
+static void
+download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
+{
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (!dlstatus) {
+    /* Insert before we reset */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    download_status_cert_init(dlstatus);
+  }
+  tor_assert(dlstatus);
+  /* Go ahead and reset it */
+  download_status_reset(dlstatus);
+}
+
+/**
+ * Return true if the download for this signing key digest in cl is ready
+ * to be re-attempted.
+ */
+static int
+download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
+                                     const char *digest,
+                                     time_t now)
+{
+  int rv = 0;
+  download_status_t *dlstatus = NULL;
+
+  tor_assert(cl);
+  tor_assert(digest);
+
+  /* Make sure we have a dsmap */
+  if (!(cl->dl_status_map)) {
+    cl->dl_status_map = dsmap_new();
+  }
+  /* Look for a download_status_t in the map with this digest */
+  dlstatus = dsmap_get(cl->dl_status_map, digest);
+  /* Got one? */
+  if (dlstatus) {
+    /* Use download_status_is_ready() */
+    rv = download_status_is_ready(dlstatus, now);
+  } else {
+    /*
+     * If we don't know anything about it, return 1, since we haven't
+     * tried this one before.  We need to create a new entry here,
+     * too.
+     */
+    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
+    download_status_cert_init(dlstatus);
+    dsmap_set(cl->dl_status_map, digest, dlstatus);
+    rv = 1;
+  }
+
+  return rv;
+}
+
+/** Helper: Return the cert_list_t for an authority whose authority ID is
+ * <b>id_digest</b>, allocating a new list if necessary. */
+static cert_list_t *
+get_cert_list(const char *id_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs)
+    trusted_dir_certs = digestmap_new();
+  cl = digestmap_get(trusted_dir_certs, id_digest);
+  if (!cl) {
+    cl = tor_malloc_zero(sizeof(cert_list_t));
+    download_status_cert_init(&cl->dl_status_by_id);
+    cl->certs = smartlist_new();
+    cl->dl_status_map = dsmap_new();
+    digestmap_set(trusted_dir_certs, id_digest, cl);
+  }
+  return cl;
+}
+
+/** Return a list of authority ID digests with potentially enumerable lists
+ * of download_status_t objects; used by controller GETINFO queries.
+ */
+
+MOCK_IMPL(smartlist_t *,
+list_authority_ids_with_downloads, (void))
+{
+  smartlist_t *ids = smartlist_new();
+  digestmap_iter_t *i;
+  const char *digest;
+  char *tmp;
+  void *cl;
+
+  if (trusted_dir_certs) {
+    for (i = digestmap_iter_init(trusted_dir_certs);
+         !(digestmap_iter_done(i));
+         i = digestmap_iter_next(trusted_dir_certs, i)) {
+      /*
+       * We always have at least dl_status_by_id to query, so no need to
+       * probe deeper than the existence of a cert_list_t.
+       */
+      digestmap_iter_get(i, &digest, &cl);
+      tmp = tor_malloc(DIGEST_LEN);
+      memcpy(tmp, digest, DIGEST_LEN);
+      smartlist_add(ids, tmp);
+    }
+  }
+  /* else definitely no downloads going since nothing even has a cert list */
+
+  return ids;
+}
+
+/** Given an authority ID digest, return a pointer to the default download
+ * status, or NULL if there is no such entry in trusted_dir_certs */
+
+MOCK_IMPL(download_status_t *,
+id_only_download_status_for_authority_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      dl = &(cl->dl_status_by_id);
+    }
+  }
+
+  return dl;
+}
+
+/** Given an authority ID digest, return a smartlist of signing key digests
+ * for which download_status_t is potentially queryable, or NULL if no such
+ * authority ID digest is known. */
+
+MOCK_IMPL(smartlist_t *,
+list_sk_digests_for_authority_id, (const char *digest))
+{
+  smartlist_t *sks = NULL;
+  cert_list_t *cl;
+  dsmap_iter_t *i;
+  const char *sk_digest;
+  char *tmp;
+  download_status_t *dl;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, digest);
+    if (cl) {
+      sks = smartlist_new();
+      if (cl->dl_status_map) {
+        for (i = dsmap_iter_init(cl->dl_status_map);
+             !(dsmap_iter_done(i));
+             i = dsmap_iter_next(cl->dl_status_map, i)) {
+          /* Pull the digest out and add it to the list */
+          dsmap_iter_get(i, &sk_digest, &dl);
+          tmp = tor_malloc(DIGEST_LEN);
+          memcpy(tmp, sk_digest, DIGEST_LEN);
+          smartlist_add(sks, tmp);
+        }
+      }
+    }
+  }
+
+  return sks;
+}
+
+/** Given an authority ID digest and a signing key digest, return the
+ * download_status_t or NULL if none exists. */
+
+MOCK_IMPL(download_status_t *,
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
+{
+  download_status_t *dl = NULL;
+  cert_list_t *cl = NULL;
+
+  if (trusted_dir_certs) {
+    cl = digestmap_get(trusted_dir_certs, id_digest);
+    if (cl && cl->dl_status_map) {
+      dl = dsmap_get(cl->dl_status_map, sk_digest);
+    }
+  }
+
+  return dl;
+}
+
+#define cert_list_free(val) \
+  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
+/** Release all space held by a cert_list_t */
+static void
+cert_list_free_(cert_list_t *cl)
+{
+  if (!cl)
+    return;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+                    authority_cert_free(cert));
+  smartlist_free(cl->certs);
+  dsmap_free(cl->dl_status_map, tor_free_);
+  tor_free(cl);
+}
+
+/** Wrapper for cert_list_free so we can pass it to digestmap_free */
+static void
+cert_list_free_void(void *cl)
+{
+  cert_list_free_(cl);
+}
+
+/** Reload the cached v3 key certificates from the cached-certs file in
+ * the data directory. Return 0 on success, -1 on failure. */
+int
+trusted_dirs_reload_certs(void)
+{
+  char *filename;
+  char *contents;
+  int r;
+
+  filename = get_cachedir_fname("cached-certs");
+  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  if (!contents)
+    return 0;
+  r = trusted_dirs_load_certs_from_string(
+        contents,
+        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
+  tor_free(contents);
+  return r;
+}
+
+/** Helper: return true iff we already have loaded the exact cert
+ * <b>cert</b>. */
+static inline int
+already_have_cert(authority_cert_t *cert)
+{
+  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+  {
+    if (tor_memeq(c->cache_info.signed_descriptor_digest,
+                cert->cache_info.signed_descriptor_digest,
+                DIGEST_LEN))
+      return 1;
+  });
+  return 0;
+}
+
+/** Load a bunch of new key certificates from the string <b>contents</b>.  If
+ * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
+ * from the cache, and we don't need to flush them to disk.  If we are a
+ * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
+ * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
+ * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
+ * need to flush any changed certificates to disk now.  Return 0 on success,
+ * -1 if any certs fail to parse.
+ *
+ * If source_dir is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved certificates from, so try it first to
+ * fetch any missing certificates.
+ */
+int
+trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                    int flush, const char *source_dir)
+{
+  dir_server_t *ds;
+  const char *s, *eos;
+  int failure_code = 0;
+  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
+  int added_trusted_cert = 0;
+
+  for (s = contents; *s; s = eos) {
+    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    cert_list_t *cl;
+    if (!cert) {
+      failure_code = -1;
+      break;
+    }
+    ds = trusteddirserver_get_by_v3_auth_digest(
+                                       cert->cache_info.identity_digest);
+    log_debug(LD_DIR, "Parsed certificate for %s",
+              ds ? ds->nickname : "unknown authority");
+
+    if (already_have_cert(cert)) {
+      /* we already have this one. continue. */
+      log_info(LD_DIR, "Skipping %s certificate for %s that we "
+               "already have.",
+               from_store ? "cached" : "downloaded",
+               ds ? ds->nickname : "an old or new authority");
+
+      /*
+       * A duplicate on download should be treated as a failure, so we call
+       * authority_cert_dl_failed() to reset the download status to make sure
+       * we can't try again.  Since we've implemented the fp-sk mechanism
+       * to download certs by signing key, this should be much rarer than it
+       * was and is perhaps cause for concern.
+       */
+      if (!from_store) {
+        if (authdir_mode(get_options())) {
+          log_warn(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        } else {
+          log_info(LD_DIR,
+                   "Got a certificate for %s, but we already have it. "
+                   "Maybe they haven't updated it. Waiting for a while.",
+                   ds ? ds->nickname : "an old or new authority");
+        }
+
+        /*
+         * This is where we care about the source; authority_cert_dl_failed()
+         * needs to know whether the download was by fp or (fp,sk) pair to
+         * twiddle the right bit in the download map.
+         */
+        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   NULL, 404);
+        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
+          authority_cert_dl_failed(cert->cache_info.identity_digest,
+                                   cert->signing_key_digest, 404);
+        }
+      }
+
+      authority_cert_free(cert);
+      continue;
+    }
+
+    if (ds) {
+      added_trusted_cert = 1;
+      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
+               "signing key %s", from_store ? "cached" : "downloaded",
+               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
+    } else {
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
+      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
+               "authority with signing key %s",
+               adding ? "Adding" : "Not adding",
+               from_store ? "cached" : "downloaded",
+               hex_str(cert->signing_key_digest,DIGEST_LEN));
+      if (!adding) {
+        authority_cert_free(cert);
+        continue;
+      }
+    }
+
+    cl = get_cert_list(cert->cache_info.identity_digest);
+    smartlist_add(cl->certs, cert);
+    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
+      /* Check to see whether we should update our view of the authority's
+       * address. */
+      if (cert->addr && cert->dir_port &&
+          (ds->addr != cert->addr ||
+           ds->dir_port != cert->dir_port)) {
+        char *a = tor_dup_ip(cert->addr);
+        log_notice(LD_DIR, "Updating address for directory authority %s "
+                   "from %s:%d to %s:%d based on certificate.",
+                   ds->nickname, ds->address, (int)ds->dir_port,
+                   a, cert->dir_port);
+        tor_free(a);
+        ds->addr = cert->addr;
+        ds->dir_port = cert->dir_port;
+      }
+      ds->addr_current_at = cert->cache_info.published_on;
+    }
+
+    if (!from_store)
+      trusted_dir_servers_certs_changed = 1;
+  }
+
+  if (flush)
+    trusted_dirs_flush_certs_to_disk();
+
+  /* call this even if failure_code is <0, since some certs might have
+   * succeeded, but only pass source_dir if there were no failures,
+   * and at least one more authority certificate was added to the store.
+   * This avoids retrying a directory that's serving bad or entirely duplicate
+   * certificates. */
+  if (failure_code == 0 && added_trusted_cert) {
+    networkstatus_note_certs_arrived(source_dir);
+  } else {
+    networkstatus_note_certs_arrived(NULL);
+  }
+
+  return failure_code;
+}
+
+/** Save all v3 key certificates to the cached-certs file. */
+void
+trusted_dirs_flush_certs_to_disk(void)
+{
+  char *filename;
+  smartlist_t *chunks;
+
+  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
+    return;
+
+  chunks = smartlist_new();
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+          {
+            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
+            c->bytes = cert->cache_info.signed_descriptor_body;
+            c->len = cert->cache_info.signed_descriptor_len;
+            smartlist_add(chunks, c);
+          });
+  } DIGESTMAP_FOREACH_END;
+
+  filename = get_cachedir_fname("cached-certs");
+  if (write_chunks_to_file(filename, chunks, 0, 0)) {
+    log_warn(LD_FS, "Error writing certificates to disk.");
+  }
+  tor_free(filename);
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
+  smartlist_free(chunks);
+
+  trusted_dir_servers_certs_changed = 0;
+}
+
+static int
+compare_certs_by_pubdates(const void **_a, const void **_b)
+{
+  const authority_cert_t *cert1 = *_a, *cert2=*_b;
+
+  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
+    return -1;
+  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
+    return 1;
+  else
+    return 0;
+}
+
+/** Remove all expired v3 authority certificates that have been superseded for
+ * more than 48 hours or, if not expired, that were published more than 7 days
+ * before being superseded. (If the most recent cert was published more than 48
+ * hours ago, then we aren't going to get any consensuses signed with older
+ * keys.) */
+void
+trusted_dirs_remove_old_certs(void)
+{
+  time_t now = time(NULL);
+#define DEAD_CERT_LIFETIME (2*24*60*60)
+#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    /* Sort the list from first-published to last-published */
+    smartlist_sort(cl->certs, compare_certs_by_pubdates);
+
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
+        /* This is the most recently published cert.  Keep it. */
+        continue;
+      }
+      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
+      const time_t next_cert_published = next_cert->cache_info.published_on;
+      if (next_cert_published > now) {
+        /* All later certs are published in the future. Keep everything
+         * we didn't discard. */
+        break;
+      }
+      int should_remove = 0;
+      if (cert->expires + DEAD_CERT_LIFETIME < now) {
+        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
+         * Remove it. */
+        should_remove = 1;
+      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
+        /* Certificate has been superseded for OLD_CERT_LIFETIME.
+         * Remove it.
+         */
+        should_remove = 1;
+      }
+      if (should_remove) {
+        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
+        authority_cert_free(cert);
+        trusted_dir_servers_certs_changed = 1;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+
+  } DIGESTMAP_FOREACH_END;
+#undef DEAD_CERT_LIFETIME
+#undef OLD_CERT_LIFETIME
+
+  trusted_dirs_flush_certs_to_disk();
+}
+
+/** Return the newest v3 authority certificate whose v3 authority identity key
+ * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
+ * or it has no certificate. */
+authority_cert_t *
+authority_cert_get_newest_by_id(const char *id_digest)
+{
+  cert_list_t *cl;
+  authority_cert_t *best = NULL;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+  {
+    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
+      best = cert;
+  });
+  return best;
+}
+
+/** Return the newest v3 authority certificate whose directory signing key has
+ * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
+ */
+authority_cert_t *
+authority_cert_get_by_sk_digest(const char *sk_digest)
+{
+  authority_cert_t *c;
+  if (!trusted_dir_certs)
+    return NULL;
+
+  if ((c = get_my_v3_authority_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+  if ((c = get_my_v3_legacy_cert()) &&
+      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
+    return c;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    {
+      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+        return cert;
+    });
+  } DIGESTMAP_FOREACH_END;
+  return NULL;
+}
+
+/** Return the v3 authority certificate with signing key matching
+ * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
+ * Return NULL if no such authority is known. */
+authority_cert_t *
+authority_cert_get_by_digests(const char *id_digest,
+                              const char *sk_digest)
+{
+  cert_list_t *cl;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return NULL;
+  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
+    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
+      return cert; );
+
+  return NULL;
+}
+
+/** Add every known authority_cert_t to <b>certs_out</b>. */
+void
+authority_cert_get_all(smartlist_t *certs_out)
+{
+  tor_assert(certs_out);
+  if (!trusted_dir_certs)
+    return;
+
+  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
+    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
+                      smartlist_add(certs_out, c));
+  } DIGESTMAP_FOREACH_END;
+}
+
+/** Called when an attempt to download a certificate with the authority with
+ * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
+ * fails with HTTP response code <b>status</b>: remember the failure, so we
+ * don't try again immediately. */
+void
+authority_cert_dl_failed(const char *id_digest,
+                         const char *signing_key_digest, int status)
+{
+  cert_list_t *cl;
+  download_status_t *dlstatus = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return;
+
+  /*
+   * Are we noting a failed download of the latest cert for the id digest,
+   * or of a download by (id, signing key) digest pair?
+   */
+  if (!signing_key_digest) {
+    /* Just by id digest */
+    download_status_failed(&cl->dl_status_by_id, status);
+  } else {
+    /* Reset by (id, signing key) digest pair
+     *
+     * Look for a download_status_t in the map with this digest
+     */
+    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
+    /* Got one? */
+    if (dlstatus) {
+      download_status_failed(dlstatus, status);
+    } else {
+      /*
+       * Do this rather than hex_str(), since hex_str clobbers
+       * old results and we call twice in the param list.
+       */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    id_digest, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    signing_key_digest, DIGEST_LEN);
+      log_warn(LD_BUG,
+               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
+               "status %d, but knew nothing about the download.",
+               id_digest_str, sk_digest_str, status);
+    }
+  }
+}
+
+static const char *BAD_SIGNING_KEYS[] = {
+  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
+  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
+  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
+  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
+  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
+  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
+  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
+  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
+  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
+  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
+  NULL,
+};
+
+/** Return true iff <b>cert</b> authenticates some atuhority signing key
+ * which, because of the old openssl heartbleed vulnerability, should
+ * never be trusted. */
+int
+authority_cert_is_blacklisted(const authority_cert_t *cert)
+{
+  char hex_digest[HEX_DIGEST_LEN+1];
+  int i;
+  base16_encode(hex_digest, sizeof(hex_digest),
+                cert->signing_key_digest, sizeof(cert->signing_key_digest));
+
+  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
+    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/** Return true iff when we've been getting enough failures when trying to
+ * download the certificate with ID digest <b>id_digest</b> that we're willing
+ * to start bugging the user about it. */
+int
+authority_cert_dl_looks_uncertain(const char *id_digest)
+{
+#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
+  cert_list_t *cl;
+  int n_failures;
+  if (!trusted_dir_certs ||
+      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
+    return 0;
+
+  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
+  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
+}
+
+/* Fetch the authority certificates specified in resource.
+ * If we are a bridge client, and node is a configured bridge, fetch from node
+ * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
+ * rs. Otherwise, fetch from a random directory mirror. */
+static void
+authority_certs_fetch_resource_impl(const char *resource,
+                                    const char *dir_hint,
+                                    const node_t *node,
+                                    const routerstatus_t *rs)
+{
+  const or_options_t *options = get_options();
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
+
+  /* Make sure bridge clients never connect to anything but a bridge */
+  if (options->UseBridges) {
+    if (node && !node_is_a_configured_bridge(node)) {
+      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
+      get_via_tor = 1;
+    } else if (!node) {
+      /* If we're using bridges, and there's no node, use a 3-hop path. */
+      get_via_tor = 1;
+    }
+  }
+
+  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
+                                                    : DIRIND_ONEHOP;
+
+  directory_request_t *req = NULL;
+  /* If we've just downloaded a consensus from a bridge, re-use that
+   * bridge */
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
+    /* clients always make OR connections to bridges */
+    tor_addr_port_t or_ap;
+    /* we are willing to use a non-preferred address if we need to */
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
+                                         &or_ap);
+
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
+     * directory */
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
+    return;
+  }
+
+  /* Otherwise, we want certs from a random fallback or directory
+   * mirror, because they will almost always succeed. */
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                               resource, PDS_RETRY_IF_NO_SERVERS,
+                               DL_WANT_ANY_DIRSERVER);
+}
+
+/** Try to download any v3 authority certificates that we may be missing.  If
+ * <b>status</b> is provided, try to get all the ones that were used to sign
+ * <b>status</b>.  Additionally, try to have a non-expired certificate for
+ * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
+ * already have.
+ *
+ * If dir_hint is non-NULL, it's the identity digest for a directory that
+ * we've just successfully retrieved a consensus or certificates from, so try
+ * it first to fetch any missing certificates.
+ **/
+void
+authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                              const char *dir_hint)
+{
+  /*
+   * The pending_id digestmap tracks pending certificate downloads by
+   * identity digest; the pending_cert digestmap tracks pending downloads
+   * by (identity digest, signing key digest) pairs.
+   */
+  digestmap_t *pending_id;
+  fp_pair_map_t *pending_cert;
+  /*
+   * The missing_id_digests smartlist will hold a list of id digests
+   * we want to fetch the newest cert for; the missing_cert_digests
+   * smartlist will hold a list of fp_pair_t with an identity and
+   * signing key digest.
+   */
+  smartlist_t *missing_cert_digests, *missing_id_digests;
+  char *resource = NULL;
+  cert_list_t *cl;
+  const or_options_t *options = get_options();
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
+  fp_pair_t *fp_tmp = NULL;
+  char id_digest_str[2*DIGEST_LEN+1];
+  char sk_digest_str[2*DIGEST_LEN+1];
+
+  if (should_delay_dir_fetches(options, NULL))
+    return;
+
+  pending_cert = fp_pair_map_new();
+  pending_id = digestmap_new();
+  missing_cert_digests = smartlist_new();
+  missing_id_digests = smartlist_new();
+
+  /*
+   * First, we get the lists of already pending downloads so we don't
+   * duplicate effort.
+   */
+  list_pending_downloads(pending_id, NULL,
+                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
+  list_pending_fpsk_downloads(pending_cert);
+
+  /*
+   * Now, we download any trusted authority certs we don't have by
+   * identity digest only.  This gets the latest cert for that authority.
+   */
+  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
+                          dir_server_t *, ds) {
+    int found = 0;
+    if (!(ds->type & V3_DIRINFO))
+      continue;
+    if (smartlist_contains_digest(missing_id_digests,
+                                  ds->v3_identity_digest))
+      continue;
+    cl = get_cert_list(ds->v3_identity_digest);
+    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
+      if (now < cert->expires) {
+        /* It's not expired, and we weren't looking for something to
+         * verify a consensus with.  Call it done. */
+        download_status_reset(&(cl->dl_status_by_id));
+        /* No sense trying to download it specifically by signing key hash */
+        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(cert);
+    if (!found &&
+        download_status_is_ready(&(cl->dl_status_by_id), now) &&
+        !digestmap_get(pending_id, ds->v3_identity_digest)) {
+      log_info(LD_DIR,
+               "No current certificate known for authority %s "
+               "(ID digest %s); launching request.",
+               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
+      smartlist_add(missing_id_digests, ds->v3_identity_digest);
+    }
+  } SMARTLIST_FOREACH_END(ds);
+
+  /*
+   * Next, if we have a consensus, scan through it and look for anything
+   * signed with a key from a cert we don't have.  Those get downloaded
+   * by (fp,sk) pair, but if we don't know any certs at all for the fp
+   * (identity digest), and it's one of the trusted dir server certs
+   * we started off above or a pending download in pending_id, don't
+   * try to get it yet.  Most likely, the one we'll get for that will
+   * have the right signing key too, and we'd just be downloading
+   * redundantly.
+   */
+  if (status) {
+    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
+                            voter) {
+      if (!smartlist_len(voter->sigs))
+        continue; /* This authority never signed this consensus, so don't
+                   * go looking for a cert with key digest 0000000000. */
+      if (!keep_unknown &&
+          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
+
+      /*
+       * If we don't know *any* cert for this authority, and a download by ID
+       * is pending or we added it to missing_id_digests above, skip this
+       * one for now to avoid duplicate downloads.
+       */
+      cl = get_cert_list(voter->identity_digest);
+      if (smartlist_len(cl->certs) == 0) {
+        /* We have no certs at all for this one */
+
+        /* Do we have a download of one pending? */
+        if (digestmap_get(pending_id, voter->identity_digest))
+          continue;
+
+        /*
+         * Are we about to launch a download of one due to the trusted
+         * dir server check above?
+         */
+        if (smartlist_contains_digest(missing_id_digests,
+                                      voter->identity_digest))
+          continue;
+      }
+
+      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
+        authority_cert_t *cert =
+          authority_cert_get_by_digests(voter->identity_digest,
+                                        sig->signing_key_digest);
+        if (cert) {
+          if (now < cert->expires)
+            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
+          continue;
+        }
+        if (download_status_is_ready_by_sk_in_cl(
+              cl, sig->signing_key_digest, now) &&
+            !fp_pair_map_get_by_digests(pending_cert,
+                                        voter->identity_digest,
+                                        sig->signing_key_digest)) {
+          /*
+           * Do this rather than hex_str(), since hex_str clobbers
+           * old results and we call twice in the param list.
+           */
+          base16_encode(id_digest_str, sizeof(id_digest_str),
+                        voter->identity_digest, DIGEST_LEN);
+          base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                        sig->signing_key_digest, DIGEST_LEN);
+
+          if (voter->nickname) {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority %s "
+                     "(ID digest %s) with signing key %s: "
+                     "launching request.",
+                     voter->nickname, id_digest_str, sk_digest_str);
+          } else {
+            log_info(LD_DIR,
+                     "We're missing a certificate from authority ID digest "
+                     "%s with signing key %s: launching request.",
+                     id_digest_str, sk_digest_str);
+          }
+
+          /* Allocate a new fp_pair_t to append */
+          fp_tmp = tor_malloc(sizeof(*fp_tmp));
+          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
+          memcpy(fp_tmp->second, sig->signing_key_digest,
+                 sizeof(fp_tmp->second));
+          smartlist_add(missing_cert_digests, fp_tmp);
+        }
+      } SMARTLIST_FOREACH_END(sig);
+    } SMARTLIST_FOREACH_END(voter);
+  }
+
+  /* Bridge clients look up the node for the dir_hint */
+  const node_t *node = NULL;
+  /* All clients, including bridge clients, look up the routerstatus for the
+   * dir_hint */
+  const routerstatus_t *rs = NULL;
+
+  /* If we still need certificates, try the directory that just successfully
+   * served us a consensus or certificates.
+   * As soon as the directory fails to provide additional certificates, we try
+   * another, randomly selected directory. This avoids continual retries.
+   * (We only ever have one outstanding request per certificate.)
+   */
+  if (dir_hint) {
+    if (options->UseBridges) {
+      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
+       * or something else fetched over tor, we won't find the node here, but
+       * we will find the rs. */
+      node = node_get_by_id(dir_hint);
+    }
+
+    /* All clients try the consensus routerstatus, then the fallback
+     * routerstatus */
+    rs = router_get_consensus_status_by_id(dir_hint);
+    if (!rs) {
+      /* This will also find authorities */
+      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
+                                                                    dir_hint);
+      if (ds) {
+        rs = &ds->fake_status;
+      }
+    }
+
+    if (!node && !rs) {
+      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
+               "no routerstatus could be found for it.",
+               options->UseBridges ? "no node and " : "",
+               hex_str(dir_hint, DIGEST_LEN));
+    }
+  }
+
+  /* Do downloads by identity digest */
+  if (smartlist_len(missing_id_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fps = smartlist_new();
+
+    smartlist_add_strdup(fps, "fp/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
+      char *fp = NULL;
+
+      if (digestmap_get(pending_id, d))
+        continue;
+
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d, DIGEST_LEN);
+
+      if (need_plus) {
+        tor_asprintf(&fp, "+%s", id_digest_str);
+      } else {
+        /* No need for tor_asprintf() in this case; first one gets no '+' */
+        fp = tor_strdup(id_digest_str);
+        need_plus = 1;
+      }
+
+      smartlist_add(fps, fp);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fps) > 1) {
+      resource = smartlist_join_strings(fps, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else we didn't add any: they were all pending */
+
+    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
+    smartlist_free(fps);
+  }
+
+  /* Do downloads by identity digest/signing key pair */
+  if (smartlist_len(missing_cert_digests) > 0) {
+    int need_plus = 0;
+    smartlist_t *fp_pairs = smartlist_new();
+
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
+
+    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
+      char *fp_pair = NULL;
+
+      if (fp_pair_map_get(pending_cert, d))
+        continue;
+
+      /* Construct string encodings of the digests */
+      base16_encode(id_digest_str, sizeof(id_digest_str),
+                    d->first, DIGEST_LEN);
+      base16_encode(sk_digest_str, sizeof(sk_digest_str),
+                    d->second, DIGEST_LEN);
+
+      /* Now tor_asprintf() */
+      if (need_plus) {
+        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
+      } else {
+        /* First one in the list doesn't get a '+' */
+        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
+        need_plus = 1;
+      }
+
+      /* Add it to the list of pairs to request */
+      smartlist_add(fp_pairs, fp_pair);
+    } SMARTLIST_FOREACH_END(d);
+
+    if (smartlist_len(fp_pairs) > 1) {
+      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
+      /* node and rs are directories that just gave us a consensus or
+       * certificates */
+      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
+      tor_free(resource);
+    }
+    /* else they were all pending */
+
+    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
+    smartlist_free(fp_pairs);
+  }
+
+  smartlist_free(missing_id_digests);
+  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
+  smartlist_free(missing_cert_digests);
+  digestmap_free(pending_id, NULL);
+  fp_pair_map_free(pending_cert, NULL);
+}
+
+void
+authcert_free_all(void)
+{
+  if (trusted_dir_certs) {
+    digestmap_free(trusted_dir_certs, cert_list_free_void);
+    trusted_dir_certs = NULL;
+  }
+}
+
+/** Free storage held in <b>cert</b>. */
+void
+authority_cert_free_(authority_cert_t *cert)
+{
+  if (!cert)
+    return;
+
+  tor_free(cert->cache_info.signed_descriptor_body);
+  crypto_pk_free(cert->signing_key);
+  crypto_pk_free(cert->identity_key);
+
+  tor_free(cert);
+}
+
+/** For every certificate we are currently downloading by (identity digest,
+ * signing key digest) pair, set result[fp_pair] to (void *1).
+ */
+static void
+list_pending_fpsk_downloads(fp_pair_map_t *result)
+{
+  const char *pfx = "fp-sk/";
+  smartlist_t *tmp;
+  smartlist_t *conns;
+  const char *resource;
+
+  tor_assert(result);
+
+  tmp = smartlist_new();
+  conns = get_connection_array();
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_DIR &&
+        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
+        !conn->marked_for_close) {
+      resource = TO_DIR_CONN(conn)->requested_resource;
+      if (!strcmpstart(resource, pfx))
+        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
+                                                  tmp);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
+    fp_pair_map_set(result, fp, (void*)1);
+    tor_free(fp);
+  } SMARTLIST_FOREACH_END(fp);
+
+  smartlist_free(tmp);
+}

src/feature/nodelist/authcert.h

@@ -0,0 +1,60 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file authcert.h
+ * \brief Header file for authcert.c
+ **/
+
+#ifndef TOR_AUTHCERT_H
+#define TOR_AUTHCERT_H
+
+#include "lib/testsupport/testsupport.h"
+
+int trusted_dirs_reload_certs(void);
+
+/*
+ * Pass one of these as source to trusted_dirs_load_certs_from_string()
+ * to indicate whence string originates; this controls error handling
+ * behavior such as marking downloads as failed.
+ */
+
+#define TRUSTED_DIRS_CERTS_SRC_SELF 0
+#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
+#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
+#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
+
+int trusted_dirs_load_certs_from_string(const char *contents, int source,
+                                        int flush, const char *source_dir);
+void trusted_dirs_remove_old_certs(void);
+void trusted_dirs_flush_certs_to_disk(void);
+authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
+authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
+authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
+                                                const char *sk_digest);
+void authority_cert_get_all(smartlist_t *certs_out);
+void authority_cert_dl_failed(const char *id_digest,
+                              const char *signing_key_digest, int status);
+void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
+                                   const char *dir_hint);
+int authority_cert_dl_looks_uncertain(const char *id_digest);
+int authority_cert_is_blacklisted(const authority_cert_t *cert);
+
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
+
+MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
+MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
+          (const char *digest));
+MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
+          (const char *digest));
+MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
+    (const char *id_digest, const char *sk_digest));
+
+void authcert_free_all(void);
+
+#endif

src/feature/nodelist/dirlist.c

@@ -0,0 +1,421 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.c
+ * \brief Code to maintain our lists of directory authorities and
+ *    fallback directories.
+ *
+ * For the directory authorities, we have a list containing the public
+ * identity key, and contact points, for each authority.  The
+ * authorities receive descriptors from relays, and publish consensuses,
+ * descriptors, and microdescriptors.  This list is pre-configured.
+ *
+ * Fallback directories are well-known, stable, but untrusted directory
+ * caches that clients which have not yet bootstrapped can use to get
+ * their first networkstatus consensus, in order to find out where the
+ * Tor network really is.  This list is pre-configured in
+ * fallback_dirs.inc.  Every authority also serves as a fallback.
+ *
+ * Both fallback directories and directory authorities are are
+ * represented by a dir_server_t.
+ */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/or/policies.h"
+#include "feature/control/control.h"
+#include "feature/dircache/directory.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/net/resolve.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/node_st.h"
+
+/** Global list of a dir_server_t object for each directory
+ * authority. */
+static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
+
+/** Return the number of directory authorities whose type matches some bit set
+ * in <b>type</b>  */
+int
+get_n_authorities(dirinfo_type_t type)
+{
+  int n = 0;
+  if (!trusted_dir_servers)
+    return 0;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+                    if (ds->type & type)
+                      ++n);
+  return n;
+}
+
+/** Return a smartlist containing a list of dir_server_t * for all
+ * known trusted dirservers.  Callers must not modify the list or its
+ * contents.
+ */
+smartlist_t *
+router_get_trusted_dir_servers_mutable(void)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+
+  return trusted_dir_servers;
+}
+
+smartlist_t *
+router_get_fallback_dir_servers_mutable(void)
+{
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  return fallback_dir_servers;
+}
+
+const smartlist_t *
+router_get_trusted_dir_servers(void)
+{
+  return router_get_trusted_dir_servers_mutable();
+}
+
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+  return router_get_fallback_dir_servers_mutable();
+}
+
+/** Reset all internal variables used to count failed downloads of network
+ * status objects. */
+void
+router_reset_status_download_failures(void)
+{
+  mark_all_dirservers_up(fallback_dir_servers);
+}
+
+/** Return the dir_server_t for the directory authority whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_trusteddirserver_by_digest(const char *digest)
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of
+ * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir
+ * and UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+  if (!fallback_dir_servers)
+    return NULL;
+
+  if (!digest)
+    return NULL;
+
+  SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>,
+ * or 0 if no such fallback is in the list of fallback_dir_servers.
+ * (fallback_dir_servers is affected by the FallbackDir and
+ * UseDefaultFallbackDirs torrc options.)
+ * The list of fallback directories includes the list of authorities.
+ */
+int
+router_digest_is_fallback_dir(const char *digest)
+{
+  return (router_get_fallback_dirserver_by_digest(digest) != NULL);
+}
+
+/** Return the dir_server_t for the directory authority whose
+ * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
+ * is known.
+ */
+MOCK_IMPL(dir_server_t *,
+trusteddirserver_get_by_v3_auth_digest, (const char *digest))
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
+           (ds->type & V3_DIRINFO))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Mark as running every dir_server_t in <b>server_list</b>. */
+void
+mark_all_dirservers_up(smartlist_t *server_list)
+{
+  if (server_list) {
+    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
+      routerstatus_t *rs;
+      node_t *node;
+      dir->is_running = 1;
+      node = node_get_mutable_by_id(dir->digest);
+      if (node)
+        node->is_running = 1;
+      rs = router_get_mutable_consensus_status_by_id(dir->digest);
+      if (rs) {
+        rs->last_dir_503_at = 0;
+        control_event_networkstatus_changed_single(rs);
+      }
+    } SMARTLIST_FOREACH_END(dir);
+  }
+  router_dir_info_changed();
+}
+
+/** Return true iff <b>digest</b> is the digest of the identity key of a
+ * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
+ * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
+int
+router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
+{
+  if (!trusted_dir_servers)
+    return 0;
+  if (authdir_mode(get_options()) && router_digest_is_me(digest))
+    return 1;
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
+    if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
+      return (!type) || ((type & ent->type) != 0);
+    });
+  return 0;
+}
+
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
+ * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
+ * the new directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+               const char *nickname,
+               const tor_addr_t *addr,
+               const char *hostname,
+               uint16_t dir_port, uint16_t or_port,
+               const tor_addr_port_t *addrport_ipv6,
+               const char *digest, const char *v3_auth_digest,
+               dirinfo_type_t type,
+               double weight)
+{
+  dir_server_t *ent;
+  uint32_t a;
+  char *hostname_ = NULL;
+
+  tor_assert(digest);
+
+  if (weight < 0)
+    return NULL;
+
+  if (tor_addr_family(addr) == AF_INET)
+    a = tor_addr_to_ipv4h(addr);
+  else
+    return NULL;
+
+  if (!hostname)
+    hostname_ = tor_addr_to_str_dup(addr);
+  else
+    hostname_ = tor_strdup(hostname);
+
+  ent = tor_malloc_zero(sizeof(dir_server_t));
+  ent->nickname = nickname ? tor_strdup(nickname) : NULL;
+  ent->address = hostname_;
+  ent->addr = a;
+  ent->dir_port = dir_port;
+  ent->or_port = or_port;
+  ent->is_running = 1;
+  ent->is_authority = is_authority;
+  ent->type = type;
+  ent->weight = weight;
+  if (addrport_ipv6) {
+    if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) {
+      log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6.");
+      tor_addr_make_unspec(&ent->ipv6_addr);
+    } else {
+      tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr);
+      ent->ipv6_orport = addrport_ipv6->port;
+    }
+  } else {
+    tor_addr_make_unspec(&ent->ipv6_addr);
+  }
+
+  memcpy(ent->digest, digest, DIGEST_LEN);
+  if (v3_auth_digest && (type & V3_DIRINFO))
+    memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
+
+  if (nickname)
+    tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d",
+                 nickname, hostname_, (int)dir_port);
+  else
+    tor_asprintf(&ent->description, "directory server at %s:%d",
+                 hostname_, (int)dir_port);
+
+  ent->fake_status.addr = ent->addr;
+  tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr);
+  memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(ent->fake_status.nickname, nickname,
+            sizeof(ent->fake_status.nickname));
+  else
+    ent->fake_status.nickname[0] = '\0';
+  ent->fake_status.dir_port = ent->dir_port;
+  ent->fake_status.or_port = ent->or_port;
+  ent->fake_status.ipv6_orport = ent->ipv6_orport;
+
+  return ent;
+}
+
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
+ * <b>address</b> is NULL, add ourself.  Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *ipv6_addrport,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight)
+{
+  uint32_t a;
+  tor_addr_t addr;
+  char *hostname=NULL;
+  dir_server_t *result;
+
+  if (!address) { /* The address is us; we should guess. */
+    if (resolve_my_address(LOG_WARN, get_options(),
+                           &a, NULL, &hostname) < 0) {
+      log_warn(LD_CONFIG,
+               "Couldn't find a suitable address when adding ourself as a "
+               "trusted directory server.");
+      return NULL;
+    }
+    if (!hostname)
+      hostname = tor_dup_ip(a);
+  } else {
+    if (tor_lookup_hostname(address, &a)) {
+      log_warn(LD_CONFIG,
+               "Unable to lookup address for directory server at '%s'",
+               address);
+      return NULL;
+    }
+    hostname = tor_strdup(address);
+  }
+  tor_addr_from_ipv4h(&addr, a);
+
+  result = dir_server_new(1, nickname, &addr, hostname,
+                          dir_port, or_port,
+                          ipv6_addrport,
+                          digest,
+                          v3_auth_digest, type, weight);
+  tor_free(hostname);
+  return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+                        uint16_t dir_port, uint16_t or_port,
+                        const tor_addr_port_t *addrport_ipv6,
+                        const char *id_digest, double weight)
+{
+  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port,
+                        addrport_ipv6,
+                        id_digest,
+                        NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  if (ent->is_authority)
+    smartlist_add(trusted_dir_servers, ent);
+
+  smartlist_add(fallback_dir_servers, ent);
+  router_dir_info_changed();
+}
+
+#define dir_server_free(val) \
+  FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
+/** Free storage held in <b>ds</b>. */
+static void
+dir_server_free_(dir_server_t *ds)
+{
+  if (!ds)
+    return;
+
+  tor_free(ds->nickname);
+  tor_free(ds->description);
+  tor_free(ds->address);
+  tor_free(ds);
+}
+
+/** Remove all members from the list of dir servers. */
+void
+clear_dir_servers(void)
+{
+  if (fallback_dir_servers) {
+    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+                      dir_server_free(ent));
+    smartlist_clear(fallback_dir_servers);
+  } else {
+    fallback_dir_servers = smartlist_new();
+  }
+  if (trusted_dir_servers) {
+    smartlist_clear(trusted_dir_servers);
+  } else {
+    trusted_dir_servers = smartlist_new();
+  }
+  router_dir_info_changed();
+}
+
+void
+dirlist_free_all(void)
+{
+  clear_dir_servers();
+  smartlist_free(trusted_dir_servers);
+  smartlist_free(fallback_dir_servers);
+  trusted_dir_servers = fallback_dir_servers = NULL;
+}

src/feature/nodelist/dirlist.h

@@ -0,0 +1,47 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirlist.h
+ * \brief Header file for dirlist.c
+ **/
+
+#ifndef TOR_DIRLIST_H
+#define TOR_DIRLIST_H
+
+int get_n_authorities(dirinfo_type_t type);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
+smartlist_t *router_get_trusted_dir_servers_mutable(void);
+smartlist_t *router_get_fallback_dir_servers_mutable(void);
+void mark_all_dirservers_up(smartlist_t *server_list);
+
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+                                                   const char *digest);
+int router_digest_is_fallback_dir(const char *digest);
+MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
+          (const char *d));
+
+int router_digest_is_trusted_dir_type(const char *digest,
+                                      dirinfo_type_t type);
+#define router_digest_is_trusted_dir(d) \
+  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const tor_addr_port_t *addrport_ipv6,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+                                      uint16_t dir_port, uint16_t or_port,
+                                      const tor_addr_port_t *addrport_ipv6,
+                                      const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
+void clear_dir_servers(void);
+void dirlist_free_all(void);
+
+#endif

src/feature/nodelist/fmt_routerstatus.c

@@ -0,0 +1,253 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_routerstatus.h
+ * \brief Format routerstatus entries for controller, vote, or consensus.
+ *
+ * (Because controllers consume this format, we can't make this
+ * code dirauth-only.)
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/fmt_routerstatus.h"
+
+/* #include "lib/container/buffers.h" */
+/* #include "app/config/config.h" */
+/* #include "app/config/confparse.h" */
+/* #include "core/or/channel.h" */
+/* #include "core/or/channeltls.h" */
+/* #include "core/or/command.h" */
+/* #include "core/mainloop/connection.h" */
+/* #include "core/or/connection_or.h" */
+/* #include "feature/dircache/conscache.h" */
+/* #include "feature/dircache/consdiffmgr.h" */
+/* #include "feature/control/control.h" */
+/* #include "feature/dircache/directory.h" */
+/* #include "feature/dircache/dirserv.h" */
+/* #include "feature/hibernate/hibernate.h" */
+/* #include "feature/dirauth/keypin.h" */
+/* #include "core/mainloop/main.h" */
+/* #include "feature/nodelist/microdesc.h" */
+/* #include "feature/nodelist/networkstatus.h" */
+/* #include "feature/nodelist/nodelist.h" */
+#include "core/or/policies.h"
+/* #include "core/or/protover.h" */
+/* #include "feature/stats/rephist.h" */
+/* #include "feature/relay/router.h" */
+/* #include "feature/nodelist/dirlist.h" */
+#include "feature/nodelist/routerlist.h"
+
+/* #include "feature/nodelist/routerparse.h" */
+/* #include "feature/nodelist/routerset.h" */
+/* #include "feature/nodelist/torcert.h" */
+/* #include "feature/dircommon/voting_schedule.h" */
+
+#include "feature/dirauth/dirvote.h"
+
+/* #include "feature/dircache/cached_dir_st.h" */
+/* #include "feature/dircommon/dir_connection_st.h" */
+/* #include "feature/nodelist/extrainfo_st.h" */
+/* #include "feature/nodelist/microdesc_st.h" */
+/* #include "feature/nodelist/node_st.h" */
+#include "feature/nodelist/routerinfo_st.h"
+/* #include "feature/nodelist/routerlist_st.h" */
+/* #include "core/or/tor_version_st.h" */
+#include "feature/nodelist/vote_routerstatus_st.h"
+
+/* #include "lib/compress/compress.h" */
+/* #include "lib/container/order.h" */
+#include "lib/crypt_ops/crypto_format.h"
+/* #include "lib/encoding/confline.h" */
+
+/* #include "lib/encoding/keyval.h" */
+
+/** Helper: write the router-status information in <b>rs</b> into a newly
+ * allocated character buffer.  Use the same format as in network-status
+ * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
+ *
+ * consensus_method is the current consensus method when format is
+ * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
+ * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
+ *
+ * Return 0 on success, -1 on failure.
+ *
+ * The format argument has one of the following values:
+ *   NS_V2 - Output an entry suitable for a V2 NS opinion document
+ *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
+ *        for consensus_method.
+ *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
+ *        consensus entry for consensus_method.
+ *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
+ *        it contains additional information for the vote.
+ *   NS_CONTROL_PORT - Output a NS document for the control port.
+ */
+char *
+routerstatus_format_entry(const routerstatus_t *rs, const char *version,
+                          const char *protocols,
+                          routerstatus_format_type_t format,
+                          int consensus_method,
+                          const vote_routerstatus_t *vrs)
+{
+  char *summary;
+  char *result = NULL;
+
+  char published[ISO_TIME_LEN+1];
+  char identity64[BASE64_DIGEST_LEN+1];
+  char digest64[BASE64_DIGEST_LEN+1];
+  smartlist_t *chunks = smartlist_new();
+
+  format_iso_time(published, rs->published_on);
+  digest_to_base64(identity64, rs->identity_digest);
+  digest_to_base64(digest64, rs->descriptor_digest);
+
+  smartlist_add_asprintf(chunks,
+                   "r %s %s %s%s%s %s %d %d\n",
+                   rs->nickname,
+                   identity64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
+                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
+                   published,
+                   fmt_addr32(rs->addr),
+                   (int)rs->or_port,
+                   (int)rs->dir_port);
+
+  /* TODO: Maybe we want to pass in what we need to build the rest of
+   * this here, instead of in the caller. Then we could use the
+   * networkstatus_type_t values, with an additional control port value
+   * added -MP */
+
+  /* V3 microdesc consensuses only have "a" lines in later consensus methods
+   */
+  if (format == NS_V3_CONSENSUS_MICRODESC &&
+      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
+    goto done;
+
+  /* Possible "a" line. At most one for now. */
+  if (!tor_addr_is_null(&rs->ipv6_addr)) {
+    smartlist_add_asprintf(chunks, "a %s\n",
+                           fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+  }
+
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
+    goto done;
+
+  smartlist_add_asprintf(chunks,
+                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                  /* These must stay in alphabetical order. */
+                   rs->is_authority?" Authority":"",
+                   rs->is_bad_exit?" BadExit":"",
+                   rs->is_exit?" Exit":"",
+                   rs->is_fast?" Fast":"",
+                   rs->is_possible_guard?" Guard":"",
+                   rs->is_hs_dir?" HSDir":"",
+                   rs->is_flagged_running?" Running":"",
+                   rs->is_stable?" Stable":"",
+                   rs->is_v2_dir?" V2Dir":"",
+                   rs->is_valid?" Valid":"");
+
+  /* length of "opt v \n" */
+#define V_LINE_OVERHEAD 7
+  if (version && strlen(version) < MAX_V_LINE_LEN - V_LINE_OVERHEAD) {
+    smartlist_add_asprintf(chunks, "v %s\n", version);
+  }
+  if (protocols) {
+    smartlist_add_asprintf(chunks, "pr %s\n", protocols);
+  }
+
+  if (format != NS_V2) {
+    const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest);
+    uint32_t bw_kb;
+
+    if (format != NS_CONTROL_PORT) {
+      /* Blow up more or less nicely if we didn't get anything or not the
+       * thing we expected.
+       */
+      if (!desc) {
+        char id[HEX_DIGEST_LEN+1];
+        char dd[HEX_DIGEST_LEN+1];
+
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN);
+        log_warn(LD_BUG, "Cannot get any descriptor for %s "
+            "(wanted descriptor %s).",
+            id, dd);
+        goto err;
+      }
+
+      /* This assert could fire for the control port, because
+       * it can request NS documents before all descriptors
+       * have been fetched. Therefore, we only do this test when
+       * format != NS_CONTROL_PORT. */
+      if (tor_memneq(desc->cache_info.signed_descriptor_digest,
+            rs->descriptor_digest,
+            DIGEST_LEN)) {
+        char rl_d[HEX_DIGEST_LEN+1];
+        char rs_d[HEX_DIGEST_LEN+1];
+        char id[HEX_DIGEST_LEN+1];
+
+        base16_encode(rl_d, sizeof(rl_d),
+            desc->cache_info.signed_descriptor_digest, DIGEST_LEN);
+        base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN);
+        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
+        log_err(LD_BUG, "descriptor digest in routerlist does not match "
+            "the one in routerstatus: %s vs %s "
+            "(router %s)\n",
+            rl_d, rs_d, id);
+
+        tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest,
+              rs->descriptor_digest,
+              DIGEST_LEN));
+      }
+    }
+
+    if (format == NS_CONTROL_PORT && rs->has_bandwidth) {
+      bw_kb = rs->bandwidth_kb;
+    } else {
+      tor_assert(desc);
+      bw_kb = router_get_advertised_bandwidth_capped(desc) / 1000;
+    }
+    smartlist_add_asprintf(chunks,
+                     "w Bandwidth=%d", bw_kb);
+
+    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
+      smartlist_add_asprintf(chunks,
+                       " Measured=%d", vrs->measured_bw_kb);
+    }
+    /* Write down guardfraction information if we have it. */
+    if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) {
+      smartlist_add_asprintf(chunks,
+                             " GuardFraction=%d",
+                             vrs->status.guardfraction_percentage);
+    }
+
+    smartlist_add_strdup(chunks, "\n");
+
+    if (desc) {
+      summary = policy_summarize(desc->exit_policy, AF_INET);
+      smartlist_add_asprintf(chunks, "p %s\n", summary);
+      tor_free(summary);
+    }
+
+    if (format == NS_V3_VOTE && vrs) {
+      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+        smartlist_add_strdup(chunks, "id ed25519 none\n");
+      } else {
+        char ed_b64[BASE64_DIGEST256_LEN+1];
+        digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
+        smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64);
+      }
+    }
+  }
+
+ done:
+  result = smartlist_join_strings(chunks, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+
+  return result;
+}

src/feature/nodelist/fmt_routerstatus.h

@@ -0,0 +1,41 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fmt_routerstatus.h
+ * \brief Header file for fmt_routerstatus.c.
+ **/
+
+#ifndef TOR_FMT_ROUTERSTATUS_H
+#define TOR_FMT_ROUTERSTATUS_H
+
+/** An enum to describe what format we're generating a routerstatus line in.
+ */
+typedef enum {
+  /** For use in a v2 opinion */
+  NS_V2,
+  /** For use in a consensus networkstatus document (ns flavor) */
+  NS_V3_CONSENSUS,
+  /** For use in a vote networkstatus document */
+  NS_V3_VOTE,
+  /** For passing to the controlport in response to a GETINFO request */
+  NS_CONTROL_PORT,
+  /** For use in a consensus networkstatus document (microdesc flavor) */
+  NS_V3_CONSENSUS_MICRODESC
+} routerstatus_format_type_t;
+
+/** Maximum allowable length of a version line in a networkstatus. */
+#define MAX_V_LINE_LEN 128
+
+char *routerstatus_format_entry(
+                              const routerstatus_t *rs,
+                              const char *version,
+                              const char *protocols,
+                              routerstatus_format_type_t format,
+                              int consensus_method,
+                              const vote_routerstatus_t *vrs);
+
+#endif /* !defined(TOR_FMT_ROUTERSTATUS_H) */

src/feature/nodelist/microdesc.c

@@ -22,6 +22,7 @@
 #include "feature/nodelist/nodelist.h"
 #include "core/or/policies.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 

src/feature/nodelist/networkstatus.c

@@ -53,6 +53,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircache/directory.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/reachability.h"
 #include "core/or/dos.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"
@@ -63,6 +64,9 @@
 #include "core/or/protover.h"
 #include "core/or/relay.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "core/or/scheduler.h"
@@ -70,10 +74,12 @@
 #include "feature/nodelist/torcert.h"
 #include "core/or/channelpadding.h"
 #include "feature/dircommon/voting_schedule.h"
+#include "feature/nodelist/fmt_routerstatus.h"
 
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/mode.h"
 #include "feature/dirauth/shared_random.h"
+#include "feature/dirauth/voteflags.h"
 
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/dircommon/dir_connection_st.h"

src/feature/nodelist/node_select.c

@@ -0,0 +1,1108 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.c
+ * \brief Code to choose nodes randomly based on restrictions and
+ *   weighted probabilities.
+ **/
+
+#define NODE_SELECT_PRIVATE
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/policies.h"
+#include "core/or/reasons.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dircache/directory.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/math/fp.h"
+
+#include "feature/dirclient/dir_server_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+
+static int compute_weighted_bandwidths(const smartlist_t *sl,
+                                       bandwidth_weight_rule_t rule,
+                                       double **bandwidths_out,
+                                       double *total_bandwidth_out);
+static const routerstatus_t *router_pick_trusteddirserver_impl(
+                const smartlist_t *sourcelist, dirinfo_type_t auth,
+                int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+                              smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags);
+
+/** Try to find a running dirserver that supports operations of <b>type</b>.
+ *
+ * If there are no running dirservers in our routerlist and the
+ * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones
+ * (including authorities) as running again, and pick one.
+ *
+ * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include
+ * dirservers that we can't reach.
+ *
+ * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself
+ * (if we're a dirserver).
+ *
+ * Don't pick a fallback directory mirror if any non-fallback is viable;
+ * (the fallback directory mirrors include the authorities)
+ * try to avoid using servers that have returned 503 recently.
+ */
+const routerstatus_t *
+router_pick_directory_server(dirinfo_type_t type, int flags)
+{
+  int busy = 0;
+  const routerstatus_t *choice;
+
+  choice = router_pick_directory_server_impl(type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No reachable router entries for dirservers. "
+           "Trying them all again.");
+  /* mark all fallback directory mirrors as up again */
+  router_reset_status_download_failures();
+  /* try again */
+  choice = router_pick_directory_server_impl(type, flags, NULL);
+  return choice;
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags)
+{
+  const routerstatus_t *choice;
+  int busy = 0;
+
+  if (smartlist_len(sourcelist) == 1) {
+    /* If there's only one choice, then we should disable the logic that
+     * would otherwise prevent us from choosing ourself. */
+    flags |= PDS_ALLOW_SELF;
+  }
+
+  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
+  if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
+    return choice;
+  if (busy) {
+    /* If the reason that we got no server is that servers are "busy",
+     * we must be excluding good servers because we already have serverdesc
+     * fetches with them.  Do not mark down servers up because of this. */
+    tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
+                         PDS_NO_EXISTING_MICRODESC_FETCH)));
+    return NULL;
+  }
+
+  log_info(LD_DIR,
+           "No dirservers are reachable. Trying them all again.");
+  mark_all_dirservers_up(sourcelist);
+  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
+}
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version.
+ * Must be called before RETRY_WITHOUT_EXCLUDE().
+ *
+ * If we got no result, and we are applying IP preferences, and we are a
+ * client that could use an alternate IP version, try again with the
+ * opposite preferences. */
+#define RETRY_ALTERNATE_IP_VERSION(retry_label)                               \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_ip_pref && options->ClientUseIPv4               \
+        && fascist_firewall_use_ipv6(options) && !server_mode(options)        \
+        && !n_busy) {                                                         \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 0;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END                                                                    \
+
+/* Common retry code for router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
+ * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION().
+ *
+ * If we got no result, and we are excluding nodes, and StrictNodes is
+ * not set, try again without excluding nodes. */
+#define RETRY_WITHOUT_EXCLUDE(retry_label)                                    \
+  STMT_BEGIN                                                                  \
+    if (result == NULL && try_excluding && !options->StrictNodes              \
+        && n_excluded && !n_busy) {                                           \
+      try_excluding = 0;                                                      \
+      n_excluded = 0;                                                         \
+      n_busy = 0;                                                             \
+      try_ip_pref = 1;                                                        \
+      goto retry_label;                                                       \
+    }                                                                         \
+  STMT_END
+
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
+#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0
+#endif
+
+/* Log a message if rs is not found or not a preferred address */
+static void
+router_picked_poor_directory_log(const routerstatus_t *rs)
+{
+  const networkstatus_t *usable_consensus;
+  usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL),
+                                                 usable_consensus_flavor());
+
+#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+  /* Don't log early in the bootstrap process, it's normal to pick from a
+   * small pool of nodes. Of course, this won't help if we're trying to
+   * diagnose bootstrap issues. */
+  if (!smartlist_len(nodelist_get_list()) || !usable_consensus
+      || !router_have_minimum_dir_info()) {
+    return;
+  }
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
+
+  /* We couldn't find a node, or the one we have doesn't fit our preferences.
+   * Sometimes this is normal, sometimes it can be a reachability issue. */
+  if (!rs) {
+    /* This happens a lot, so it's at debug level */
+    log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but "
+              "we couldn't find a directory that fit our criteria. "
+              "Perhaps we will succeed next time with less strict criteria.");
+  } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1)
+             && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1)
+             ) {
+    /* This is rare, and might be interesting to users trying to diagnose
+     * connection issues on dual-stack machines. */
+    log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir "
+             "addresses for launching an outgoing connection: "
+             "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d",
+             routerstatus_describe(rs),
+             fmt_addr32(rs->addr), rs->or_port,
+             rs->dir_port, fmt_addr(&rs->ipv6_addr),
+             rs->ipv6_orport, rs->dir_port);
+  }
+}
+
+#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP
+
+/* Check if we already have a directory fetch from ap, for serverdesc
+ * (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0.
+ */
+STATIC int
+router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
+                               int microdesc)
+{
+  if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) {
+    return 0;
+  }
+
+  /* XX/teor - we're not checking tunnel connections here, see #17848
+   */
+  if (serverdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC)
+  || connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) {
+    return 1;
+  }
+
+  if (microdesc && (
+     connection_get_by_type_addr_port_purpose(
+       CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
+ * If so, return 1, if not, return 0.
+ */
+static int
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
+{
+  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
+
+  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
+
+  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
+       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
+}
+
+/** Pick a random running valid directory server/mirror from our
+ * routerlist.  Arguments are as for router_pick_directory_server(), except:
+ *
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
+ * directories that we excluded for no other reason than
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
+ */
+STATIC const routerstatus_t *
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  const node_t *result;
+  smartlist_t *direct, *tunnel;
+  smartlist_t *trusted_direct, *trusted_tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  time_t now = time(NULL);
+  const networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  int try_excluding = 1, n_excluded = 0, n_busy = 0;
+  int try_ip_pref = 1;
+
+  if (!consensus)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  trusted_direct = smartlist_new();
+  trusted_tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  /* Find all the running dirservers we know about. */
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
+    int is_trusted;
+    int is_overloaded;
+    const routerstatus_t *status = node->rs;
+    const country_t country = node->country;
+    if (!status)
+      continue;
+
+    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
+      continue;
+    if (requireother && router_digest_is_me(node->identity))
+      continue;
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
+    if (try_excluding &&
+        routerset_contains_routerstatus(options->ExcludeNodes, status,
+                                        country)) {
+      ++n_excluded;
+      continue;
+    }
+
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
+      ++n_busy;
+      continue;
+    }
+
+    is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
+
+    /* Clients use IPv6 addresses if the server has one and the client
+     * prefers IPv6.
+     * Add the router if its preferred address and port are reachable.
+     * If we don't get any routers, we'll try again with the non-preferred
+     * address for each router (if any). (To ensure correct load-balancing
+     * we try routers that only have one address both times.)
+     */
+    if (!fascistfirewall || skip_or_fw ||
+        fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
+                                     try_ip_pref))
+      smartlist_add(is_trusted ? trusted_tunnel :
+                    is_overloaded ? overloaded_tunnel : tunnel, (void*)node);
+    else if (!must_have_or && (skip_dir_fw ||
+             fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION,
+                                          try_ip_pref)))
+      smartlist_add(is_trusted ? trusted_direct :
+                    is_overloaded ? overloaded_direct : direct, (void*)node);
+  } SMARTLIST_FOREACH_END(node);
+
+  if (smartlist_len(tunnel)) {
+    result = node_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_tunnel)) {
+    result = node_sl_choose_by_bandwidth(overloaded_tunnel,
+                                                 WEIGHT_FOR_DIR);
+  } else if (smartlist_len(trusted_tunnel)) {
+    /* FFFF We don't distinguish between trusteds and overloaded trusteds
+     * yet. Maybe one day we should. */
+    /* FFFF We also don't load balance over authorities yet. I think this
+     * is a feature, but it could easily be a bug. -RD */
+    result = smartlist_choose(trusted_tunnel);
+  } else if (smartlist_len(direct)) {
+    result = node_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR);
+  } else if (smartlist_len(overloaded_direct)) {
+    result = node_sl_choose_by_bandwidth(overloaded_direct,
+                                         WEIGHT_FOR_DIR);
+  } else {
+    result = smartlist_choose(trusted_direct);
+  }
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(trusted_direct);
+  smartlist_free(trusted_tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+
+  router_picked_poor_directory_log(result ? result->rs : NULL);
+
+  return result ? result->rs : NULL;
+}
+
+/** Given an array of double/uint64_t unions that are currently being used as
+ * doubles, convert them to uint64_t, and try to scale them linearly so as to
+ * much of the range of uint64_t. If <b>total_out</b> is provided, set it to
+ * the sum of all elements in the array _before_ scaling. */
+STATIC void
+scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in,
+                            int n_entries,
+                            uint64_t *total_out)
+{
+  double total = 0.0;
+  double scale_factor = 0.0;
+  int i;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries_in[i];
+
+  if (total > 0.0) {
+    scale_factor = ((double)INT64_MAX) / total;
+    scale_factor /= 4.0; /* make sure we're very far away from overflowing */
+  }
+
+  for (i = 0; i < n_entries; ++i)
+    entries_out[i] = tor_llround(entries_in[i] * scale_factor);
+
+  if (total_out)
+    *total_out = (uint64_t) total;
+}
+
+/** Pick a random element of <b>n_entries</b>-element array <b>entries</b>,
+ * choosing each element with a probability proportional to its (uint64_t)
+ * value, and return the index of that element.  If all elements are 0, choose
+ * an index at random. Return -1 on error.
+ */
+STATIC int
+choose_array_element_by_weight(const uint64_t *entries, int n_entries)
+{
+  int i;
+  uint64_t rand_val;
+  uint64_t total = 0;
+
+  for (i = 0; i < n_entries; ++i)
+    total += entries[i];
+
+  if (n_entries < 1)
+    return -1;
+
+  if (total == 0)
+    return crypto_rand_int(n_entries);
+
+  tor_assert(total < INT64_MAX);
+
+  rand_val = crypto_rand_uint64(total);
+
+  return select_array_member_cumulative_timei(
+                           entries, n_entries, total, rand_val);
+}
+
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static inline int32_t
+kb_to_bytes(uint32_t bw)
+{
+  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
+}
+
+/** Helper function:
+ * choose a random element of smartlist <b>sl</b> of nodes, weighted by
+ * the advertised bandwidth of each element using the consensus
+ * bandwidth weights.
+ *
+ * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all
+ * nodes' bandwidth equally regardless of their Exit status, since there may
+ * be some in the list because they exit to obscure ports. If
+ * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight
+ * exit-node's bandwidth less depending on the smallness of the fraction of
+ * Exit-to-total bandwidth.  If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a
+ * guard node: consider all guard's bandwidth equally. Otherwise, weight
+ * guards proportionally less.
+ */
+static const node_t *
+smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl,
+                                           bandwidth_weight_rule_t rule)
+{
+  double *bandwidths_dbl=NULL;
+  uint64_t *bandwidths_u64=NULL;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl, NULL) < 0)
+    return NULL;
+
+  bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t));
+  scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl,
+                              smartlist_len(sl), NULL);
+
+  {
+    int idx = choose_array_element_by_weight(bandwidths_u64,
+                                             smartlist_len(sl));
+    tor_free(bandwidths_dbl);
+    tor_free(bandwidths_u64);
+    return idx < 0 ? NULL : smartlist_get(sl, idx);
+  }
+}
+
+/** When weighting bridges, enforce these values as lower and upper
+ * bound for believable bandwidth, because there is no way for us
+ * to verify a bridge's bandwidth currently. */
+#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000  /* 20 kB/sec */
+#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */
+
+/** Return the smaller of the router's configured BandwidthRate
+ * and its advertised capacity, making sure to stay within the
+ * interval between bridge-min-believe-bw and
+ * bridge-max-believe-bw. */
+static uint32_t
+bridge_get_advertised_bandwidth_bounded(routerinfo_t *router)
+{
+  uint32_t result = router->bandwidthcapacity;
+  if (result > router->bandwidthrate)
+    result = router->bandwidthrate;
+  if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH;
+  else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH)
+    result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH;
+  return result;
+}
+
+/** Given a list of routers and a weighting rule as in
+ * smartlist_choose_node_by_bandwidth_weights, compute weighted bandwidth
+ * values for each node and store them in a freshly allocated
+ * *<b>bandwidths_out</b> of the same length as <b>sl</b>, and holding results
+ * as doubles. If <b>total_bandwidth_out</b> is non-NULL, set it to the total
+ * of all the bandwidths.
+ * Return 0 on success, -1 on failure. */
+static int
+compute_weighted_bandwidths(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            double **bandwidths_out,
+                            double *total_bandwidth_out)
+{
+  int64_t weight_scale;
+  double Wg = -1, Wm = -1, We = -1, Wd = -1;
+  double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1;
+  guardfraction_bandwidth_t guardfraction_bw;
+  double *bandwidths = NULL;
+  double total_bandwidth = 0.0;
+
+  tor_assert(sl);
+  tor_assert(bandwidths_out);
+
+  /* Can't choose exit and guard at same time */
+  tor_assert(rule == NO_WEIGHTING ||
+             rule == WEIGHT_FOR_EXIT ||
+             rule == WEIGHT_FOR_GUARD ||
+             rule == WEIGHT_FOR_MID ||
+             rule == WEIGHT_FOR_DIR);
+
+  *bandwidths_out = NULL;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = 0.0;
+  }
+
+  if (smartlist_len(sl) == 0) {
+    log_info(LD_CIRC,
+             "Empty routerlist passed in to consensus weight node "
+             "selection for rule %s",
+             bandwidth_weight_rule_to_string(rule));
+    return -1;
+  }
+
+  weight_scale = networkstatus_get_weight_scale_param(NULL);
+
+  if (rule == WEIGHT_FOR_GUARD) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */
+    We = 0;
+    Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_MID) {
+    Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1);
+    We = networkstatus_get_bw_weight(NULL, "Wme", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1);
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_EXIT) {
+    // Guards CAN be exits if they have weird exit policies
+    // They are d then I guess...
+    We = networkstatus_get_bw_weight(NULL, "Wee", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */
+    Wd = networkstatus_get_bw_weight(NULL, "Wed", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */
+
+    Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1);
+    Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1);
+    Web = networkstatus_get_bw_weight(NULL, "Web", -1);
+    Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1);
+  } else if (rule == WEIGHT_FOR_DIR) {
+    We = networkstatus_get_bw_weight(NULL, "Wbe", -1);
+    Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1);
+    Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1);
+    Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1);
+
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  } else if (rule == NO_WEIGHTING) {
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0
+      || Web < 0) {
+    log_debug(LD_CIRC,
+              "Got negative bandwidth weights. Defaulting to naive selection"
+              " algorithm.");
+    Wg = Wm = We = Wd = weight_scale;
+    Wgb = Wmb = Web = Wdb = weight_scale;
+  }
+
+  Wg /= weight_scale;
+  Wm /= weight_scale;
+  We /= weight_scale;
+  Wd /= weight_scale;
+
+  Wgb /= weight_scale;
+  Wmb /= weight_scale;
+  Web /= weight_scale;
+  Wdb /= weight_scale;
+
+  bandwidths = tor_calloc(smartlist_len(sl), sizeof(double));
+
+  // Cycle through smartlist and total the bandwidth.
+  static int warned_missing_bw = 0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0;
+    double weight = 1;
+    double weight_without_guard_flag = 0; /* Used for guardfraction */
+    double final_weight = 0;
+    is_exit = node->is_exit && ! node->is_bad_exit;
+    is_guard = node->is_possible_guard;
+    is_dir = node_is_dir(node);
+    if (node->rs) {
+      if (!node->rs->has_bandwidth) {
+        /* This should never happen, unless all the authorities downgrade
+         * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
+        if (! warned_missing_bw) {
+          log_warn(LD_BUG,
+                 "Consensus is missing some bandwidths. Using a naive "
+                 "router selection algorithm");
+          warned_missing_bw = 1;
+        }
+        this_bw = 30000; /* Chosen arbitrarily */
+      } else {
+        this_bw = kb_to_bytes(node->rs->bandwidth_kb);
+      }
+    } else if (node->ri) {
+      /* bridge or other descriptor not in our consensus */
+      this_bw = bridge_get_advertised_bandwidth_bounded(node->ri);
+    } else {
+      /* We can't use this one. */
+      continue;
+    }
+
+    if (is_guard && is_exit) {
+      weight = (is_dir ? Wdb*Wd : Wd);
+      weight_without_guard_flag = (is_dir ? Web*We : We);
+    } else if (is_guard) {
+      weight = (is_dir ? Wgb*Wg : Wg);
+      weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm);
+    } else if (is_exit) {
+      weight = (is_dir ? Web*We : We);
+    } else { // middle
+      weight = (is_dir ? Wmb*Wm : Wm);
+    }
+    /* These should be impossible; but overflows here would be bad, so let's
+     * make sure. */
+    if (this_bw < 0)
+      this_bw = 0;
+    if (weight < 0.0)
+      weight = 0.0;
+    if (weight_without_guard_flag < 0.0)
+      weight_without_guard_flag = 0.0;
+
+    /* If guardfraction information is available in the consensus, we
+     * want to calculate this router's bandwidth according to its
+     * guardfraction. Quoting from proposal236:
+     *
+     *    Let Wpf denote the weight from the 'bandwidth-weights' line a
+     *    client would apply to N for position p if it had the guard
+     *    flag, Wpn the weight if it did not have the guard flag, and B the
+     *    measured bandwidth of N in the consensus.  Then instead of choosing
+     *    N for position p proportionally to Wpf*B or Wpn*B, clients should
+     *    choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+     */
+    if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) {
+      /* XXX The assert should actually check for is_guard. However,
+       * that crashes dirauths because of #13297. This should be
+       * equivalent: */
+      tor_assert(node->rs->is_possible_guard);
+
+      guard_get_guardfraction_bandwidth(&guardfraction_bw,
+                                        this_bw,
+                                        node->rs->guardfraction_percentage);
+
+      /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */
+      final_weight =
+        guardfraction_bw.guard_bw * weight +
+        guardfraction_bw.non_guard_bw * weight_without_guard_flag;
+
+      log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)",
+                node->rs->nickname, final_weight, weight*this_bw,
+                bandwidth_weight_rule_to_string(rule));
+    } else { /* no guardfraction information. calculate the weight normally. */
+      final_weight = weight*this_bw;
+    }
+
+    bandwidths[node_sl_idx] = final_weight;
+    total_bandwidth += final_weight;
+  } SMARTLIST_FOREACH_END(node);
+
+  log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based "
+            "on weights "
+            "Wg=%f Wm=%f We=%f Wd=%f with total bw %f",
+            bandwidth_weight_rule_to_string(rule),
+            Wg, Wm, We, Wd, total_bandwidth);
+
+  *bandwidths_out = bandwidths;
+
+  if (total_bandwidth_out) {
+    *total_bandwidth_out = total_bandwidth;
+  }
+
+  return 0;
+}
+
+/** For all nodes in <b>sl</b>, return the fraction of those nodes, weighted
+ * by their weighted bandwidths with rule <b>rule</b>, for which we have
+ * descriptors.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect
+ * to it indirectly, or use it as if we were connecting to it indirectly. */
+double
+frac_nodes_with_descriptors(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule,
+                            int for_direct_conn)
+{
+  double *bandwidths = NULL;
+  double total, present;
+
+  if (smartlist_len(sl) == 0)
+    return 0.0;
+
+  if (compute_weighted_bandwidths(sl, rule, &bandwidths, &total) < 0 ||
+      total <= 0.0) {
+    int n_with_descs = 0;
+    SMARTLIST_FOREACH(sl, const node_t *, node, {
+      if (node_has_preferred_descriptor(node, for_direct_conn))
+        n_with_descs++;
+    });
+    tor_free(bandwidths);
+    return ((double)n_with_descs) / smartlist_len(sl);
+  }
+
+  present = 0.0;
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    if (node_has_preferred_descriptor(node, for_direct_conn))
+      present += bandwidths[node_sl_idx];
+  } SMARTLIST_FOREACH_END(node);
+
+  tor_free(bandwidths);
+
+  return present / total;
+}
+
+/** Choose a random element of status list <b>sl</b>, weighted by
+ * the advertised bandwidth of each node */
+const node_t *
+node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                            bandwidth_weight_rule_t rule)
+{ /*XXXX MOVE */
+  return smartlist_choose_node_by_bandwidth_weights(sl, rule);
+}
+
+/** Given a <b>router</b>, add every node_t in its family (including the
+ * node itself!) to <b>sl</b>.
+ *
+ * Note the type mismatch: This function takes a routerinfo, but adds nodes
+ * to the smartlist!
+ */
+static void
+routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
+{
+  /* XXXX MOVE ? */
+  node_t fake_node;
+  const node_t *node = node_get_by_id(router->cache_info.identity_digest);
+  if (node == NULL) {
+    memset(&fake_node, 0, sizeof(fake_node));
+    fake_node.ri = (routerinfo_t *)router;
+    memcpy(fake_node.identity, router->cache_info.identity_digest, DIGEST_LEN);
+    node = &fake_node;
+  }
+  nodelist_add_node_and_family(sl, node);
+}
+
+/** Return a random running node from the nodelist. Never
+ * pick a node that is in
+ * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
+ * even if they are the only nodes available.
+ * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than
+ * a minimum uptime, return one of those.
+ * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
+ * advertised capacity of each router.
+ * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
+ * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
+ * picking an exit node, otherwise we weight bandwidths for picking a relay
+ * node (that is, possibly discounting exit nodes).
+ * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that
+ * have a routerinfo or microdescriptor -- that is, enough info to be
+ * used to build a circuit.
+ * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that
+ * have an address that is preferred by the ClientPreferIPv6ORPort setting
+ * (regardless of this flag, we exclude nodes that aren't allowed by the
+ * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0).
+ */
+const node_t *
+router_choose_random_node(smartlist_t *excludedsmartlist,
+                          routerset_t *excludedset,
+                          router_crn_flags_t flags)
+{ /* XXXX MOVE */
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int need_guard = (flags & CRN_NEED_GUARD) != 0;
+  const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
+  const int need_desc = (flags & CRN_NEED_DESC) != 0;
+  const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
+
+  smartlist_t *sl=smartlist_new(),
+    *excludednodes=smartlist_new();
+  const node_t *choice = NULL;
+  const routerinfo_t *r;
+  bandwidth_weight_rule_t rule;
+
+  tor_assert(!(weight_for_exit && need_guard));
+  rule = weight_for_exit ? WEIGHT_FOR_EXIT :
+    (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
+
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+    if (node_allows_single_hop_exits(node)) {
+      /* Exclude relays that allow single hop exit circuits. This is an
+       * obsolete option since 0.2.9.2-alpha and done by default in
+       * 0.3.1.0-alpha. */
+      smartlist_add(excludednodes, node);
+    } else if (rendezvous_v3 &&
+               !node_supports_v3_rendezvous_point(node)) {
+      /* Exclude relays that do not support to rendezvous for a hidden service
+       * version 3. */
+      smartlist_add(excludednodes, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* If the node_t is not found we won't be to exclude ourself but we
+   * won't be able to pick ourself in router_choose_random_node() so
+   * this is fine to at least try with our routerinfo_t object. */
+  if ((r = router_get_my_routerinfo()))
+    routerlist_add_node_and_family(excludednodes, r);
+
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
+                                        need_guard, need_desc, pref_addr,
+                                        direct_conn);
+  log_debug(LD_CIRC,
+           "We found %d running nodes.",
+            smartlist_len(sl));
+
+  smartlist_subtract(sl,excludednodes);
+  log_debug(LD_CIRC,
+            "We removed %d excludednodes, leaving %d nodes.",
+            smartlist_len(excludednodes),
+            smartlist_len(sl));
+
+  if (excludedsmartlist) {
+    smartlist_subtract(sl,excludedsmartlist);
+    log_debug(LD_CIRC,
+              "We removed %d excludedsmartlist, leaving %d nodes.",
+              smartlist_len(excludedsmartlist),
+              smartlist_len(sl));
+  }
+  if (excludedset) {
+    routerset_subtract_nodes(sl,excludedset);
+    log_debug(LD_CIRC,
+              "We removed excludedset, leaving %d nodes.",
+              smartlist_len(sl));
+  }
+
+  // Always weight by bandwidth
+  choice = node_sl_choose_by_bandwidth(sl, rule);
+
+  smartlist_free(sl);
+  if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) {
+    /* try once more -- recurse but with fewer restrictions. */
+    log_info(LD_CIRC,
+             "We couldn't find any live%s%s%s routers; falling back "
+             "to list of all routers.",
+             need_capacity?", fast":"",
+             need_uptime?", stable":"",
+             need_guard?", guard":"");
+    flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD|
+                CRN_PREF_ADDR);
+    choice = router_choose_random_node(
+                     excludedsmartlist, excludedset, flags);
+  }
+  smartlist_free(excludednodes);
+  if (!choice) {
+    log_warn(LD_CIRC,
+             "No available nodes when trying to choose node. Failing.");
+  }
+  return choice;
+}
+
+/** Try to find a running directory authority. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(
+                                  router_get_trusted_dir_servers_mutable(),
+                                  type, flags);
+}
+
+/** Try to find a running fallback directory. Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(
+                                  router_get_fallback_dir_servers_mutable(),
+                                  type, flags);
+}
+
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+  int n = smartlist_len(servers);
+  int i;
+  double *weights_dbl;
+  uint64_t *weights_u64;
+  const dir_server_t *ds;
+
+  weights_dbl = tor_calloc(n, sizeof(double));
+  weights_u64 = tor_calloc(n, sizeof(uint64_t));
+  for (i = 0; i < n; ++i) {
+    ds = smartlist_get(servers, i);
+    weights_dbl[i] = ds->weight;
+    if (ds->is_authority)
+      weights_dbl[i] *= authority_weight;
+  }
+
+  scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL);
+  i = choose_array_element_by_weight(weights_u64, n);
+  tor_free(weights_dbl);
+  tor_free(weights_u64);
+  return (i < 0) ? NULL : smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
+ */
+static const routerstatus_t *
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+                                  dirinfo_type_t type, int flags,
+                                  int *n_busy_out)
+{
+  const or_options_t *options = get_options();
+  smartlist_t *direct, *tunnel;
+  smartlist_t *overloaded_direct, *overloaded_tunnel;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const routerstatus_t *result = NULL;
+  time_t now = time(NULL);
+  const int requireother = ! (flags & PDS_ALLOW_SELF);
+  const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
+  const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
+  const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  const double auth_weight =
+    (sourcelist == router_get_fallback_dir_servers()) ?
+    options->DirAuthorityFallbackRate : 1.0;
+  smartlist_t *pick_from;
+  int n_busy = 0;
+  int try_excluding = 1, n_excluded = 0;
+  int try_ip_pref = 1;
+
+  if (!sourcelist)
+    return NULL;
+
+ retry_search:
+
+  direct = smartlist_new();
+  tunnel = smartlist_new();
+  overloaded_direct = smartlist_new();
+  overloaded_tunnel = smartlist_new();
+
+  const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
+  const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
+  const int must_have_or = directory_must_use_begindir(options);
+
+  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
+    {
+      int is_overloaded =
+          d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
+      if (!d->is_running) continue;
+      if ((type & d->type) == 0)
+        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
+      if (requireother && me && router_digest_is_me(d->digest))
+        continue;
+      if (try_excluding &&
+          routerset_contains_routerstatus(options->ExcludeNodes,
+                                          &d->fake_status, -1)) {
+        ++n_excluded;
+        continue;
+      }
+
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
+        ++n_busy;
+        continue;
+      }
+
+      /* Clients use IPv6 addresses if the server has one and the client
+       * prefers IPv6.
+       * Add the router if its preferred address and port are reachable.
+       * If we don't get any routers, we'll try again with the non-preferred
+       * address for each router (if any). (To ensure correct load-balancing
+       * we try routers that only have one address both times.)
+       */
+      if (!fascistfirewall || skip_or_fw ||
+          fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION,
+                                             try_ip_pref))
+        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
+      else if (!must_have_or && (skip_dir_fw ||
+               fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION,
+                                                  try_ip_pref)))
+        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
+    }
+  SMARTLIST_FOREACH_END(d);
+
+  if (smartlist_len(tunnel)) {
+    pick_from = tunnel;
+  } else if (smartlist_len(overloaded_tunnel)) {
+    pick_from = overloaded_tunnel;
+  } else if (smartlist_len(direct)) {
+    pick_from = direct;
+  } else {
+    pick_from = overloaded_direct;
+  }
+
+  {
+    const dir_server_t *selection =
+      dirserver_choose_by_weight(pick_from, auth_weight);
+
+    if (selection)
+      result = &selection->fake_status;
+  }
+
+  smartlist_free(direct);
+  smartlist_free(tunnel);
+  smartlist_free(overloaded_direct);
+  smartlist_free(overloaded_tunnel);
+
+  RETRY_ALTERNATE_IP_VERSION(retry_search);
+
+  RETRY_WITHOUT_EXCLUDE(retry_search);
+
+  router_picked_poor_directory_log(result);
+
+  if (n_busy_out)
+    *n_busy_out = n_busy;
+  return result;
+}

src/feature/nodelist/node_select.h

@@ -0,0 +1,102 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file node_select.h
+ * \brief Header file for node_select.c
+ **/
+
+#ifndef TOR_NODE_SELECT_H
+#define TOR_NODE_SELECT_H
+
+/** Flags to be passed to control router_choose_random_node() to indicate what
+ * kind of nodes to pick according to what algorithm. */
+typedef enum router_crn_flags_t {
+  CRN_NEED_UPTIME = 1<<0,
+  CRN_NEED_CAPACITY = 1<<1,
+  CRN_NEED_GUARD = 1<<2,
+  /* XXXX not used, apparently. */
+  CRN_WEIGHT_AS_EXIT = 1<<5,
+  CRN_NEED_DESC = 1<<6,
+  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
+  CRN_PREF_ADDR = 1<<7,
+  /* On clients, only provide nodes that we can connect to directly, based on
+   * our firewall rules */
+  CRN_DIRECT_CONN = 1<<8,
+  /* On clients, only provide nodes with HSRend >= 2 protocol version which
+   * is required for hidden service version >= 3. */
+  CRN_RENDEZVOUS_V3 = 1<<9,
+} router_crn_flags_t;
+
+/** Possible ways to weight routers when choosing one randomly.  See
+ * routerlist_sl_choose_by_bandwidth() for more information.*/
+typedef enum bandwidth_weight_rule_t {
+  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
+  WEIGHT_FOR_DIR
+} bandwidth_weight_rule_t;
+
+/* Flags for pick_directory_server() and pick_trusteddirserver(). */
+/** Flag to indicate that we should not automatically be willing to use
+ * ourself to answer a directory request.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_ALLOW_SELF                 (1<<0)
+/** Flag to indicate that if no servers seem to be up, we should mark all
+ * directory servers as up and try again.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
+/** Flag to indicate that we should not exclude directory servers that
+ * our ReachableAddress settings would exclude.  This usually means that
+ * we're going to connect to the server over Tor, and so we don't need to
+ * worry about our firewall telling us we can't.
+ * Passed to router_pick_directory_server (et al).*/
+#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading server descriptors
+ * or extrainfo documents.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
+/** Flag to indicate that we should not use any directory authority to which
+ * we have an existing directory connection for downloading microdescs.
+ *
+ * Passed to router_pick_directory_server (et al)
+ */
+#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
+
+const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
+                                                   int flags);
+
+int router_get_my_share_of_directory_requests(double *v3_share_out);
+
+const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
+                                          bandwidth_weight_rule_t rule);
+double frac_nodes_with_descriptors(const smartlist_t *sl,
+                                   bandwidth_weight_rule_t rule,
+                                   int for_direct_conn);
+const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
+                                        struct routerset_t *excludedset,
+                                        router_crn_flags_t flags);
+
+const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
+                                                   int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+                                                     int flags);
+
+#ifdef NODE_SELECT_PRIVATE
+STATIC int choose_array_element_by_weight(const uint64_t *entries,
+                                          int n_entries);
+STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
+                                        const double *entries_in,
+                                        int n_entries,
+                                        uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+                                           dirinfo_type_t auth, int flags,
+                                           int *n_busy_out);
+STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
+                                          int serverdesc, int microdesc);
+#endif
+
+#endif

src/feature/nodelist/nodelist.c

@@ -47,6 +47,7 @@
 #include "app/config/config.h"
 #include "feature/control/control.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/process_descs.h"
 #include "feature/client/entrynodes.h"
 #include "feature/stats/geoip.h"
 #include "feature/hs/hs_common.h"
@@ -59,6 +60,8 @@
 #include "core/or/protover.h"
 #include "feature/rend/rendservice.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"

src/feature/nodelist/routerlist.h

@@ -42,108 +42,13 @@ typedef enum was_router_added_t {
   ROUTER_CERTS_EXPIRED = -8
 } was_router_added_t;
 
-/** Flags to be passed to control router_choose_random_node() to indicate what
- * kind of nodes to pick according to what algorithm. */
-typedef enum router_crn_flags_t {
-  CRN_NEED_UPTIME = 1<<0,
-  CRN_NEED_CAPACITY = 1<<1,
-  CRN_NEED_GUARD = 1<<2,
-  /* XXXX not used, apparently. */
-  CRN_WEIGHT_AS_EXIT = 1<<5,
-  CRN_NEED_DESC = 1<<6,
-  /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */
-  CRN_PREF_ADDR = 1<<7,
-  /* On clients, only provide nodes that we can connect to directly, based on
-   * our firewall rules */
-  CRN_DIRECT_CONN = 1<<8,
-  /* On clients, only provide nodes with HSRend >= 2 protocol version which
-   * is required for hidden service version >= 3. */
-  CRN_RENDEZVOUS_V3 = 1<<9,
-} router_crn_flags_t;
+/** How long do we avoid using a directory server after it's given us a 503? */
+#define DIR_503_TIMEOUT (60*60)
 
-/** Possible ways to weight routers when choosing one randomly.  See
- * routerlist_sl_choose_by_bandwidth() for more information.*/
-typedef enum bandwidth_weight_rule_t {
-  NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD,
-  WEIGHT_FOR_DIR
-} bandwidth_weight_rule_t;
-
-/* Flags for pick_directory_server() and pick_trusteddirserver(). */
-/** Flag to indicate that we should not automatically be willing to use
- * ourself to answer a directory request.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_ALLOW_SELF                 (1<<0)
-/** Flag to indicate that if no servers seem to be up, we should mark all
- * directory servers as up and try again.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_RETRY_IF_NO_SERVERS        (1<<1)
-/** Flag to indicate that we should not exclude directory servers that
- * our ReachableAddress settings would exclude.  This usually means that
- * we're going to connect to the server over Tor, and so we don't need to
- * worry about our firewall telling us we can't.
- * Passed to router_pick_directory_server (et al).*/
-#define PDS_IGNORE_FASCISTFIREWALL     (1<<2)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading server descriptors
- * or extrainfo documents.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3)
-/** Flag to indicate that we should not use any directory authority to which
- * we have an existing directory connection for downloading microdescs.
- *
- * Passed to router_pick_directory_server (et al)
- */
-#define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
-
-int get_n_authorities(dirinfo_type_t type);
-int trusted_dirs_reload_certs(void);
-
-/*
- * Pass one of these as source to trusted_dirs_load_certs_from_string()
- * to indicate whence string originates; this controls error handling
- * behavior such as marking downloads as failed.
- */
-
-#define TRUSTED_DIRS_CERTS_SRC_SELF 0
-#define TRUSTED_DIRS_CERTS_SRC_FROM_STORE 1
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST 2
-#define TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST 3
-#define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4
-
-int trusted_dirs_load_certs_from_string(const char *contents, int source,
-                                        int flush, const char *source_dir);
-void trusted_dirs_flush_certs_to_disk(void);
-authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest);
-authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest);
-authority_cert_t *authority_cert_get_by_digests(const char *id_digest,
-                                                const char *sk_digest);
-void authority_cert_get_all(smartlist_t *certs_out);
-void authority_cert_dl_failed(const char *id_digest,
-                              const char *signing_key_digest, int status);
-void authority_certs_fetch_missing(networkstatus_t *status, time_t now,
-                                   const char *dir_hint);
 int router_reload_router_list(void);
-int authority_cert_dl_looks_uncertain(const char *id_digest);
-const smartlist_t *router_get_trusted_dir_servers(void);
-const smartlist_t *router_get_fallback_dir_servers(void);
-int authority_cert_is_blacklisted(const authority_cert_t *cert);
 
-const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
-                                                   int flags);
-dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
-dir_server_t *router_get_fallback_dirserver_by_digest(
-                                                   const char *digest);
-int router_digest_is_fallback_dir(const char *digest);
-MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest,
-          (const char *d));
-const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
-                                                   int flags);
-const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
-                                                     int flags);
 int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
-int router_get_my_share_of_directory_requests(double *v3_share_out);
+int router_skip_dir_reachability(const or_options_t *options, int try_ip_pref);
 void router_reset_status_download_failures(void);
 int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
 void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
@@ -155,21 +60,6 @@ const routerinfo_t *routerlist_find_my_routerinfo(void);
 uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
 uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router);
 
-const node_t *node_sl_choose_by_bandwidth(const smartlist_t *sl,
-                                          bandwidth_weight_rule_t rule);
-double frac_nodes_with_descriptors(const smartlist_t *sl,
-                                   bandwidth_weight_rule_t rule,
-                                   int for_direct_conn);
-
-const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
-                                        struct routerset_t *excludedset,
-                                        router_crn_flags_t flags);
-
-int router_digest_is_trusted_dir_type(const char *digest,
-                                      dirinfo_type_t type);
-#define router_digest_is_trusted_dir(d) \
-  router_digest_is_trusted_dir_type((d), NO_DIRINFO)
-
 int hexdigest_to_digest(const char *hexdigest, char *digest);
 const routerinfo_t *router_get_by_id_digest(const char *digest);
 routerinfo_t *router_get_mutable_by_digest(const char *digest);
@@ -194,13 +84,10 @@ void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
 void routerlist_free_all(void);
 void routerlist_reset_warnings(void);
 
-MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void));
-MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id,
-          (const char *digest));
-MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id,
-          (const char *digest));
-MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
-    (const char *id_digest, const char *sk_digest));
+/* XXXX move this */
+void list_pending_downloads(digestmap_t *result,
+                            digest256map_t *result256,
+                            int purpose, const char *prefix);
 
 static int WRA_WAS_ADDED(was_router_added_t s);
 static int WRA_WAS_OUTDATED(was_router_added_t s);
@@ -269,21 +156,6 @@ void routerlist_retry_directory_downloads(time_t now);
 
 int router_exit_policy_rejects_all(const routerinfo_t *router);
 
-dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
-                       uint16_t dir_port, uint16_t or_port,
-                       const tor_addr_port_t *addrport_ipv6,
-                       const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type, double weight);
-dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
-                                      uint16_t dir_port, uint16_t or_port,
-                                      const tor_addr_port_t *addrport_ipv6,
-                                      const char *id_digest, double weight);
-void dir_server_add(dir_server_t *ent);
-
-void authority_cert_free_(authority_cert_t *cert);
-#define authority_cert_free(cert) \
-  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
-void clear_dir_servers(void);
 void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
                                                   networkstatus_t *consensus);
 void update_router_descriptor_downloads(time_t now);
@@ -321,16 +193,6 @@ int hex_digest_nickname_matches(const char *hexdigest,
                                 const char *nickname);
 
 #ifdef ROUTERLIST_PRIVATE
-STATIC int choose_array_element_by_weight(const uint64_t *entries,
-                                          int n_entries);
-STATIC void scale_array_elements_to_u64(uint64_t *entries_out,
-                                        const double *entries_in,
-                                        int n_entries,
-                                        uint64_t *total_out);
-STATIC const routerstatus_t *router_pick_directory_server_impl(
-                                           dirinfo_type_t auth, int flags,
-                                           int *n_busy_out);
-
 MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
                                                  int seconds));
 MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
@@ -339,8 +201,6 @@ MOCK_DECL(STATIC was_router_added_t, extrainfo_insert,
 MOCK_DECL(STATIC void, initiate_descriptor_downloads,
           (const routerstatus_t *source, int purpose, smartlist_t *digests,
            int lo, int hi, int pds_flags));
-STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
-                                          int serverdesc, int microdesc);
 
 #endif /* defined(ROUTERLIST_PRIVATE) */
 

src/feature/nodelist/routerparse.c

@@ -61,7 +61,6 @@
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/shared_random.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/client/entrynodes.h"
 #include "lib/memarea/memarea.h"
 #include "feature/nodelist/microdesc.h"
@@ -75,6 +74,7 @@
 #include "feature/relay/routerkeys.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
+#include "feature/nodelist/authcert.h"
 #include "lib/sandbox/sandbox.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/nodelist/torcert.h"

src/feature/relay/router.c

@@ -18,6 +18,7 @@
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "feature/dircache/directory.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/process_descs.h"
 #include "feature/relay/dns.h"
 #include "feature/stats/geoip.h"
 #include "feature/hibernate/hibernate.h"
@@ -30,6 +31,8 @@
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "app/config/statefile.h"

src/feature/rend/rendservice.c

@@ -33,7 +33,7 @@
 #include "core/or/relay.h"
 #include "feature/stats/rephist.h"
 #include "feature/hs_common/replaycache.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"
 #include "lib/encoding/confline.h"

src/test/test_config.c

@@ -38,6 +38,7 @@
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
 #include "app/config/statefile.h"

src/test/test_controller.c

@@ -10,7 +10,7 @@
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/rend/rendservice.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/authcert.h"
 #include "feature/nodelist/nodelist.h"
 #include "test/test.h"
 #include "test/test_helpers.h"

src/test/test_dir.c

@@ -6,16 +6,19 @@
 #include "orconfig.h"
 #include <math.h>
 
+#define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
 #define CONTROL_PRIVATE
 #define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
-#define ROUTER_PRIVATE
-#define ROUTERLIST_PRIVATE
-#define ROUTERPARSE_PRIVATE
 #define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
+#define NODE_SELECT_PRIVATE
 #define RELAY_PRIVATE
+#define ROUTERLIST_PRIVATE
+#define ROUTERPARSE_PRIVATE
+#define ROUTER_PRIVATE
+#define VOTEFLAGS_PRIVATE
 
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
@@ -28,8 +31,12 @@
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircache/directory.h"
+#include "feature/dirauth/bwauth.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/recommend_pkg.h"
+#include "feature/dirauth/voteflags.h"
 #include "feature/client/entrynodes.h"
 #include "feature/dircommon/fp_pair.h"
 #include "feature/hibernate/hibernate.h"
@@ -38,6 +45,9 @@
 #include "feature/nodelist/networkstatus.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/routerset.h"

src/test/test_dir_handle_get.c

@@ -19,6 +19,8 @@
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendcache.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerlist.h"
 #include "test/rend_test_helpers.h"
 #include "feature/nodelist/microdesc.h"

src/test/test_guardfraction.c

@@ -1,14 +1,14 @@
 /* Copyright (c) 2014-2018, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define DIRSERV_PRIVATE
+#define GUARDFRACTION_PRIVATE
 #define ROUTERPARSE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirauth/guardfraction.h"
 #include "feature/client/entrynodes.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/nodelist/networkstatus.h"
@@ -422,4 +422,3 @@ struct testcase_t guardfraction_tests[] = {
 
   END_OF_TESTCASES
 };
-

src/test/test_routerlist.c

@@ -12,6 +12,7 @@
 #define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define ROUTERLIST_PRIVATE
+#define NODE_SELECT_PRIVATE
 #define TOR_UNIT_TESTING
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -27,6 +28,8 @@
 #include "feature/nodelist/nodelist.h"
 #include "core/or/policies.h"
 #include "feature/relay/router.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/node_select.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
 #include "feature/nodelist/routerparse.h"

src/test/test_shared_random.c

@@ -17,7 +17,8 @@
 #include "feature/nodelist/networkstatus.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/routerparse.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dircommon/voting_schedule.h"