Merge branch 'feature12538_028_01_squashed'

changes/feature12538

@@ -0,0 +1,6 @@
+  o Minor features (directory system):
+    Previously only relays who explicitly opened a directory port (DirPort)
+    accepted directory requests from clients.  Now all relays, with and without
+    a DirPort, who do not disable the DirCache option accept and serve
+    directory requests sent (tunnelled) through their ORPort.
+    Closes ticket 12538.

doc/tor.1.txt

@@ -1987,6 +1987,12 @@ if DirPort is non-zero):
     except that port specifiers are ignored. Any address not matched by
     some entry in the policy is accepted.
 
+[[DirCache]] **DirCache** **0**|**1**::
+    When this option is set, Tor caches all current directory documents and
+    accepts client requests for them. Setting DirPort is not required for this,
+    because clients connect via the ORPort by default. Setting either DirPort
+    or BridgeRelay and setting DirCache to 0 is not supported.  (Default: 1)
+
 
 DIRECTORY AUTHORITY SERVER OPTIONS
 ----------------------------------

src/or/config.c

@@ -222,6 +222,7 @@ static config_var_t option_vars_[] = {
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
+  V(DirCache,                    BOOL,     "1"),
   V(DirAuthorityFallbackRate,    DOUBLE,   "1.0"),
   V(DisableAllSwap,              BOOL,     "0"),
   V(DisableDebuggerAttachment,   BOOL,     "1"),
@@ -3457,6 +3458,24 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("AccountingRule must be 'sum' or 'max'");
   }
 
+  if (options->DirPort_set && !options->DirCache) {
+    REJECT("DirPort configured but DirCache disabled. DirPort requires "
+           "DirCache.");
+  }
+
+  if (options->BridgeRelay && !options->DirCache) {
+    REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires "
+           "DirCache.");
+  }
+
+  if (server_mode(options)) {
+    char *msg = NULL;
+    if (have_enough_mem_for_dircache(options, 0, &msg)) {
+      log_warn(LD_CONFIG, "%s", msg);
+      tor_free(msg);
+    }
+  }
+
   if (options->HTTPProxy) { /* parse it now */
     if (tor_addr_port_lookup(options->HTTPProxy,
                         &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0)
@@ -4065,6 +4084,48 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
   }
 }
 
+/* If we have less than 300 MB suggest disabling dircache */
+#define DIRCACHE_MIN_MB_BANDWIDTH 300
+#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE)
+#define STRINGIFY(val) #val
+
+/** Create a warning message for emitting if we are a dircache but may not have
+ * enough system memory, or if we are not a dircache but probably should be.
+ * Return -1 when a message is returned in *msg*, else return 0. */
+STATIC int
+have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
+                             char **msg)
+{
+  *msg = NULL;
+  if (total_mem == 0) {
+    if (get_total_system_memory(&total_mem) < 0)
+      total_mem = options->MaxMemInQueues;
+  }
+  if (options->DirCache) {
+    if (total_mem < DIRCACHE_MIN_BANDWIDTH) {
+      if (options->BridgeRelay) {
+        *msg = strdup("Running a Bridge with less than "
+                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
+                      "not recommended.");
+      } else {
+        *msg = strdup("Being a directory cache (default) with less than "
+                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
+                      "not recommended and may consume most of the available "
+                      "resources, consider disabling this functionality by "
+                      "setting the DirCache option to 0.");
+      }
+    }
+  } else {
+    if (total_mem >= DIRCACHE_MIN_BANDWIDTH) {
+      *msg = strdup("DirCache is disabled and we are configured as a "
+               "relay. This may disqualify us from becoming a guard in the "
+               "future.");
+    }
+  }
+  return *msg == NULL ? 0 : -1;
+}
+#undef STRINGIFY
+
 /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL
  * equal strings. */
 static int
@@ -4253,7 +4314,8 @@ options_transition_affects_descriptor(const or_options_t *old_options,
       !opt_streq(old_options->MyFamily, new_options->MyFamily) ||
       !opt_streq(old_options->AccountingStart, new_options->AccountingStart) ||
       old_options->AccountingMax != new_options->AccountingMax ||
-      public_server_mode(old_options) != public_server_mode(new_options))
+      public_server_mode(old_options) != public_server_mode(new_options) ||
+      old_options->DirCache != new_options->DirCache)
     return 1;
 
   return 0;

src/or/config.h

@@ -158,6 +158,8 @@ STATIC int parse_dir_authority_line(const char *line,
                                     dirinfo_type_t required_type,
                                     int validate_only);
 STATIC int parse_dir_fallback_line(const char *line, int validate_only);
+STATIC int have_enough_mem_for_dircache(const or_options_t *options,
+                                        size_t total_mem, char **msg);
 #endif
 
 #endif

src/or/directory.c

@@ -943,6 +943,15 @@ directory_initiate_command_rend(const tor_addr_t *_addr,
   log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
             anonymized_connection, use_begindir);
 
+  if (!dir_port && !use_begindir) {
+    char ipaddr[TOR_ADDR_BUF_LEN];
+    tor_addr_to_str(ipaddr, _addr, TOR_ADDR_BUF_LEN, 0);
+    log_warn(LD_BUG, "Cannot use directory server without dirport or "
+                     "begindir! Address: %s, ORPort: %d, DirPort: %d",
+                     escaped_safe_str_client(ipaddr), or_port, dir_port);
+    return;
+  }
+
   log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
 
 #ifndef NON_ANONYMOUS_MODE_ENABLED
@@ -3664,8 +3673,7 @@ connection_dir_finished_connecting(dir_connection_t *conn)
 static const smartlist_t *
 find_dl_schedule(download_status_t *dls, const or_options_t *options)
 {
-  /* XX/teor Replace with dir_server_mode from #12538 */
-  const int dir_server = options->DirPort_set;
+  const int dir_server = dir_server_mode(options);
   const int multi_d = networkstatus_consensus_can_use_multiple_directories(
                                                                     options);
   const int we_are_bootstrapping = networkstatus_consensus_is_boostrapping(

src/or/directory.h

@@ -101,7 +101,8 @@ time_t download_status_increment_attempt(download_status_t *dls,
  * the optional status code <b>sc</b>. */
 #define download_status_failed(dls, sc)                                 \
   download_status_increment_failure((dls), (sc), NULL,                  \
-                                    get_options()->DirPort_set, time(NULL))
+                                    dir_server_mode(get_options()), \
+                                    time(NULL))
 
 void download_status_reset(download_status_t *dls);
 static int download_status_is_ready(download_status_t *dls, time_t now,

src/or/dirserv.c

@@ -1091,13 +1091,13 @@ directory_fetches_from_authorities(const or_options_t *options)
     return 1; /* we don't know our IP address; ask an authority. */
   refuseunknown = ! router_my_exit_policy_is_reject_star() &&
     should_refuse_unknown_exits(options);
-  if (!options->DirPort_set && !refuseunknown)
+  if (!dir_server_mode(options) && !refuseunknown)
     return 0;
   if (!server_mode(options) || !advertised_server_mode())
     return 0;
   me = router_get_my_routerinfo();
-  if (!me || (!me->dir_port && !refuseunknown))
-    return 0; /* if dirport not advertised, return 0 too */
+  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
+    return 0; /* if we don't service directory requests, return 0 too */
   return 1;
 }
 
@@ -1128,7 +1128,7 @@ directory_fetches_dir_info_later(const or_options_t *options)
 int
 directory_caches_unknown_auth_certs(const or_options_t *options)
 {
-  return options->DirPort_set || options->BridgeRelay;
+  return dir_server_mode(options) || options->BridgeRelay;
 }
 
 /** Return 1 if we want to keep descriptors, networkstatuses, etc around
@@ -1137,7 +1137,7 @@ directory_caches_unknown_auth_certs(const or_options_t *options)
 int
 directory_caches_dir_info(const or_options_t *options)
 {
-  if (options->BridgeRelay || options->DirPort_set)
+  if (options->BridgeRelay || dir_server_mode(options))
     return 1;
   if (!server_mode(options) || !advertised_server_mode())
     return 0;
@@ -1153,7 +1153,7 @@ directory_caches_dir_info(const or_options_t *options)
 int
 directory_permits_begindir_requests(const or_options_t *options)
 {
-  return options->BridgeRelay != 0 || options->DirPort_set;
+  return options->BridgeRelay != 0 || dir_server_mode(options);
 }
 
 /** Return 1 if we have no need to fetch new descriptors. This generally
@@ -1350,8 +1350,9 @@ dirserv_thinks_router_is_unreliable(time_t now,
 }
 
 /** Return true iff <b>router</b> should be assigned the "HSDir" flag.
+ *
  * Right now this means it advertises support for it, it has a high uptime,
- * it has a DirPort open, it has the Stable and Fast flag and it's currently
+ * it's a directory cache, it has the Stable and Fast flags, and it's currently
  * considered Running.
  *
  * This function needs to be called after router-\>is_running has
@@ -1378,7 +1379,8 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
   else
     uptime = real_uptime(router, now);
 
-  return (router->wants_to_be_hs_dir && router->dir_port &&
+  return (router->wants_to_be_hs_dir &&
+          router->supports_tunnelled_dir_requests &&
           node->is_stable && node->is_fast &&
           uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
           router_is_active(router, node, now));
@@ -1921,7 +1923,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    rs->is_hs_dir?" HSDir":"",
                    rs->is_flagged_running?" Running":"",
                    rs->is_stable?" Stable":"",
-                   (rs->dir_port!=0)?" V2Dir":"",
+                   rs->is_v2_dir?" V2Dir":"",
                    rs->is_valid?" Valid":"");
 
   /* length of "opt v \n" */
@@ -2185,6 +2187,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
   strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
   rs->or_port = ri->or_port;
   rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
   if (options->AuthDirHasIPv6Connectivity == 1 &&
       !tor_addr_is_null(&ri->ipv6_addr) &&
       node->last_reachable6 >= now - REACHABLE_TIMEOUT) {

src/or/dirvote.c

@@ -54,7 +54,6 @@ static int dirvote_perform_vote(void);
 static void dirvote_clear_votes(int all_votes);
 static int dirvote_compute_consensuses(void);
 static int dirvote_publish_consensus(void);
-static char *make_consensus_method_list(int low, int high, const char *sep);
 
 /* =====
  * Voting
@@ -564,7 +563,7 @@ consensus_method_is_supported(int method)
 
 /** Return a newly allocated string holding the numbers between low and high
  * (inclusive) that are supported consensus methods. */
-static char *
+STATIC char *
 make_consensus_method_list(int low, int high, const char *separator)
 {
   char *list;

src/or/dirvote.h

@@ -177,6 +177,7 @@ STATIC char *format_networkstatus_vote(crypto_pk_t *private_key,
 STATIC char *dirvote_compute_params(smartlist_t *votes, int method,
                              int total_authorities);
 STATIC char *compute_consensus_package_lines(smartlist_t *votes);
+STATIC char *make_consensus_method_list(int low, int high, const char *sep);
 #endif
 
 #endif

src/or/networkstatus.c

@@ -1461,6 +1461,38 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c,
   } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
 }
 
+#ifdef TOR_UNIT_TESTS
+/**Accept a <b>flavor</b> consensus <b>c</b> without any additional
+ * validation. This is exclusively for unit tests.
+ * We copy any ancillary information from a pre-existing consensus
+ * and then free the current one and replace it with the newly
+ * provided instance. Returns -1 on unrecognized flavor, 0 otherwise.
+ */
+int
+networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                            const char *flavor)
+{
+  int flav = networkstatus_parse_flavor_name(flavor);
+  switch (flav) {
+    case FLAV_NS:
+      if (current_ns_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_ns_consensus);
+        networkstatus_vote_free(current_ns_consensus);
+      }
+      current_ns_consensus = c;
+      break;
+    case FLAV_MICRODESC:
+      if (current_md_consensus) {
+        networkstatus_copy_old_consensus_info(c, current_md_consensus);
+        networkstatus_vote_free(current_md_consensus);
+      }
+      current_md_consensus = c;
+      break;
+  }
+  return current_md_consensus ? 0 : -1;
+}
+#endif //TOR_UNIT_TESTS
+
 /** Try to replace the current cached v3 networkstatus with the one in
  * <b>consensus</b>.  If we don't have enough certificates to validate it,
  * store it in consensus_waiting_for_certs and launch a certificate fetch.

src/or/networkstatus.h

@@ -114,6 +114,10 @@ int networkstatus_get_weight_scale_param(networkstatus_t *ns);
 
 #ifdef NETWORKSTATUS_PRIVATE
 STATIC void vote_routerstatus_free(vote_routerstatus_t *rs);
+#ifdef TOR_UNIT_TESTS
+STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
+                                                const char *flavor);
+#endif // TOR_UNIT_TESTS
 #endif
 
 #endif

src/or/nodelist.c

@@ -644,12 +644,19 @@ node_is_named(const node_t *node)
 int
 node_is_dir(const node_t *node)
 {
-  if (node->rs)
-    return node->rs->dir_port != 0;
-  else if (node->ri)
-    return node->ri->dir_port != 0;
-  else
+  if (node->rs) {
+    routerstatus_t * rs = node->rs;
+    /* This is true if supports_tunnelled_dir_requests is true which
+     * indicates that we support directory request tunnelled or through the
+     * DirPort. */
+    return rs->is_v2_dir;
+  } else if (node->ri) {
+    routerinfo_t * ri = node->ri;
+    /* Both tunnelled request is supported or DirPort is set. */
+    return ri->supports_tunnelled_dir_requests;
+  } else {
     return 0;
+  }
 }
 
 /** Return true iff <b>node</b> has either kind of usable descriptor -- that

src/or/or.h

@@ -2147,6 +2147,11 @@ typedef struct {
    * tests for it. */
   unsigned int needs_retest_if_added:1;
 
+  /** True iff this router included "tunnelled-dir-server" in its descriptor,
+   * implying it accepts tunnelled directory requests, or it advertised
+   * dir_port > 0. */
+  unsigned int supports_tunnelled_dir_requests:1;
+
 /** Tor can use this router for general positions in circuits; we got it
  * from a directory server as usual, or we're an authority and a server
  * uploaded it. */
@@ -2224,6 +2229,9 @@ typedef struct routerstatus_t {
                                * an exit node. */
   unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden
                              * service directory. */
+  unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
+                             * or it claims to accept tunnelled dir requests.
+                             */
   /** True iff we know version info for this router. (i.e., a "v" entry was
    * included.)  We'll replace all these with a big tor_version_t or a char[]
    * if the number of traits we care about ever becomes incredibly big. */
@@ -3961,6 +3969,10 @@ typedef struct {
   /** Should we fetch our dir info at the start of the consensus period? */
   int FetchDirInfoExtraEarly;
 
+  int DirCache; /**< Cache all directory documents and accept requests via
+                 * tunnelled dir conns from clients. If 1, enabled (default);
+                 * If 0, disabled. */
+
   char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual
                                  * MAPADDRESS requests for IPv4 addresses */
   char *VirtualAddrNetworkIPv6; /**< Address and mask to hand out for virtual

src/or/router.c

@@ -1099,39 +1099,40 @@ check_whether_dirport_reachable(void)
          can_reach_dir_port;
 }
 
-/** Look at a variety of factors, and return 0 if we don't want to
- * advertise the fact that we have a DirPort open. Else return the
- * DirPort we want to advertise.
- *
- * Log a helpful message if we change our mind about whether to publish
- * a DirPort.
+/** The lower threshold of remaining bandwidth required to advertise (or
+ * automatically provide) directory services */
+/* XXX Should this be increased? */
+#define MIN_BW_TO_ADVERTISE_DIRSERVER 51200
+
+/** Return true iff we have enough configured bandwidth to cache directory
+ * information. */
+static int
+router_has_bandwidth_to_be_dirserver(const or_options_t *options)
+{
+  if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+    return 0;
+  }
+  if (options->RelayBandwidthRate > 0 &&
+      options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+    return 0;
+  }
+  return 1;
+}
+
+/** Helper: Return 1 if we have sufficient resources for serving directory
+ * requests, return 0 otherwise.
+ * dir_port is either 0 or the configured DirPort number.
+ * If AccountingMax is set less than our advertised bandwidth, then don't
+ * serve requests. Likewise, if our advertised bandwidth is less than
+ * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests.
  */
 static int
-decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+router_should_be_directory_server(const or_options_t *options, int dir_port)
 {
   static int advertising=1; /* start out assuming we will advertise */
   int new_choice=1;
   const char *reason = NULL;
 
-  /* Section one: reasons to publish or not publish that aren't
-   * worth mentioning to the user, either because they're obvious
-   * or because they're normal behavior. */
-
-  if (!dir_port) /* short circuit the rest of the function */
-    return 0;
-  if (authdir_mode(options)) /* always publish */
-    return dir_port;
-  if (net_is_disabled())
-    return 0;
-  if (!check_whether_dirport_reachable())
-    return 0;
-  if (!router_get_advertised_dir_port(options, dir_port))
-    return 0;
-
-  /* Section two: reasons to publish or not publish that the user
-   * might find surprising. These are generally config options that
-   * make us choose not to publish. */
-
   if (accounting_is_enabled(options)) {
     /* Don't spend bytes for directory traffic if we could end up hibernating,
      * but allow DirPort otherwise. Some people set AccountingMax because
@@ -1158,10 +1159,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
       new_choice = 0;
       reason = "AccountingMax enabled";
     }
-#define MIN_BW_TO_ADVERTISE_DIRPORT 51200
-  } else if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRPORT ||
-             (options->RelayBandwidthRate > 0 &&
-              options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRPORT)) {
+  } else if (! router_has_bandwidth_to_be_dirserver(options)) {
     /* if we're advertising a small amount */
     new_choice = 0;
     reason = "BandwidthRate under 50KB";
@@ -1169,15 +1167,63 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
 
   if (advertising != new_choice) {
     if (new_choice == 1) {
-      log_notice(LD_DIR, "Advertising DirPort as %d", dir_port);
+      if (dir_port > 0)
+        log_notice(LD_DIR, "Advertising DirPort as %d", dir_port);
+      else
+        log_notice(LD_DIR, "Advertising directory service support");
     } else {
       tor_assert(reason);
-      log_notice(LD_DIR, "Not advertising DirPort (Reason: %s)", reason);
+      log_notice(LD_DIR, "Not advertising Dir%s (Reason: %s)",
+                 dir_port ? "Port" : "ectory Service support", reason);
     }
     advertising = new_choice;
   }
 
-  return advertising ? dir_port : 0;
+  return advertising;
+}
+
+/** Return 1 if we are configured to accept either relay or directory requests
+ * from clients and we aren't at risk of exceeding our bandwidth limits, thus
+ * we should be a directory server. If not, return 0.
+ */
+int
+dir_server_mode(const or_options_t *options)
+{
+  if (!options->DirCache)
+    return 0;
+  return options->DirPort_set ||
+    (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
+}
+
+/** Look at a variety of factors, and return 0 if we don't want to
+ * advertise the fact that we have a DirPort open, else return the
+ * DirPort we want to advertise.
+ *
+ * Log a helpful message if we change our mind about whether to publish
+ * a DirPort.
+ */
+static int
+decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+{
+  /* Part one: reasons to publish or not publish that aren't
+   * worth mentioning to the user, either because they're obvious
+   * or because they're normal behavior. */
+
+  if (!dir_port) /* short circuit the rest of the function */
+    return 0;
+  if (authdir_mode(options)) /* always publish */
+    return dir_port;
+  if (net_is_disabled())
+    return 0;
+  if (!check_whether_dirport_reachable())
+    return 0;
+  if (!router_get_advertised_dir_port(options, dir_port))
+    return 0;
+
+  /* Part two: reasons to publish or not publish that the user
+   * might find surprising. router_should_be_directory_server()
+   * considers config options that make us choose not to publish. */
+  return router_should_be_directory_server(options, dir_port) ? dir_port : 0;
 }
 
 /** Allocate and return a new extend_info_t that can be used to build
@@ -1866,6 +1912,8 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->addr = addr;
   ri->or_port = router_get_advertised_or_port(options);
   ri->dir_port = router_get_advertised_dir_port(options, 0);
+  ri->supports_tunnelled_dir_requests = dir_server_mode(options) &&
+    router_should_be_directory_server(options, ri->dir_port);
   ri->cache_info.published_on = time(NULL);
   ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from
                                                         * main thread */
@@ -2642,6 +2690,10 @@ router_dump_router_to_string(routerinfo_t *router,
     tor_free(p6);
   }
 
+  if (router->supports_tunnelled_dir_requests) {
+    smartlist_add(chunks, tor_strdup("tunnelled-dir-server\n"));
+  }
+
   /* Sign the descriptor with Ed25519 */
   if (emit_ed_sigs)  {
     smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));

src/or/router.h

@@ -41,6 +41,7 @@ int init_keys_client(void);
 
 int check_whether_orport_reachable(void);
 int check_whether_dirport_reachable(void);
+int dir_server_mode(const or_options_t *options);
 void consider_testing_reachability(int test_or, int test_dir);
 void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);

src/or/routerlist.c

@@ -67,8 +67,6 @@ typedef struct cert_list_t cert_list_t;
 static int compute_weighted_bandwidths(const smartlist_t *sl,
                                        bandwidth_weight_rule_t rule,
                                        u64_dbl_t **bandwidths_out);
-static const routerstatus_t *router_pick_directory_server_impl(
-                              dirinfo_type_t auth, int flags, int *n_busy_out);
 static const routerstatus_t *router_pick_trusteddirserver_impl(
                 const smartlist_t *sourcelist, dirinfo_type_t auth,
                 int flags, int *n_busy_out);
@@ -1472,7 +1470,7 @@ router_pick_dirserver_generic(smartlist_t *sourcelist,
  * directories that we excluded for no other reason than
  * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
  */
-static const routerstatus_t *
+STATIC const routerstatus_t *
 router_pick_directory_server_impl(dirinfo_type_t type, int flags,
                                   int *n_busy_out)
 {
@@ -1512,7 +1510,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
     if (!status)
       continue;
 
-    if (!node->is_running || !status->dir_port || !node->is_valid)
+    if (!node->is_running || !node_is_dir(node) || !node->is_valid)
       continue;
     if (requireother && router_digest_is_me(node->identity))
       continue;
@@ -3238,7 +3236,11 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd)
   return ri;
 }
 
-/** Free all memory held by the routerlist module. */
+/** Free all memory held by the routerlist module.
+ * Note: Calling routerlist_free_all() should always be paired with
+ * a call to nodelist_free_all(). These should only be called during
+ * cleanup.
+ */
 void
 routerlist_free_all(void)
 {
@@ -4902,7 +4904,9 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
       (r1->contact_info && r2->contact_info &&
        strcasecmp(r1->contact_info, r2->contact_info)) ||
       r1->is_hibernating != r2->is_hibernating ||
-      cmp_addr_policies(r1->exit_policy, r2->exit_policy))
+      cmp_addr_policies(r1->exit_policy, r2->exit_policy) ||
+      (r1->supports_tunnelled_dir_requests !=
+       r2->supports_tunnelled_dir_requests))
     return 0;
   if ((r1->declared_family == NULL) != (r2->declared_family == NULL))
     return 0;

src/or/routerlist.h

@@ -233,6 +233,9 @@ STATIC int choose_array_element_by_weight(const u64_dbl_t *entries,
                                           int n_entries);
 STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries,
                                         uint64_t *total_out);
+STATIC const routerstatus_t *router_pick_directory_server_impl(
+                                           dirinfo_type_t auth, int flags,
+                                           int *n_busy_out);
 
 MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router,
                                                  int seconds));

src/or/routerparse.c

@@ -35,8 +35,9 @@
 /****************************************************************************/
 
 /** Enumeration of possible token types.  The ones starting with K_ correspond
- * to directory 'keywords'. ERR_ is an error in the tokenizing process, EOF_
- * is an end-of-file marker, and NIL_ is used to encode not-a-token.
+ * to directory 'keywords'. A_ is for an annotation, R or C is related to
+ * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+ * end-of-file marker, and NIL_ is used to encode not-a-token.
  */
 typedef enum {
   K_ACCEPT = 0,
@@ -125,6 +126,7 @@ typedef enum {
   K_DIR_KEY_CERTIFICATION,
   K_DIR_KEY_CROSSCERT,
   K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
 
   K_VOTE_STATUS,
   K_VALID_AFTER,
@@ -318,6 +320,7 @@ static token_rule_t routerdesc_token_table[] = {
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   T1( "bandwidth",           K_BANDWIDTH,           GE(3),   NO_OBJ ),
   A01("@purpose",            A_PURPOSE,             GE(1),   NO_OBJ ),
+  T01("tunnelled-dir-server",K_DIR_TUNNELLED,       NO_ARGS, NO_OBJ ),
 
   END_OF_TABLE
 };
@@ -1609,6 +1612,12 @@ router_parse_entry_from_string(const char *s, const char *end,
     router->wants_to_be_hs_dir = 1;
   }
 
+  /* This router accepts tunnelled directory requests via begindir if it has
+   * an open dirport or it included "tunnelled-dir-server". */
+  if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) {
+    router->supports_tunnelled_dir_requests = 1;
+  }
+
   tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
   note_crypto_pk_op(VERIFY_RTR);
 #ifdef COUNT_DISTINCT_DIGESTS
@@ -2294,6 +2303,8 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         rs->is_unnamed = 1;
       } else if (!strcmp(tok->args[i], "HSDir")) {
         rs->is_hs_dir = 1;
+      } else if (!strcmp(tok->args[i], "V2Dir")) {
+        rs->is_v2_dir = 1;
       }
     }
   }

src/test/include.am

@@ -79,6 +79,7 @@ src_test_test_SOURCES = \
 	src/test/test_crypto.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
+	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \

src/test/test_dir.c

@@ -26,6 +26,7 @@
 #include "routerparse.h"
 #include "routerset.h"
 #include "test.h"
+#include "test_dir_common.h"
 #include "torcert.h"
 
 static void
@@ -110,6 +111,7 @@ test_dir_formats(void *arg)
   r1->cache_info.published_on = 0;
   r1->or_port = 9000;
   r1->dir_port = 9003;
+  r1->supports_tunnelled_dir_requests = 1;
   tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
   r1->ipv6_orport = 9999;
   r1->onion_pkey = crypto_pk_dup_key(pk1);
@@ -154,6 +156,7 @@ test_dir_formats(void *arg)
   r2->cache_info.published_on = 5;
   r2->or_port = 9005;
   r2->dir_port = 0;
+  r2->supports_tunnelled_dir_requests = 1;
   r2->onion_pkey = crypto_pk_dup_key(pk2);
   curve25519_keypair_t r2_onion_keypair;
   curve25519_keypair_generate(&r2_onion_keypair, 0);
@@ -174,7 +177,9 @@ test_dir_formats(void *arg)
   /* XXXX025 router_dump_to_string should really take this from ri.*/
   options->ContactInfo = tor_strdup("Magri White "
                                     "<magri@elsewhere.example.com>");
+
   buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
+
   tor_free(options->ContactInfo);
   tt_assert(buf);
 
@@ -200,7 +205,8 @@ test_dir_formats(void *arg)
   strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
   strlcat(buf2, "contact Magri White <magri@elsewhere.example.com>\n",
           sizeof(buf2));
-  strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2));
+  strlcat(buf2, "reject *:*\n", sizeof(buf2));
+  strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2));
   buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
                              * twice */
 
@@ -214,12 +220,13 @@ test_dir_formats(void *arg)
   tt_assert(rp1);
   tt_int_op(rp1->addr,OP_EQ, r1->addr);
   tt_int_op(rp1->or_port,OP_EQ, r1->or_port);
-  //test_eq(rp1->dir_port, r1->dir_port);
+  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port);
   tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
   tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
   tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
   tt_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0);
   tt_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0);
+  tt_assert(rp1->supports_tunnelled_dir_requests);
   //tt_assert(rp1->exit_policy == NULL);
   tor_free(buf);
 
@@ -290,6 +297,7 @@ test_dir_formats(void *arg)
                 BASE64_ENCODE_MULTILINE);
   strlcat(buf2, cert_buf, sizeof(buf2));
   strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2));
+  strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2));
   strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2));
 
   buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2);
@@ -301,6 +309,8 @@ test_dir_formats(void *arg)
   tor_free(buf);
 
   buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL);
+
+  /* Reset for later */
   cp = buf;
   rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
   tt_assert(rp2);
@@ -315,6 +325,7 @@ test_dir_formats(void *arg)
              CURVE25519_PUBKEY_LEN);
   tt_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0);
   tt_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0);
+  tt_assert(rp2->supports_tunnelled_dir_requests);
 
   tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
 
@@ -1477,13 +1488,6 @@ test_dir_param_voting(void *arg)
   return;
 }
 
-extern const char AUTHORITY_CERT_1[];
-extern const char AUTHORITY_SIGNKEY_1[];
-extern const char AUTHORITY_CERT_2[];
-extern const char AUTHORITY_SIGNKEY_2[];
-extern const char AUTHORITY_CERT_3[];
-extern const char AUTHORITY_SIGNKEY_3[];
-
 /** Helper: Test that two networkstatus_voter_info_t do in fact represent the
  * same voting authority, and that they do in fact have all the same
  * information. */
@@ -1503,42 +1507,6 @@ test_same_voter(networkstatus_voter_info_t *v1,
   ;
 }
 
-/** Helper: Make a new routerinfo containing the right information for a
- * given vote_routerstatus_t. */
-static routerinfo_t *
-generate_ri_from_rs(const vote_routerstatus_t *vrs)
-{
-  routerinfo_t *r;
-  const routerstatus_t *rs = &vrs->status;
-  static time_t published = 0;
-
-  r = tor_malloc_zero(sizeof(routerinfo_t));
-  r->cert_expiration_time = TIME_MAX;
-  memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN);
-  memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest,
-         DIGEST_LEN);
-  r->cache_info.do_not_cache = 1;
-  r->cache_info.routerlist_index = -1;
-  r->cache_info.signed_descriptor_body =
-    tor_strdup("123456789012345678901234567890123");
-  r->cache_info.signed_descriptor_len =
-    strlen(r->cache_info.signed_descriptor_body);
-  r->exit_policy = smartlist_new();
-  r->cache_info.published_on = ++published + time(NULL);
-  if (rs->has_bandwidth) {
-    /*
-     * Multiply by 1000 because the routerinfo_t and the routerstatus_t
-     * seem to use different units (*sigh*) and because we seem stuck on
-     * icky and perverse decimal kilobytes (*double sigh*) - see
-     * router_get_advertised_bandwidth_capped() of routerlist.c and
-     * routerstatus_format_entry() of dirserv.c.
-     */
-    r->bandwidthrate = rs->bandwidth_kb * 1000;
-    r->bandwidthcapacity = rs->bandwidth_kb * 1000;
-  }
-  return r;
-}
-
 /** Helper: get a detached signatures document for one or two
  * consensuses. */
 static char *
@@ -1556,100 +1524,6 @@ get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2)
   return r;
 }
 
-/**
- * Generate a routerstatus for v3_networkstatus test
- */
-static vote_routerstatus_t *
-gen_routerstatus_for_v3ns(int idx, time_t now)
-{
-  vote_routerstatus_t *vrs=NULL;
-  routerstatus_t *rs;
-  tor_addr_t addr_ipv6;
-
-  switch (idx) {
-    case 0:
-      /* Generate the first routerstatus. */
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      vrs->version = tor_strdup("0.1.2.14");
-      rs->published_on = now-1500;
-      strlcpy(rs->nickname, "router2", sizeof(rs->nickname));
-      memset(rs->identity_digest, 3, DIGEST_LEN);
-      memset(rs->descriptor_digest, 78, DIGEST_LEN);
-      rs->addr = 0x99008801;
-      rs->or_port = 443;
-      rs->dir_port = 8000;
-      /* all flags but running cleared */
-      rs->is_flagged_running = 1;
-      break;
-    case 1:
-      /* Generate the second routerstatus. */
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      vrs->version = tor_strdup("0.2.0.5");
-      rs->published_on = now-1000;
-      strlcpy(rs->nickname, "router1", sizeof(rs->nickname));
-      memset(rs->identity_digest, 5, DIGEST_LEN);
-      memset(rs->descriptor_digest, 77, DIGEST_LEN);
-      rs->addr = 0x99009901;
-      rs->or_port = 443;
-      rs->dir_port = 0;
-      tor_addr_parse(&addr_ipv6, "[1:2:3::4]");
-      tor_addr_copy(&rs->ipv6_addr, &addr_ipv6);
-      rs->ipv6_orport = 4711;
-      rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running =
-        rs->is_valid = rs->is_possible_guard = 1;
-      break;
-    case 2:
-      /* Generate the third routerstatus. */
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      vrs->version = tor_strdup("0.1.0.3");
-      rs->published_on = now-1000;
-      strlcpy(rs->nickname, "router3", sizeof(rs->nickname));
-      memset(rs->identity_digest, 33, DIGEST_LEN);
-      memset(rs->descriptor_digest, 79, DIGEST_LEN);
-      rs->addr = 0xAA009901;
-      rs->or_port = 400;
-      rs->dir_port = 9999;
-      rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
-        rs->is_flagged_running = rs->is_valid =
-        rs->is_possible_guard = 1;
-      break;
-    case 3:
-      /* Generate a fourth routerstatus that is not running. */
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      vrs->version = tor_strdup("0.1.6.3");
-      rs->published_on = now-1000;
-      strlcpy(rs->nickname, "router4", sizeof(rs->nickname));
-      memset(rs->identity_digest, 34, DIGEST_LEN);
-      memset(rs->descriptor_digest, 47, DIGEST_LEN);
-      rs->addr = 0xC0000203;
-      rs->or_port = 500;
-      rs->dir_port = 1999;
-      /* Running flag (and others) cleared */
-      break;
-    case 4:
-      /* No more for this test; return NULL */
-      vrs = NULL;
-      break;
-    default:
-      /* Shouldn't happen */
-      tt_assert(0);
-  }
-  if (vrs) {
-    vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
-    tor_asprintf(&vrs->microdesc->microdesc_hash_line,
-                 "m 9,10,11,12,13,14,15,16,17 "
-                 "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n",
-                 idx);
-  }
-
- done:
-  return vrs;
-}
-
 /** Apply tweaks to the vote list for each voter */
 static int
 vote_tweaks_for_v3ns(networkstatus_t *v, int voter, time_t now)
@@ -1681,7 +1555,7 @@ vote_tweaks_for_v3ns(networkstatus_t *v, int voter, time_t now)
       vrs = smartlist_get(v->routerstatus_list, 0);
       memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN);
       tt_assert(router_add_to_routerlist(
-                  generate_ri_from_rs(vrs), &msg,0,0) >= 0);
+                  dir_common_generate_ri_from_rs(vrs), &msg,0,0) >= 0);
     }
   }
 
@@ -1746,11 +1620,11 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now)
     tt_assert(tor_addr_eq(&rs->ipv6_addr, &addr_ipv6));
     tt_int_op(rs->ipv6_orport,OP_EQ, 4711);
     if (voter == 1) {
-      /* all except "authority" (1) and "v2dir" (64) */
-      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(190));
+      /* all except "authority" (1) */
+      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(254));
     } else {
-      /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) - v2dir(256) */
-      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(718));
+      /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */
+      tt_u64_op(vrs->flags, OP_EQ, U64_LITERAL(974));
     }
   } else if (tor_memeq(rs->identity_digest,
                        "\x33\x33\x33\x33\x33\x33\x33\x33\x33\x33"
@@ -1820,6 +1694,7 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now)
     tt_assert(rs->is_flagged_running);
     tt_assert(!rs->is_valid);
     tt_assert(!rs->is_named);
+    tt_assert(rs->is_v2_dir);
     /* XXXX check version */
   } else if (tor_memeq(rs->identity_digest,
                        "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5"
@@ -1845,6 +1720,7 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now)
     tt_assert(rs->is_stable);
     tt_assert(rs->is_flagged_running);
     tt_assert(rs->is_valid);
+    tt_assert(rs->is_v2_dir);
     tt_assert(!rs->is_named);
     /* XXXX check version */
   } else {
@@ -1869,7 +1745,6 @@ test_a_networkstatus(
   authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL;
   crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL;
   crypto_pk_t *sign_skey_leg1=NULL;
-  const char *msg=NULL;
   /*
    * Sum the non-zero returns from vote_tweaks() we've seen; if vote_tweaks()
    * returns non-zero, it changed net_params and we should skip the tests for
@@ -1885,8 +1760,7 @@ test_a_networkstatus(
   vote_routerstatus_t *vrs;
   routerstatus_t *rs;
   int idx, n_rs, n_vrs;
-  char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL,
-    *cp=NULL;
+  char *consensus_text=NULL, *cp=NULL;
   smartlist_t *votes = smartlist_new();
 
   /* For generating the two other consensuses. */
@@ -1901,79 +1775,13 @@ test_a_networkstatus(
   tt_assert(rs_test);
   tt_assert(vrs_test);
 
-  /* Parse certificates and keys. */
-  cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
-  tt_assert(cert1);
-  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
-  tt_assert(cert2);
-  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
-  tt_assert(cert3);
-  sign_skey_1 = crypto_pk_new();
-  sign_skey_2 = crypto_pk_new();
-  sign_skey_3 = crypto_pk_new();
+  tt_assert(!dir_common_authority_pk_init(&cert1, &cert2, &cert3,
+                                          &sign_skey_1, &sign_skey_2,
+                                          &sign_skey_3));
   sign_skey_leg1 = pk_generate(4);
 
-  tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1,
-                                                   AUTHORITY_SIGNKEY_1, -1));
-  tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_2,
-                                                   AUTHORITY_SIGNKEY_2, -1));
-  tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_3,
-                                                   AUTHORITY_SIGNKEY_3, -1));
-
-  tt_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key));
-  tt_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key));
-
-  /*
-   * Set up a vote; generate it; try to parse it.
-   */
-  vote = tor_malloc_zero(sizeof(networkstatus_t));
-  vote->type = NS_TYPE_VOTE;
-  vote->published = now;
-  vote->valid_after = now+1000;
-  vote->fresh_until = now+2000;
-  vote->valid_until = now+3000;
-  vote->vote_seconds = 100;
-  vote->dist_seconds = 200;
-  vote->supported_methods = smartlist_new();
-  smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1);
-  vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15");
-  vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16");
-  vote->known_flags = smartlist_new();
-  smartlist_split_string(vote->known_flags,
-                     "Authority Exit Fast Guard Running Stable V2Dir Valid",
-                     0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  vote->voters = smartlist_new();
-  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
-  voter->nickname = tor_strdup("Voter1");
-  voter->address = tor_strdup("1.2.3.4");
-  voter->addr = 0x01020304;
-  voter->dir_port = 80;
-  voter->or_port = 9000;
-  voter->contact = tor_strdup("voter@example.com");
-  crypto_pk_get_digest(cert1->identity_key, voter->identity_digest);
-  smartlist_add(vote->voters, voter);
-  vote->cert = authority_cert_dup(cert1);
-  vote->net_params = smartlist_new();
-  smartlist_split_string(vote->net_params, "circuitwindow=101 foo=990",
-                         NULL, 0, 0);
-  vote->routerstatus_list = smartlist_new();
-  /* add routerstatuses */
-  idx = 0;
-  do {
-    vrs = vrs_gen(idx, now);
-    if (vrs) {
-      smartlist_add(vote->routerstatus_list, vrs);
-      tt_assert(router_add_to_routerlist(generate_ri_from_rs(vrs),
-                                           &msg,0,0)>=0);
-      ++idx;
-    }
-  } while (vrs);
-  n_vrs = idx;
-
-  /* dump the vote and try to parse it. */
-  v1_text = format_networkstatus_vote(sign_skey_1, vote);
-  tt_assert(v1_text);
-  v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE);
+  tt_assert(!dir_common_construct_vote_1(&vote, cert1, sign_skey_1, vrs_gen,
+                                         &v1, &n_vrs, now, 1));
   tt_assert(v1);
 
   /* Make sure the parsed thing was right. */
@@ -2000,6 +1808,7 @@ test_a_networkstatus(
   tt_str_op(cp,OP_EQ, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid");
   tor_free(cp);
   tt_int_op(smartlist_len(v1->routerstatus_list),OP_EQ, n_vrs);
+  tor_free(vote);
 
   if (vote_tweaks) params_tweaked += vote_tweaks(v1, 1, now);
 
@@ -2011,33 +1820,10 @@ test_a_networkstatus(
   }
 
   /* Generate second vote. It disagrees on some of the times,
-   * and doesn't list versions, and knows some crazy flags */
-  vote->published = now+1;
-  vote->fresh_until = now+3005;
-  vote->dist_seconds = 300;
-  authority_cert_free(vote->cert);
-  vote->cert = authority_cert_dup(cert2);
-  SMARTLIST_FOREACH(vote->net_params, char *, c, tor_free(c));
-  smartlist_clear(vote->net_params);
-  smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20",
-                         NULL, 0, 0);
-  tor_free(vote->client_versions);
-  tor_free(vote->server_versions);
-  voter = smartlist_get(vote->voters, 0);
-  tor_free(voter->nickname);
-  tor_free(voter->address);
-  voter->nickname = tor_strdup("Voter2");
-  voter->address = tor_strdup("2.3.4.5");
-  voter->addr = 0x02030405;
-  crypto_pk_get_digest(cert2->identity_key, voter->identity_digest);
-  smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese"));
-  smartlist_add(vote->known_flags, tor_strdup("MadeOfTin"));
-  smartlist_sort_strings(vote->known_flags);
-
-  /* generate and parse v2. */
-  v2_text = format_networkstatus_vote(sign_skey_2, vote);
-  tt_assert(v2_text);
-  v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE);
+   * and doesn't list versions, and knows some crazy flags.
+   * Generate and parse v2. */
+  tt_assert(!dir_common_construct_vote_2(&vote, cert2, sign_skey_2, vrs_gen,
+                                         &v2, &n_vrs, now, 1));
   tt_assert(v2);
 
   if (vote_tweaks) params_tweaked += vote_tweaks(v2, 2, now);
@@ -2055,34 +1841,11 @@ test_a_networkstatus(
     tt_assert(vrs);
     vrs_test(vrs, 2, now);
   }
+  tor_free(vote);
 
-  /* Generate the third vote. */
-  vote->published = now;
-  vote->fresh_until = now+2003;
-  vote->dist_seconds = 250;
-  authority_cert_free(vote->cert);
-  vote->cert = authority_cert_dup(cert3);
-  SMARTLIST_FOREACH(vote->net_params, char *, c, tor_free(c));
-  smartlist_clear(vote->net_params);
-  smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660",
-                         NULL, 0, 0);
-  smartlist_add(vote->supported_methods, tor_strdup("4"));
-  vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17");
-  vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16");
-  voter = smartlist_get(vote->voters, 0);
-  tor_free(voter->nickname);
-  tor_free(voter->address);
-  voter->nickname = tor_strdup("Voter3");
-  voter->address = tor_strdup("3.4.5.6");
-  voter->addr = 0x03040506;
-  crypto_pk_get_digest(cert3->identity_key, voter->identity_digest);
-  /* This one has a legacy id. */
-  memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN);
-
-  v3_text = format_networkstatus_vote(sign_skey_3, vote);
-  tt_assert(v3_text);
-
-  v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE);
+  /* Generate the third vote with a legacy id. */
+  tt_assert(!dir_common_construct_vote_3(&vote, cert3, sign_skey_3, vrs_gen,
+                                         &v3, &n_vrs, now, 1));
   tt_assert(v3);
 
   if (vote_tweaks) params_tweaked += vote_tweaks(v3, 3, now);
@@ -2153,12 +1916,20 @@ test_a_networkstatus(
 
   /* Check the routerstatuses. */
   n_rs = smartlist_len(con->routerstatus_list);
+  tt_assert(n_rs);
   for (idx = 0; idx < n_rs; ++idx) {
     rs = smartlist_get(con->routerstatus_list, idx);
     tt_assert(rs);
     rs_test(rs, now);
   }
 
+  n_rs = smartlist_len(con_md->routerstatus_list);
+  tt_assert(n_rs);
+  for (idx = 0; idx < n_rs; ++idx) {
+    rs = smartlist_get(con_md->routerstatus_list, idx);
+    tt_assert(rs);
+  }
+
   /* Check signatures.  the first voter is a pseudo-entry with a legacy key.
    * The second one hasn't signed.  The fourth one has signed: validate it. */
   voter = smartlist_get(con->voters, 1);
@@ -2309,9 +2080,6 @@ test_a_networkstatus(
  done:
   tor_free(cp);
   smartlist_free(votes);
-  tor_free(v1_text);
-  tor_free(v2_text);
-  tor_free(v3_text);
   tor_free(consensus_text);
   tor_free(consensus_text_md);
 
@@ -2368,7 +2136,7 @@ static void
 test_dir_v3_networkstatus(void *arg)
 {
   (void)arg;
-  test_a_networkstatus(gen_routerstatus_for_v3ns,
+  test_a_networkstatus(dir_common_gen_routerstatus_for_v3ns,
                        vote_tweaks_for_v3ns,
                        test_vrs_for_v3ns,
                        test_consensus_for_v3ns,
@@ -2965,6 +2733,7 @@ test_dir_fmt_control_ns(void *arg)
   rs.is_fast = 1;
   rs.is_flagged_running = 1;
   rs.has_bandwidth = 1;
+  rs.is_v2_dir = 1;
   rs.bandwidth_kb = 1000;
 
   s = networkstatus_getinfo_helper_single(&rs);

src/test/test_dir_common.c

@@ -0,0 +1,423 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#define DIRVOTE_PRIVATE
+#include "crypto.h"
+#include "test.h"
+#include "container.h"
+#include "or.h"
+#include "dirvote.h"
+#include "nodelist.h"
+#include "routerlist.h"
+#include "test_dir_common.h"
+
+void dir_common_setup_vote(networkstatus_t **vote, time_t now);
+networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote,
+                            networkstatus_t **vote_out,
+               vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                            crypto_pk_t *sign_skey, int *n_vrs,
+                            time_t now, int clear_rl);
+
+extern const char AUTHORITY_CERT_1[];
+extern const char AUTHORITY_SIGNKEY_1[];
+extern const char AUTHORITY_CERT_2[];
+extern const char AUTHORITY_SIGNKEY_2[];
+extern const char AUTHORITY_CERT_3[];
+extern const char AUTHORITY_SIGNKEY_3[];
+
+/** Initialize and set auth certs and keys
+ * Returns 0 on success, -1 on failure. Clean up handled by caller.
+ */
+int
+dir_common_authority_pk_init(authority_cert_t **cert1,
+                             authority_cert_t **cert2,
+                             authority_cert_t **cert3,
+                             crypto_pk_t **sign_skey_1,
+                             crypto_pk_t **sign_skey_2,
+                             crypto_pk_t **sign_skey_3)
+{
+  /* Parse certificates and keys. */
+  authority_cert_t *cert;
+  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  tt_assert(cert);
+  tt_assert(cert->identity_key);
+  *cert1 = cert;
+  tt_assert(*cert1);
+  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  tt_assert(*cert2);
+  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  tt_assert(*cert3);
+  *sign_skey_1 = crypto_pk_new();
+  *sign_skey_2 = crypto_pk_new();
+  *sign_skey_3 = crypto_pk_new();
+
+  tt_assert(!crypto_pk_read_private_key_from_string(*sign_skey_1,
+                                                   AUTHORITY_SIGNKEY_1, -1));
+  tt_assert(!crypto_pk_read_private_key_from_string(*sign_skey_2,
+                                                   AUTHORITY_SIGNKEY_2, -1));
+  tt_assert(!crypto_pk_read_private_key_from_string(*sign_skey_3,
+                                                   AUTHORITY_SIGNKEY_3, -1));
+
+  tt_assert(!crypto_pk_cmp_keys(*sign_skey_1, (*cert1)->signing_key));
+  tt_assert(!crypto_pk_cmp_keys(*sign_skey_2, (*cert2)->signing_key));
+
+  return 0;
+ done:
+  return -1;
+}
+
+/**
+ * Generate a routerstatus for v3_networkstatus test.
+ */
+vote_routerstatus_t *
+dir_common_gen_routerstatus_for_v3ns(int idx, time_t now)
+{
+  vote_routerstatus_t *vrs=NULL;
+  routerstatus_t *rs = NULL;
+  tor_addr_t addr_ipv6;
+  char *method_list = NULL;
+
+  switch (idx) {
+    case 0:
+      /* Generate the first routerstatus. */
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      vrs->version = tor_strdup("0.1.2.14");
+      rs->published_on = now-1500;
+      strlcpy(rs->nickname, "router2", sizeof(rs->nickname));
+      memset(rs->identity_digest, TEST_DIR_ROUTER_ID_1, DIGEST_LEN);
+      memset(rs->descriptor_digest, TEST_DIR_ROUTER_DD_1, DIGEST_LEN);
+      rs->addr = 0x99008801;
+      rs->or_port = 443;
+      rs->dir_port = 8000;
+      /* all flags but running and v2dir cleared */
+      rs->is_flagged_running = 1;
+      rs->is_v2_dir = 1;
+      break;
+    case 1:
+      /* Generate the second routerstatus. */
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      vrs->version = tor_strdup("0.2.0.5");
+      rs->published_on = now-1000;
+      strlcpy(rs->nickname, "router1", sizeof(rs->nickname));
+      memset(rs->identity_digest, TEST_DIR_ROUTER_ID_2, DIGEST_LEN);
+      memset(rs->descriptor_digest, TEST_DIR_ROUTER_DD_2, DIGEST_LEN);
+      rs->addr = 0x99009901;
+      rs->or_port = 443;
+      rs->dir_port = 0;
+      tor_addr_parse(&addr_ipv6, "[1:2:3::4]");
+      tor_addr_copy(&rs->ipv6_addr, &addr_ipv6);
+      rs->ipv6_orport = 4711;
+      rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running =
+        rs->is_valid = rs->is_possible_guard = rs->is_v2_dir = 1;
+      break;
+    case 2:
+      /* Generate the third routerstatus. */
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      vrs->version = tor_strdup("0.1.0.3");
+      rs->published_on = now-1000;
+      strlcpy(rs->nickname, "router3", sizeof(rs->nickname));
+      memset(rs->identity_digest, TEST_DIR_ROUTER_ID_3, DIGEST_LEN);
+      memset(rs->descriptor_digest, TEST_DIR_ROUTER_DD_3, DIGEST_LEN);
+      rs->addr = 0xAA009901;
+      rs->or_port = 400;
+      rs->dir_port = 9999;
+      rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
+        rs->is_flagged_running = rs->is_valid = rs->is_v2_dir =
+        rs->is_possible_guard = 1;
+      break;
+    case 3:
+      /* Generate a fourth routerstatus that is not running. */
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      vrs->version = tor_strdup("0.1.6.3");
+      rs->published_on = now-1000;
+      strlcpy(rs->nickname, "router4", sizeof(rs->nickname));
+      memset(rs->identity_digest, TEST_DIR_ROUTER_ID_4, DIGEST_LEN);
+      memset(rs->descriptor_digest, TEST_DIR_ROUTER_DD_4, DIGEST_LEN);
+      rs->addr = 0xC0000203;
+      rs->or_port = 500;
+      rs->dir_port = 1999;
+      /* Running flag (and others) cleared */
+      break;
+    case 4:
+      /* No more for this test; return NULL */
+      vrs = NULL;
+      break;
+    default:
+      /* Shouldn't happen */
+      tt_assert(0);
+  }
+  if (vrs) {
+    vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
+    method_list = make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
+                                             MAX_SUPPORTED_CONSENSUS_METHOD,
+                                             ",");
+    tor_asprintf(&vrs->microdesc->microdesc_hash_line,
+                 "m %s "
+                 "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n",
+                 method_list, idx);
+  }
+
+ done:
+  tor_free(method_list);
+  return vrs;
+}
+
+/** Initialize networkstatus vote object attributes. */
+void
+dir_common_setup_vote(networkstatus_t **vote, time_t now)
+{
+  *vote = tor_malloc_zero(sizeof(networkstatus_t));
+  (*vote)->type = NS_TYPE_VOTE;
+  (*vote)->published = now;
+  (*vote)->supported_methods = smartlist_new();
+  (*vote)->known_flags = smartlist_new();
+  (*vote)->net_params = smartlist_new();
+  (*vote)->routerstatus_list = smartlist_new();
+  (*vote)->voters = smartlist_new();
+}
+
+/** Helper: Make a new routerinfo containing the right information for a
+ * given vote_routerstatus_t. */
+routerinfo_t *
+dir_common_generate_ri_from_rs(const vote_routerstatus_t *vrs)
+{
+  routerinfo_t *r;
+  const routerstatus_t *rs = &vrs->status;
+  static time_t published = 0;
+
+  r = tor_malloc_zero(sizeof(routerinfo_t));
+  r->cert_expiration_time = TIME_MAX;
+  memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN);
+  memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest,
+         DIGEST_LEN);
+  r->cache_info.do_not_cache = 1;
+  r->cache_info.routerlist_index = -1;
+  r->cache_info.signed_descriptor_body =
+    tor_strdup("123456789012345678901234567890123");
+  r->cache_info.signed_descriptor_len =
+    strlen(r->cache_info.signed_descriptor_body);
+  r->exit_policy = smartlist_new();
+  r->cache_info.published_on = ++published + time(NULL);
+  if (rs->has_bandwidth) {
+    /*
+     * Multiply by 1000 because the routerinfo_t and the routerstatus_t
+     * seem to use different units (*sigh*) and because we seem stuck on
+     * icky and perverse decimal kilobytes (*double sigh*) - see
+     * router_get_advertised_bandwidth_capped() of routerlist.c and
+     * routerstatus_format_entry() of dirserv.c.
+     */
+    r->bandwidthrate = rs->bandwidth_kb * 1000;
+    r->bandwidthcapacity = rs->bandwidth_kb * 1000;
+  }
+  return r;
+}
+
+/** Create routerstatuses and signed vote.
+ * Create routerstatuses using *vrs_gen* and add them to global routerlist.
+ * Next, create signed vote using *sign_skey* and *vote*, which should have
+ * predefined header fields.
+ * Setting *clear_rl* clears the global routerlist before adding the new
+ * routers.
+ * Return the signed vote, same as *vote_out*. Save the number of routers added
+ * in *n_vrs*.
+ */
+networkstatus_t *
+dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out,
+                       vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                       crypto_pk_t *sign_skey, int *n_vrs, time_t now,
+                       int clear_rl)
+{
+  vote_routerstatus_t *vrs;
+  char *v_text=NULL;
+  const char *msg=NULL;
+  int idx;
+  was_router_added_t router_added = -1;
+  *vote_out = NULL;
+
+  if (clear_rl) {
+    nodelist_free_all();
+    routerlist_free_all();
+  }
+
+  idx = 0;
+  do {
+    vrs = vrs_gen(idx, now);
+    if (vrs) {
+      smartlist_add(vote->routerstatus_list, vrs);
+      router_added =
+        router_add_to_routerlist(dir_common_generate_ri_from_rs(vrs),
+                                 &msg,0,0);
+      tt_assert(router_added >= 0);
+      ++idx;
+    }
+  } while (vrs);
+  *n_vrs = idx;
+
+  /* dump the vote and try to parse it. */
+  v_text = format_networkstatus_vote(sign_skey, vote);
+  tt_assert(v_text);
+  *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE);
+
+ done:
+  if (v_text)
+    tor_free(v_text);
+
+  return *vote_out;
+}
+
+/** Create a fake *vote* where *cert* describes the signer, *sign_skey*
+ * is the signing key, and *vrs_gen* is the function we'll use to create the
+ * routers on which we're voting.
+ * We pass *vote_out*, *n_vrs*, and *clear_rl* directly to vrs_gen().
+ * Return 0 on success, return -1 on failure.
+ */
+int
+dir_common_construct_vote_1(networkstatus_t **vote, authority_cert_t *cert,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs,
+                        time_t now, int clear_rl)
+{
+  networkstatus_voter_info_t *voter;
+
+  dir_common_setup_vote(vote, now);
+  (*vote)->valid_after = now+1000;
+  (*vote)->fresh_until = now+2000;
+  (*vote)->valid_until = now+3000;
+  (*vote)->vote_seconds = 100;
+  (*vote)->dist_seconds = 200;
+  smartlist_split_string((*vote)->supported_methods, "1 2 3", NULL, 0, -1);
+  (*vote)->client_versions = tor_strdup("0.1.2.14,0.1.2.15");
+  (*vote)->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16");
+  smartlist_split_string((*vote)->known_flags,
+                     "Authority Exit Fast Guard Running Stable V2Dir Valid",
+                     0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup("Voter1");
+  voter->address = tor_strdup("1.2.3.4");
+  voter->addr = 0x01020304;
+  voter->dir_port = 80;
+  voter->or_port = 9000;
+  voter->contact = tor_strdup("voter@example.com");
+  crypto_pk_get_digest(cert->identity_key, voter->identity_digest);
+  /*
+   * Set up a vote; generate it; try to parse it.
+   */
+  smartlist_add((*vote)->voters, voter);
+  (*vote)->cert = authority_cert_dup(cert);
+  smartlist_split_string((*vote)->net_params, "circuitwindow=101 foo=990",
+                         NULL, 0, 0);
+  *n_vrs = 0;
+  /* add routerstatuses */
+  if (!dir_common_add_rs_and_parse(*vote, vote_out, vrs_gen, sign_skey,
+                                  n_vrs, now, clear_rl))
+    return -1;
+
+  return 0;
+}
+
+/** See dir_common_construct_vote_1.
+ * Produces a vote with slightly different values.
+ */
+int
+dir_common_construct_vote_2(networkstatus_t **vote, authority_cert_t *cert,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs,
+                        time_t now, int clear_rl)
+{
+  networkstatus_voter_info_t *voter;
+
+  dir_common_setup_vote(vote, now);
+  (*vote)->type = NS_TYPE_VOTE;
+  (*vote)->published += 1;
+  (*vote)->valid_after = now+1000;
+  (*vote)->fresh_until = now+3005;
+  (*vote)->valid_until = now+3000;
+  (*vote)->vote_seconds = 100;
+  (*vote)->dist_seconds = 300;
+  smartlist_split_string((*vote)->supported_methods, "1 2 3", NULL, 0, -1);
+  smartlist_split_string((*vote)->known_flags,
+                         "Authority Exit Fast Guard MadeOfCheese MadeOfTin "
+                         "Running Stable V2Dir Valid", 0,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup("Voter2");
+  voter->address = tor_strdup("2.3.4.5");
+  voter->addr = 0x02030405;
+  voter->dir_port = 80;
+  voter->or_port = 9000;
+  voter->contact = tor_strdup("voter@example.com");
+  crypto_pk_get_digest(cert->identity_key, voter->identity_digest);
+  /*
+   * Set up a vote; generate it; try to parse it.
+   */
+  smartlist_add((*vote)->voters, voter);
+  (*vote)->cert = authority_cert_dup(cert);
+  (*vote)->net_params = smartlist_new();
+  smartlist_split_string((*vote)->net_params,
+                         "bar=2000000000 circuitwindow=20",
+                         NULL, 0, 0);
+  /* add routerstatuses */
+  /* dump the vote and try to parse it. */
+  dir_common_add_rs_and_parse(*vote, vote_out, vrs_gen, sign_skey,
+                              n_vrs, now, clear_rl);
+
+  return 0;
+}
+
+/** See dir_common_construct_vote_1.
+ * Produces a vote with slightly different values. Adds a legacy key.
+ */
+int
+dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs,
+                        time_t now, int clear_rl)
+{
+  networkstatus_voter_info_t *voter;
+
+  dir_common_setup_vote(vote, now);
+  (*vote)->valid_after = now+1000;
+  (*vote)->fresh_until = now+2003;
+  (*vote)->valid_until = now+3000;
+  (*vote)->vote_seconds = 100;
+  (*vote)->dist_seconds = 250;
+  smartlist_split_string((*vote)->supported_methods, "1 2 3 4", NULL, 0, -1);
+  (*vote)->client_versions = tor_strdup("0.1.2.14,0.1.2.17");
+  (*vote)->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16");
+  smartlist_split_string((*vote)->known_flags,
+                     "Authority Exit Fast Guard Running Stable V2Dir Valid",
+                     0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup("Voter2");
+  voter->address = tor_strdup("3.4.5.6");
+  voter->addr = 0x03040506;
+  voter->dir_port = 80;
+  voter->or_port = 9000;
+  voter->contact = tor_strdup("voter@example.com");
+  crypto_pk_get_digest(cert->identity_key, voter->identity_digest);
+  memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN);
+  /*
+   * Set up a vote; generate it; try to parse it.
+   */
+  smartlist_add((*vote)->voters, voter);
+  (*vote)->cert = authority_cert_dup(cert);
+  smartlist_split_string((*vote)->net_params, "circuitwindow=80 foo=660",
+                         NULL, 0, 0);
+  /* add routerstatuses */
+  /* dump the vote and try to parse it. */
+  dir_common_add_rs_and_parse(*vote, vote_out, vrs_gen, sign_skey,
+                              n_vrs, now, clear_rl);
+
+  return 0;
+}
+

src/test/test_dir_common.h

@@ -0,0 +1,52 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "networkstatus.h"
+#include "routerparse.h"
+
+#define TEST_DIR_ROUTER_ID_1 3
+#define TEST_DIR_ROUTER_ID_2 5
+#define TEST_DIR_ROUTER_ID_3 33
+#define TEST_DIR_ROUTER_ID_4 34
+
+#define TEST_DIR_ROUTER_DD_1 78
+#define TEST_DIR_ROUTER_DD_2 77
+#define TEST_DIR_ROUTER_DD_3 79
+#define TEST_DIR_ROUTER_DD_4 44
+
+int dir_common_authority_pk_init(authority_cert_t **cert1,
+                       authority_cert_t **cert2,
+                       authority_cert_t **cert3,
+                       crypto_pk_t **sign_skey_1,
+                       crypto_pk_t **sign_skey_2,
+                       crypto_pk_t **sign_skey_3);
+
+routerinfo_t * dir_common_generate_ri_from_rs(const vote_routerstatus_t *vrs);
+
+vote_routerstatus_t * dir_common_gen_routerstatus_for_v3ns(int idx,
+                                                           time_t now);
+
+int dir_common_construct_vote_1(networkstatus_t **vote,
+                        authority_cert_t *cert1,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs, time_t now,
+                        int clear_rl);
+
+int dir_common_construct_vote_2(networkstatus_t **vote,
+                        authority_cert_t *cert2,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs, time_t now,
+                        int clear_rl);
+
+int dir_common_construct_vote_3(networkstatus_t **vote,
+                        authority_cert_t *cert3,
+                        crypto_pk_t *sign_skey,
+                        vote_routerstatus_t * (*vrs_gen)(int idx, time_t now),
+                        networkstatus_t **vote_out, int *n_vrs, time_t now,
+                        int clear_rl);
+

src/test/test_nodelist.c

@@ -60,12 +60,53 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg)
   return;
 }
 
+/** A node should be considered a directory server if it has an open dirport
+ * of it accepts tunnelled directory requests.
+ */
+static void
+test_nodelist_node_is_dir(void *arg)
+{
+  (void)arg;
+
+  routerstatus_t rs;
+  routerinfo_t ri;
+  node_t node;
+  memset(&node, 0, sizeof(node_t));
+  memset(&rs, 0, sizeof(routerstatus_t));
+  memset(&ri, 0, sizeof(routerinfo_t));
+
+  tt_assert(!node_is_dir(&node));
+
+  node.rs = &rs;
+  tt_assert(!node_is_dir(&node));
+
+  rs.is_v2_dir = 1;
+  tt_assert(node_is_dir(&node));
+
+  rs.is_v2_dir = 0;
+  rs.dir_port = 1;
+  tt_assert(node_is_dir(&node));
+
+  node.rs = NULL;
+  tt_assert(!node_is_dir(&node));
+  node.ri = &ri;
+  ri.supports_tunnelled_dir_requests = 1;
+  tt_assert(node_is_dir(&node));
+  ri.supports_tunnelled_dir_requests = 0;
+  ri.dir_port = 1;
+  tt_assert(node_is_dir(&node));
+
+  done:
+    return;
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
 struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_by_id_null_node, TT_FORK),
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
+  NODE(node_is_dir, TT_FORK),
   END_OF_TESTCASES
 };
 

src/test/test_options.c

@@ -69,22 +69,29 @@ clear_log_messages(void)
   messages = NULL;
 }
 
+#define setup_options(opt,dflt)              \
+  do {                                       \
+    opt = options_new();                     \
+    opt->command = CMD_RUN_TOR;              \
+    options_init(opt);                       \
+                                             \
+    dflt = config_dup(&options_format, opt); \
+    clear_log_messages();                    \
+  } while (0)
+
 static void
 test_options_validate_impl(const char *configuration,
                            const char *expect_errmsg,
                            int expect_log_severity,
                            const char *expect_log)
 {
-  or_options_t *opt = options_new();
+  or_options_t *opt=NULL;
   or_options_t *dflt;
   config_line_t *cl=NULL;
   char *msg=NULL;
   int r;
-  opt->command = CMD_RUN_TOR;
-  options_init(opt);
 
-  dflt = config_dup(&options_format, opt);
-  clear_log_messages();
+  setup_options(opt, dflt);
 
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
@@ -159,12 +166,110 @@ test_options_validate(void *arg)
                "ServerTransportOptions did not parse",
                LOG_WARN, "\"slingsnappy\" is not a k=v");
 
+  WANT_ERR("DirPort 8080\nDirCache 0",
+           "DirPort configured but DirCache disabled.");
+  WANT_ERR("BridgeRelay 1\nDirCache 0",
+           "We're a bridge but DirCache is disabled.");
+
   clear_log_messages();
   return;
 }
 
+#define MEGABYTEIFY(mb) (U64_LITERAL(mb) << 20)
+static void
+test_have_enough_mem_for_dircache(void *arg)
+{
+  (void)arg;
+  or_options_t *opt=NULL;
+  or_options_t *dflt;
+  config_line_t *cl=NULL;
+  char *msg=NULL;;
+  int r;
+  const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg;
+
+  setup_options(opt, dflt);
+  setup_log_callback();
+  (void)dflt;
+
+  r = config_get_lines(configuration, &cl, 1);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = config_assign(&options_format, opt, cl, 0, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* 300 MB RAM available, DirCache enabled */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(!msg);
+
+  /* 200 MB RAM available, DirCache enabled */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
+  tt_int_op(r, OP_EQ, -1);
+  expect_errmsg = "Being a directory cache (default) with less than ";
+  if (!strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  }
+  tor_free(msg);
+
+  configuration = "ORPort 8080\nDirCache 1\nBridgeRelay 1";
+  r = config_get_lines(configuration, &cl, 1);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = config_assign(&options_format, opt, cl, 0, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* 300 MB RAM available, DirCache enabled, Bridge */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(!msg);
+
+  /* 200 MB RAM available, DirCache enabled, Bridge */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
+  tt_int_op(r, OP_EQ, -1);
+  expect_errmsg = "Running a Bridge with less than ";
+  if (!strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  }
+  tor_free(msg);
+
+  configuration = "ORPort 8080\nDirCache 0";
+  r = config_get_lines(configuration, &cl, 1);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = config_assign(&options_format, opt, cl, 0, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* 200 MB RAM available, DirCache disabled */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(!msg);
+
+  /* 300 MB RAM available, DirCache disabled */
+  r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
+  tt_int_op(r, OP_EQ, -1);
+  expect_errmsg = "DirCache is disabled and we are configured as a ";
+  if (!strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  }
+  tor_free(msg);
+
+  clear_log_messages();
+
+ done:
+  if (msg)
+    tor_free(msg);
+  tor_free(dflt);
+  tor_free(opt);
+  tor_free(cl);
+  return;
+}
+
 struct testcase_t options_tests[] = {
   { "validate", test_options_validate, TT_FORK, NULL, NULL },
+  { "mem_dircache", test_have_enough_mem_for_dircache, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 

src/test/test_routerlist.c

@@ -1,11 +1,35 @@
 /* Copyright (c) 2014, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#include "orconfig.h"
+#include <math.h>
+#include <time.h>
+
+#define DIRVOTE_PRIVATE
+#define NETWORKSTATUS_PRIVATE
 #define ROUTERLIST_PRIVATE
+#define TOR_UNIT_TESTING
 #include "or.h"
-#include "routerlist.h"
+#include "config.h"
+#include "container.h"
 #include "directory.h"
+#include "dirvote.h"
+#include "networkstatus.h"
+#include "nodelist.h"
+#include "policies.h"
+#include "routerlist.h"
+#include "routerparse.h"
 #include "test.h"
+#include "test_dir_common.h"
+
+extern const char AUTHORITY_CERT_1[];
+extern const char AUTHORITY_SIGNKEY_1[];
+extern const char AUTHORITY_CERT_2[];
+extern const char AUTHORITY_SIGNKEY_2[];
+extern const char AUTHORITY_CERT_3[];
+extern const char AUTHORITY_SIGNKEY_3[];
+
+void construct_consensus(const char **consensus_text_md);
 
 /* 4 digests + 3 sep + pre + post + NULL */
 static char output[4*BASE64_DIGEST256_LEN+3+2+2+1];
@@ -94,12 +118,266 @@ test_routerlist_launch_descriptor_downloads(void *arg)
   smartlist_free(downloadable);
 }
 
+void
+construct_consensus(const char **consensus_text_md)
+{
+  networkstatus_t *vote = NULL;
+  networkstatus_t *v1 = NULL, *v2 = NULL, *v3 = NULL;
+  networkstatus_voter_info_t *voter = NULL;
+  authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL;
+  crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL;
+  crypto_pk_t *sign_skey_leg=NULL;
+  time_t now = time(NULL);
+  smartlist_t *votes = NULL;
+  addr_policy_t *pol1 = NULL, *pol2 = NULL, *pol3 = NULL;
+  int n_vrs;
+
+  tt_assert(!dir_common_authority_pk_init(&cert1, &cert2, &cert3,
+                                          &sign_skey_1, &sign_skey_2,
+                                          &sign_skey_3));
+  sign_skey_leg = pk_generate(4);
+
+  dir_common_construct_vote_1(&vote, cert1, sign_skey_1,
+                              &dir_common_gen_routerstatus_for_v3ns,
+                              &v1, &n_vrs, now, 1);
+
+  tt_assert(v1);
+  tt_int_op(n_vrs, ==, 4);
+  tt_int_op(smartlist_len(v1->routerstatus_list), ==, 4);
+
+  dir_common_construct_vote_2(&vote, cert2, sign_skey_2,
+                              &dir_common_gen_routerstatus_for_v3ns,
+                              &v2, &n_vrs, now, 1);
+
+  tt_assert(v2);
+  tt_int_op(n_vrs, ==, 4);
+  tt_int_op(smartlist_len(v2->routerstatus_list), ==, 4);
+
+  dir_common_construct_vote_3(&vote, cert3, sign_skey_3,
+                              &dir_common_gen_routerstatus_for_v3ns,
+                              &v3, &n_vrs, now, 1);
+
+  tt_assert(v3);
+  tt_int_op(n_vrs, ==, 4);
+  tt_int_op(smartlist_len(v3->routerstatus_list), ==, 4);
+
+  votes = smartlist_new();
+  smartlist_add(votes, v1);
+  smartlist_add(votes, v2);
+  smartlist_add(votes, v3);
+
+  *consensus_text_md = networkstatus_compute_consensus(votes, 3,
+                                                   cert1->identity_key,
+                                                   sign_skey_1,
+                                                   "AAAAAAAAAAAAAAAAAAAA",
+                                                   sign_skey_leg,
+                                                   FLAV_MICRODESC);
+
+  tt_assert(*consensus_text_md);
+
+ done:
+  if (vote)
+    tor_free(vote);
+  if (voter)
+    tor_free(voter);
+  if (pol1)
+    tor_free(pol1);
+  if (pol2)
+    tor_free(pol2);
+  if (pol3)
+    tor_free(pol3);
+}
+
+static void
+test_router_pick_directory_server_impl(void *arg)
+{
+  (void)arg;
+
+  networkstatus_t *con_md = NULL;
+  const char *consensus_text_md = NULL;
+  int flags = PDS_IGNORE_FASCISTFIREWALL|PDS_RETRY_IF_NO_SERVERS;
+  or_options_t *options = get_options_mutable();
+  const routerstatus_t *rs = NULL;
+  options->UseMicrodescriptors = 1;
+  char *router1_id = NULL, *router2_id = NULL, *router3_id = NULL;
+  node_t *node_router1 = NULL, *node_router2 = NULL, *node_router3 = NULL;
+  config_line_t *policy_line = NULL;
+  time_t now = time(NULL);
+  int tmp_dirport1, tmp_dirport3;
+
+  (void)arg;
+
+  /* No consensus available, fail early */
+  rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL);
+  tt_assert(rs == NULL);
+
+  construct_consensus(&consensus_text_md);
+  tt_assert(consensus_text_md);
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+                                                NS_TYPE_CONSENSUS);
+  tt_assert(con_md);
+  tt_int_op(con_md->flavor,==, FLAV_MICRODESC);
+  tt_assert(con_md->routerstatus_list);
+  tt_int_op(smartlist_len(con_md->routerstatus_list), ==, 3);
+  tt_assert(!networkstatus_set_current_consensus_from_ns(con_md,
+                                                 "microdesc"));
+  nodelist_set_consensus(con_md);
+  nodelist_assert_ok();
+
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  /* We should not fail now we have a consensus and routerstatus_list
+   * and nodelist are populated. */
+  tt_assert(rs != NULL);
+
+  /* Manipulate the nodes so we get the dir server we expect */
+  router1_id = tor_malloc(DIGEST_LEN);
+  memset(router1_id, TEST_DIR_ROUTER_ID_1, DIGEST_LEN);
+  router2_id = tor_malloc(DIGEST_LEN);
+  memset(router2_id, TEST_DIR_ROUTER_ID_2, DIGEST_LEN);
+  router3_id = tor_malloc(DIGEST_LEN);
+  memset(router3_id, TEST_DIR_ROUTER_ID_3, DIGEST_LEN);
+
+  node_router1 = node_get_mutable_by_id(router1_id);
+  node_router2 = node_get_mutable_by_id(router2_id);
+  node_router3 = node_get_mutable_by_id(router3_id);
+
+  node_router1->is_running = 0;
+  node_router3->is_running = 0;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
+  rs = NULL;
+  node_router1->is_running = 1;
+  node_router3->is_running = 1;
+
+  node_router1->rs->is_v2_dir = 0;
+  node_router3->rs->is_v2_dir = 0;
+  tmp_dirport1 = node_router1->rs->dir_port;
+  tmp_dirport3 = node_router3->rs->dir_port;
+  node_router1->rs->dir_port = 0;
+  node_router3->rs->dir_port = 0;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
+  rs = NULL;
+  node_router1->rs->is_v2_dir = 1;
+  node_router3->rs->is_v2_dir = 1;
+  node_router1->rs->dir_port = tmp_dirport1;
+  node_router3->rs->dir_port = tmp_dirport3;
+
+  node_router1->is_valid = 0;
+  node_router3->is_valid = 0;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
+  rs = NULL;
+  node_router1->is_valid = 1;
+  node_router3->is_valid = 1;
+
+  flags |= PDS_FOR_GUARD;
+  node_router1->using_as_guard = 1;
+  node_router2->using_as_guard = 1;
+  node_router3->using_as_guard = 1;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs == NULL);
+  node_router1->using_as_guard = 0;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
+  rs = NULL;
+  node_router2->using_as_guard = 0;
+  node_router3->using_as_guard = 0;
+
+  /* One not valid, one guard. This should leave one remaining */
+  node_router1->is_valid = 0;
+  node_router2->using_as_guard = 1;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
+  rs = NULL;
+  node_router1->is_valid = 1;
+  node_router2->using_as_guard = 0;
+
+  /* Manipulate overloaded */
+
+  node_router2->rs->last_dir_503_at = now;
+  node_router3->rs->last_dir_503_at = now;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
+  node_router2->rs->last_dir_503_at = 0;
+  node_router3->rs->last_dir_503_at = 0;
+
+  /* Set a Fascist firewall */
+  flags &= ! PDS_IGNORE_FASCISTFIREWALL;
+  policy_line = tor_malloc_zero(sizeof(config_line_t));
+  policy_line->key = tor_strdup("ReachableORAddresses");
+  policy_line->value = tor_strdup("accept *:442, reject *:*");
+  options->ReachableORAddresses = policy_line;
+  policies_parse_from_options(options);
+
+  node_router1->rs->or_port = 444;
+  node_router2->rs->or_port = 443;
+  node_router3->rs->or_port = 442;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
+  node_router1->rs->or_port = 442;
+  node_router2->rs->or_port = 443;
+  node_router3->rs->or_port = 444;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
+
+  /* Fascist firewall and overloaded */
+  node_router1->rs->or_port = 442;
+  node_router2->rs->or_port = 443;
+  node_router3->rs->or_port = 442;
+  node_router3->rs->last_dir_503_at = now;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
+  node_router3->rs->last_dir_503_at = 0;
+
+  /* Fascists against OR and Dir */
+  policy_line = tor_malloc_zero(sizeof(config_line_t));
+  policy_line->key = tor_strdup("ReachableAddresses");
+  policy_line->value = tor_strdup("accept *:80, reject *:*");
+  options->ReachableDirAddresses = policy_line;
+  policies_parse_from_options(options);
+  node_router1->rs->or_port = 442;
+  node_router2->rs->or_port = 441;
+  node_router3->rs->or_port = 443;
+  node_router1->rs->dir_port = 80;
+  node_router2->rs->dir_port = 80;
+  node_router3->rs->dir_port = 81;
+  node_router1->rs->last_dir_503_at = now;
+  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
+  tt_assert(rs != NULL);
+  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
+  node_router1->rs->last_dir_503_at = 0;
+
+ done:
+  if (router1_id)
+    tor_free(router1_id);
+  if (router2_id)
+    tor_free(router2_id);
+  if (router3_id)
+    tor_free(router3_id);
+  if (options->ReachableORAddresses ||
+      options->ReachableDirAddresses)
+    policies_free_all();
+}
+
 #define NODE(name, flags) \
   { #name, test_routerlist_##name, (flags), NULL, NULL }
+#define ROUTER(name,flags) \
+  { #name, test_router_##name, (flags), NULL, NULL }
 
 struct testcase_t routerlist_tests[] = {
   NODE(initiate_descriptor_downloads, 0),
   NODE(launch_descriptor_downloads, 0),
+  ROUTER(pick_directory_server_impl, TT_FORK),
   END_OF_TESTCASES
 };