Merge branch 'fallback_dirsource_v3'

changes/fallback_dirsource

@@ -0,0 +1,13 @@
+  o Major features:
+    - Add a new FallbackDir option to use when we can't use a directory
+      from the consensus (either because we lack a consensus, or because
+      they're all down).  Currently, all authorities are fallbacks by
+      default, and there are no other default fallbacks, but that will
+      change.  This option will allow us to give clients a longer list
+      of servers to try to get a consensus from when first connecting 
+      to the Tor network, and thereby reduce load on the directory
+      authorities.
+
+  o Removed features:
+    - Drop the old FallbackNetworkstatus option: we never got it working
+      well enough to use it.

changes/rename_dirserver

@@ -0,0 +1,3 @@
+  o Renamed options:
+    - The DirServer option is now DirAuthority, for consistency with current
+      naming patterns.  You can still use the old DirServer form.

doc/tor.1.txt

@@ -292,7 +292,12 @@ GENERAL OPTIONS
 **DataDirectory** __DIR__::
     Store working data in DIR (Default: @LOCALSTATEDIR@/lib/tor)
 
-**DirServer** [__nickname__] [**flags**] __address__:__port__ __fingerprint__::
+**FallbackDir** __address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__]::
+    When we're unable to connect to any directory cache for directory info
+    (usually because we don't know about any yet) we try a FallbackDir.
+    By default, the directory authorities are also FallbackDirs.
+
+**DirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated
     many times, for multiple authoritative directory servers. Flags are
@@ -305,16 +310,24 @@ GENERAL OPTIONS
     flag is set, or if the "v1" flag is set and the "no-hs" flag is **not** set.
     Tor will use this authority as a bridge authoritative directory if the
     "bridge" flag is set. If a flag "orport=**port**" is given, Tor will use the
-    given port when opening encrypted tunnels to the dirserver. Lastly, if a
+    given port when opening encrypted tunnels to the dirserver. If a flag
+    "weight=**num**" is given, then the directory server is chosen randomly
+    with probability proportional to that weight (default 1.0). Lastly, if a
     flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority
     whose v3 long-term signing key has the fingerprint **fp**. +
  +
-    If no **dirserver** line is given, Tor will use the default directory
-    servers. NOTE: this option is intended for setting up a private Tor
+    If no **DirAuthority** line is given, Tor will use the default directory
+    authorities. NOTE: this option is intended for setting up a private Tor
     network with its own directory authorities. If you use it, you will be
     distinguishable from other users, because you won't believe the same
     authorities they do.
 
+**DirAuthorityFallbackRate** __NUM__::
+    When configured to use both directory authorities and fallback
+    directories, the directory authorities also work as fallbacks. They are
+    chosen with their regular weights, multiplied by this number, which
+    should be 1.0 or less. (Default: 1.0)
+
 **DynamicDHGroups** **0**|**1**::
     If this option is set to 1, when running as a server, generate our
     own Diffie-Hellman group instead of using the one from Apache's mod_ssl.
@@ -326,7 +339,7 @@ GENERAL OPTIONS
 **AlternateHSAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__ +
 
 **AlternateBridgeAuthority** [__nickname__] [**flags**] __address__:__port__ __ fingerprint__::
-    These options behave as DirServer, but they replace fewer of the
+    These options behave as DirAuthority, but they replace fewer of the
     default directory authorities. Using
     AlternateDirAuthority replaces the default Tor directory authorities, but
     leaves the default hidden service authorities and bridge authorities in
@@ -1122,12 +1135,6 @@ The following options are useful only for clients (that is, if
     regular router descriptors. Tor does not use this information for anything
     itself; to save bandwidth, leave this option turned off. (Default: 0)
 
-**FallbackNetworkstatusFile** __FILENAME__::
-    If Tor doesn't have a cached networkstatus file, it starts out using this
-    one instead. Even if this file is out of date, Tor can still use it to
-    learn about directory mirrors, so it doesn't need to put load on the
-    authorities. (Default: None)
-
 **WarnPlaintextPorts** __port__,__port__,__...__::
     Tells Tor to issue a warnings whenever the user tries to make an anonymous
     connection to one of these ports. This option is designed to alert users

src/or/config.c

@@ -81,6 +81,7 @@ static config_abbrev_t option_abbrevs_[] = {
   { "BandwidthRateBytes", "BandwidthRate", 0, 0},
   { "BandwidthBurstBytes", "BandwidthBurst", 0, 0},
   { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0},
+  { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */
   { "MaxConn", "ConnLimit", 0, 1},
   { "ORBindAddress", "ORListenAddress", 0, 0},
   { "DirBindAddress", "DirListenAddress", 0, 0},
@@ -206,7 +207,8 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DirRecordUsageRetainIPs"),
   OBSOLETE("DirRecordUsageSaveInterval"),
   V(DirReqStatistics,            BOOL,     "1"),
-  VAR("DirServer",               LINELIST, DirServers, NULL),
+  VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
+  V(DirAuthorityFallbackRate,    DOUBLE,   "1.0"),
   V(DisableAllSwap,              BOOL,     "0"),
   V(DisableDebuggerAttachment,   BOOL,     "1"),
   V(DisableIOCP,                 BOOL,     "1"),
@@ -227,13 +229,9 @@ static config_var_t option_vars_[] = {
   V(ExitPortStatistics,          BOOL,     "0"),
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
+  V(FallbackDir,                 LINELIST, NULL),
 
-#if defined (WINCE)
-  V(FallbackNetworkstatusFile,   FILENAME, "fallback-consensus"),
-#else
-  V(FallbackNetworkstatusFile,   FILENAME,
-    SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"),
-#endif
+  OBSOLETE("FallbackNetworkstatusFile"),
   V(FascistFirewall,             BOOL,     "0"),
   V(FirewallPorts,               CSV,      ""),
   V(FastFirstHopPK,              BOOL,     "1"),
@@ -470,10 +468,11 @@ static int parse_client_transport_line(const char *line, int validate_only);
 static int parse_server_transport_line(const char *line, int validate_only);
 static char *get_bindaddr_from_transport_listen_line(const char *line,
                                                      const char *transport);
-
-static int parse_dir_server_line(const char *line,
+static int parse_dir_authority_line(const char *line,
                                  dirinfo_type_t required_type,
                                  int validate_only);
+static int parse_dir_fallback_line(const char *line,
+                                   int validate_only);
 static void port_cfg_free(port_cfg_t *port);
 static int parse_ports(or_options_t *options, int validate_only,
                               char **msg_out, int *n_ports_out);
@@ -756,7 +755,7 @@ static void
 add_default_trusted_dir_authorities(dirinfo_type_t type)
 {
   int i;
-  const char *dirservers[] = {
+  const char *authorities[] = {
     "moria1 orport=9101 no-v2 "
       "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 "
       "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31",
@@ -785,10 +784,27 @@ add_default_trusted_dir_authorities(dirinfo_type_t type)
       "154.35.32.5:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC",
     NULL
   };
-  for (i=0; dirservers[i]; i++) {
-    if (parse_dir_server_line(dirservers[i], type, 0)<0) {
-      log_err(LD_BUG, "Couldn't parse internal dirserver line %s",
-              dirservers[i]);
+  for (i=0; authorities[i]; i++) {
+    if (parse_dir_authority_line(authorities[i], type, 0)<0) {
+      log_err(LD_BUG, "Couldn't parse internal DirAuthority line %s",
+              authorities[i]);
+    }
+  }
+}
+
+/** Add the default fallback directory servers into the fallback directory
+ * server list. */
+static void
+add_default_fallback_dir_servers(void)
+{
+  int i;
+  const char *fallback[] = {
+    NULL
+  };
+  for (i=0; fallback[i]; i++) {
+    if (parse_dir_fallback_line(fallback[i], 0)<0) {
+      log_err(LD_BUG, "Couldn't parse internal FallbackDir line %s",
+              fallback[i]);
     }
   }
 }
@@ -798,28 +814,28 @@ add_default_trusted_dir_authorities(dirinfo_type_t type)
  * user if we changed any dangerous ones.
  */
 static int
-validate_dir_authorities(or_options_t *options, or_options_t *old_options)
+validate_dir_servers(or_options_t *options, or_options_t *old_options)
 {
   config_line_t *cl;
 
-  if (options->DirServers &&
+  if (options->DirAuthorities &&
       (options->AlternateDirAuthority || options->AlternateBridgeAuthority ||
        options->AlternateHSAuthority)) {
     log_warn(LD_CONFIG,
-             "You cannot set both DirServers and Alternate*Authority.");
+             "You cannot set both DirAuthority and Alternate*Authority.");
     return -1;
   }
 
   /* do we want to complain to the user about being partitionable? */
-  if ((options->DirServers &&
+  if ((options->DirAuthorities &&
        (!old_options ||
-        !config_lines_eq(options->DirServers, old_options->DirServers))) ||
+        !config_lines_eq(options->DirAuthorities, old_options->DirAuthorities))) ||
       (options->AlternateDirAuthority &&
        (!old_options ||
         !config_lines_eq(options->AlternateDirAuthority,
                          old_options->AlternateDirAuthority)))) {
     log_warn(LD_CONFIG,
-             "You have used DirServer or AlternateDirAuthority to "
+             "You have used DirAuthority or AlternateDirAuthority to "
              "specify alternate directory authorities in "
              "your configuration. This is potentially dangerous: it can "
              "make you look different from all other Tor users, and hurt "
@@ -830,17 +846,20 @@ validate_dir_authorities(or_options_t *options, or_options_t *old_options)
 
   /* Now go through the four ways you can configure an alternate
    * set of directory authorities, and make sure none are broken. */
-  for (cl = options->DirServers; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 1)<0)
+  for (cl = options->DirAuthorities; cl; cl = cl->next)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0)
       return -1;
   for (cl = options->AlternateBridgeAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 1)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0)
       return -1;
   for (cl = options->AlternateDirAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 1)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0)
       return -1;
   for (cl = options->AlternateHSAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 1)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0)
+      return -1;
+  for (cl = options->FallbackDir; cl; cl = cl->next)
+    if (parse_dir_fallback_line(cl->value, 1)<0)
       return -1;
   return 0;
 }
@@ -849,13 +868,15 @@ validate_dir_authorities(or_options_t *options, or_options_t *old_options)
  * as appropriate.
  */
 static int
-consider_adding_dir_authorities(const or_options_t *options,
-                                const or_options_t *old_options)
+consider_adding_dir_servers(const or_options_t *options,
+                            const or_options_t *old_options)
 {
   config_line_t *cl;
   int need_to_update =
-    !smartlist_len(router_get_trusted_dir_servers()) || !old_options ||
-    !config_lines_eq(options->DirServers, old_options->DirServers) ||
+    !smartlist_len(router_get_trusted_dir_servers()) ||
+    !smartlist_len(router_get_fallback_dir_servers()) || !old_options ||
+    !config_lines_eq(options->DirAuthorities, old_options->DirAuthorities) ||
+    !config_lines_eq(options->FallbackDir, old_options->FallbackDir) ||
     !config_lines_eq(options->AlternateBridgeAuthority,
                      old_options->AlternateBridgeAuthority) ||
     !config_lines_eq(options->AlternateDirAuthority,
@@ -867,9 +888,9 @@ consider_adding_dir_authorities(const or_options_t *options,
     return 0; /* all done */
 
   /* Start from a clean slate. */
-  clear_trusted_dir_servers();
+  clear_dir_servers();
 
-  if (!options->DirServers) {
+  if (!options->DirAuthorities) {
     /* then we may want some of the defaults */
     dirinfo_type_t type = NO_DIRINFO;
     if (!options->AlternateBridgeAuthority)
@@ -881,18 +902,23 @@ consider_adding_dir_authorities(const or_options_t *options,
       type |= HIDSERV_DIRINFO;
     add_default_trusted_dir_authorities(type);
   }
+  if (!options->FallbackDir)
+    add_default_fallback_dir_servers();
 
-  for (cl = options->DirServers; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 0)<0)
+  for (cl = options->DirAuthorities; cl; cl = cl->next)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
       return -1;
   for (cl = options->AlternateBridgeAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 0)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
       return -1;
   for (cl = options->AlternateDirAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 0)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
       return -1;
   for (cl = options->AlternateHSAuthority; cl; cl = cl->next)
-    if (parse_dir_server_line(cl->value, NO_DIRINFO, 0)<0)
+    if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
+      return -1;
+  for (cl = options->FallbackDir; cl; cl = cl->next)
+    if (parse_dir_fallback_line(cl->value, 0)<0)
       return -1;
   return 0;
 }
@@ -1216,7 +1242,7 @@ options_act(const or_options_t *old_options)
       return -1;
   }
 
-  if (consider_adding_dir_authorities(options, old_options) < 0)
+  if (consider_adding_dir_servers(options, old_options) < 0)
     return -1;
 
 #ifdef NON_ANONYMOUS_MODE_ENABLED
@@ -1924,18 +1950,18 @@ resolve_my_address(int warn_severity, const or_options_t *options,
   addr_string = tor_dup_ip(addr);
   if (is_internal_IP(addr, 0)) {
     /* make sure we're ok with publishing an internal IP */
-    if (!options->DirServers && !options->AlternateDirAuthority) {
-      /* if they are using the default dirservers, disallow internal IPs
+    if (!options->DirAuthorities && !options->AlternateDirAuthority) {
+      /* if they are using the default authorities, disallow internal IPs
        * always. */
       log_fn(warn_severity, LD_CONFIG,
              "Address '%s' resolves to private IP address '%s'. "
-             "Tor servers that use the default DirServers must have public "
+             "Tor servers that use the default DirAuthorities must have public "
              "IP addresses.", hostname, addr_string);
       tor_free(addr_string);
       return -1;
     }
     if (!explicit_ip) {
-      /* even if they've set their own dirservers, require an explicit IP if
+      /* even if they've set their own authorities, require an explicit IP if
        * they're using an internal address. */
       log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private "
              "IP address '%s'. Please set the Address config option to be "
@@ -2843,8 +2869,9 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (validate_addr_policies(options, msg) < 0)
     return -1;
 
-  if (validate_dir_authorities(options, old_options) < 0)
-    REJECT("Directory authority line did not parse. See logs for details.");
+  if (validate_dir_servers(options, old_options) < 0)
+    REJECT("Directory authority/fallback line did not parse. See logs "
+           "for details.");
 
   if (options->UseBridges && !options->Bridges)
     REJECT("If you set UseBridges, you must specify at least one bridge.");
@@ -2962,7 +2989,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->TestingTorNetwork &&
-      !(options->DirServers ||
+      !(options->DirAuthorities ||
         (options->AlternateDirAuthority &&
          options->AlternateBridgeAuthority))) {
     REJECT("TestingTorNetwork may only be configured in combination with "
@@ -2970,7 +2997,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "and AlternateBridgeAuthority configured.");
   }
 
-  if (options->AllowSingleHopExits && !options->DirServers) {
+  if (options->AllowSingleHopExits && !options->DirAuthorities) {
     COMPLAIN("You have set AllowSingleHopExits; now your relay will allow "
              "others to make one-hop exits. However, since by default most "
              "clients avoid relays that set this option, most clients will "
@@ -4322,15 +4349,15 @@ parse_server_transport_line(const char *line, int validate_only)
   return r;
 }
 
-/** Read the contents of a DirServer line from <b>line</b>. If
+/** Read the contents of a DirAuthority line from <b>line</b>. If
  * <b>validate_only</b> is 0, and the line is well-formed, and it
  * shares any bits with <b>required_type</b> or <b>required_type</b>
  * is 0, then add the dirserver described in the line (minus whatever
  * bits it's missing) as a valid authority. Return 0 on success,
  * or -1 if the line isn't well-formed or if we can't add it. */
 static int
-parse_dir_server_line(const char *line, dirinfo_type_t required_type,
-                      int validate_only)
+parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
+                         int validate_only)
 {
   smartlist_t *items = NULL;
   int r;
@@ -4340,6 +4367,7 @@ parse_dir_server_line(const char *line, dirinfo_type_t required_type,
   char v3_digest[DIGEST_LEN];
   dirinfo_type_t type = V2_DIRINFO;
   int is_not_hidserv_authority = 0, is_not_v2_authority = 0;
+  double weight = 1.0;
 
   items = smartlist_new();
   smartlist_split_string(items, line, NULL,
@@ -4375,6 +4403,14 @@ parse_dir_server_line(const char *line, dirinfo_type_t required_type,
       if (!ok)
         log_warn(LD_CONFIG, "Invalid orport '%s' on DirServer line.",
                  portstring);
+    } else if (!strcmpstart(flag, "weight=")) {
+      int ok;
+      const char *wstring = flag + strlen("weight=");
+      weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_CONFIG, "Invalid weight '%s' on DirAuthority line.",flag);
+        weight=1.0;
+      }
     } else if (!strcasecmpstart(flag, "v3ident=")) {
       char *idstr = flag + strlen("v3ident=");
       if (strlen(idstr) != HEX_DIGEST_LEN ||
@@ -4431,14 +4467,16 @@ parse_dir_server_line(const char *line, dirinfo_type_t required_type,
   }
 
   if (!validate_only && (!required_type || required_type & type)) {
+    dir_server_t *ds;
     if (required_type)
       type &= required_type; /* pare down what we think of them as an
                               * authority for. */
     log_debug(LD_DIR, "Trusted %d dirserver at %s:%d (%s)", (int)type,
               address, (int)dir_port, (char*)smartlist_get(items,0));
-    if (!add_trusted_dir_server(nickname, address, dir_port, or_port,
-                                digest, v3_digest, type))
+    if (!(ds = trusted_dir_server_new(nickname, address, dir_port, or_port,
+                                      digest, v3_digest, type, weight)))
       goto err;
+    dir_server_add(ds);
   }
 
   r = 0;
@@ -4457,6 +4495,99 @@ parse_dir_server_line(const char *line, dirinfo_type_t required_type,
   return r;
 }
 
+/** Read the contents of a FallbackDir line from <b>line</b>. If
+ * <b>validate_only</b> is 0, and the line is well-formed, then add the
+ * dirserver described in the line as a fallback directory. Return 0 on
+ * success, or -1 if the line isn't well-formed or if we can't add it. */
+static int
+parse_dir_fallback_line(const char *line,
+                        int validate_only)
+{
+  int r = -1;
+  smartlist_t *items = smartlist_new(), *positional = smartlist_new();
+  int orport = -1;
+  uint16_t dirport;
+  tor_addr_t addr;
+  int ok;
+  char id[DIGEST_LEN];
+  char *address=NULL;
+  double weight=1.0;
+
+  memset(id, 0, sizeof(id));
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(items, const char *, cp) {
+    const char *eq = strchr(cp, '=');
+    ok = 1;
+    if (! eq) {
+      smartlist_add(positional, (char*)cp);
+      continue;
+    }
+    if (!strcmpstart(cp, "orport=")) {
+      orport = (int)tor_parse_long(cp+strlen("orport="), 10,
+                                   1, 65535, &ok, NULL);
+    } else if (!strcmpstart(cp, "id=")) {
+      ok = !base16_decode(id, DIGEST_LEN,
+                          cp+strlen("id="), strlen(cp)-strlen("id="));
+    } else if (!strcmpstart(cp, "weight=")) {
+      int ok;
+      const char *wstring = cp + strlen("weight=");
+      weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL);
+      if (!ok) {
+        log_warn(LD_CONFIG, "Invalid weight '%s' on FallbackDir line.", cp);
+        weight=1.0;
+      }
+    }
+
+    if (!ok) {
+      log_warn(LD_CONFIG, "Bad FallbackDir option %s", escaped(cp));
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (smartlist_len(positional) != 1) {
+    log_warn(LD_CONFIG, "Couldn't parse FallbackDir line %s", escaped(line));
+    goto end;
+  }
+
+  if (tor_digest_is_zero(id)) {
+    log_warn(LD_CONFIG, "Missing identity on FallbackDir line");
+    goto end;
+  }
+
+  if (orport <= 0) {
+    log_warn(LD_CONFIG, "Missing orport on FallbackDir line");
+    goto end;
+  }
+
+  if (tor_addr_port_split(LOG_INFO, smartlist_get(positional, 0),
+                          &address, &dirport) < 0 ||
+      tor_addr_parse(&addr, address)<0) {
+    log_warn(LD_CONFIG, "Couldn't parse address:port %s on FallbackDir line",
+             (const char*)smartlist_get(positional, 0));
+    goto end;
+  }
+
+  if (!validate_only) {
+    dir_server_t *ds;
+    ds = fallback_dir_server_new(&addr, dirport, orport, id, weight);
+    if (!ds) {
+      log_warn(LD_CONFIG, "Couldn't create FallbackDir %s", escaped(line));
+      goto end;
+    }
+    dir_server_add(ds);
+  }
+
+  r = 0;
+
+ end:
+  SMARTLIST_FOREACH(items, char *, cp, tor_free(cp));
+  smartlist_free(items);
+  smartlist_free(positional);
+  tor_free(address);
+  return r;
+}
+
 /** Free all storage held in <b>port</b> */
 static void
 port_cfg_free(port_cfg_t *port)

src/or/directory.c

@@ -244,9 +244,9 @@ router_supports_extrainfo(const char *identity_digest, int is_authority)
 int
 directories_have_accepted_server_descriptor(void)
 {
-  smartlist_t *servers = router_get_trusted_dir_servers();
+  const smartlist_t *servers = router_get_trusted_dir_servers();
   const or_options_t *options = get_options();
-  SMARTLIST_FOREACH(servers, trusted_dir_server_t *, d, {
+  SMARTLIST_FOREACH(servers, dir_server_t *, d, {
     if ((d->type & options->PublishServerDescriptor_) &&
         d->has_accepted_serverdesc) {
       return 1;
@@ -280,7 +280,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
 {
   const or_options_t *options = get_options();
   int post_via_tor;
-  smartlist_t *dirservers = router_get_trusted_dir_servers();
+  const smartlist_t *dirservers = router_get_trusted_dir_servers();
   int found = 0;
   const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
                             dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
@@ -288,7 +288,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
   /* This tries dirservers which we believe to be down, but ultimately, that's
    * harmless, and we may as well err on the side of getting things uploaded.
    */
-  SMARTLIST_FOREACH_BEGIN(dirservers, trusted_dir_server_t *, ds) {
+  SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
       routerstatus_t *rs = &(ds->fake_status);
       size_t upload_len = payload_len;
       tor_addr_t ds_addr;
@@ -474,7 +474,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
         if (!rs) {
           log_info(LD_DIR, "No router found for %s; falling back to "
                    "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
-          rs = router_pick_trusteddirserver(type, pds_flags);
+          rs = router_pick_fallback_dirserver(type, pds_flags);
           if (!rs)
             get_via_tor = 1; /* last resort: try routing it via Tor */
         }
@@ -528,7 +528,7 @@ directory_get_from_all_authorities(uint8_t dir_purpose,
              dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
 
   SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
-                          trusted_dir_server_t *, ds) {
+                          dir_server_t *, ds) {
       routerstatus_t *rs;
       if (router_digest_is_me(ds->digest))
         continue;
@@ -716,8 +716,8 @@ connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn,
     /* We're a non-authoritative directory cache; try again. Ignore status
      * code, since we don't want to keep trying forever in a tight loop
      * if all the authorities are shutting us out. */
-    smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
-    SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
+    const smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
+    SMARTLIST_FOREACH(trusted_dirs, dir_server_t *, ds,
                       download_status_failed(&ds->v2_ns_dl_status, 0));
     directory_get_from_dirserver(conn->base_.purpose, conn->router_purpose,
                                  "all.z", 0 /* don't retry_if_no_servers */);
@@ -1088,7 +1088,7 @@ directory_get_consensus_url(const char *resource)
     smartlist_t *authority_digests = smartlist_new();
 
     SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
-                            trusted_dir_server_t *, ds) {
+                            dir_server_t *, ds) {
         char *hex;
         if (!(ds->type & V3_DIRINFO))
           continue;
@@ -1657,7 +1657,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
 
   if (status_code == 503) {
     routerstatus_t *rs;
-    trusted_dir_server_t *ds;
+    dir_server_t *ds;
     const char *id_digest = conn->identity_digest;
     log_info(LD_DIR,"Received http status code %d (%s) from server "
              "'%s:%d'. I'll try again soon.",
@@ -1665,7 +1665,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
              conn->base_.port);
     if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
       rs->last_dir_503_at = now;
-    if ((ds = router_get_trusteddirserver_by_digest(id_digest)))
+    if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
       ds->fake_status.last_dir_503_at = now;
 
     tor_free(body); tor_free(headers); tor_free(reason);
@@ -1764,7 +1764,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
       source = NS_FROM_DIR_ALL;
       which = smartlist_new();
       SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
-                        trusted_dir_server_t *, ds,
+                        dir_server_t *, ds,
         {
           char *hex = tor_malloc(HEX_DIGEST_LEN+1);
           base16_encode(hex, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN);
@@ -2021,7 +2021,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
   if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) {
     switch (status_code) {
       case 200: {
-          trusted_dir_server_t *ds =
+          dir_server_t *ds =
             router_get_trusteddirserver_by_digest(conn->identity_digest);
           char *rejected_hdr = http_get_header(headers,
                                                "X-Descriptor-Not-New: ");
@@ -3597,13 +3597,13 @@ dir_networkstatus_download_failed(smartlist_t *failed, int status_code)
     return;
   SMARTLIST_FOREACH_BEGIN(failed, const char *, fp) {
     char digest[DIGEST_LEN];
-    trusted_dir_server_t *dir;
+    dir_server_t *dir;
     if (base16_decode(digest, DIGEST_LEN, fp, strlen(fp))<0) {
       log_warn(LD_BUG, "Called with bad fingerprint in list: %s",
                escaped(fp));
       continue;
     }
-    dir = router_get_trusteddirserver_by_digest(digest);
+    dir = router_get_fallback_dirserver_by_digest(digest);
 
     if (dir)
       download_status_failed(&dir->v2_ns_dl_status, status_code);

src/or/dirserv.c

@@ -1471,7 +1471,6 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
                                     time_t published)
 {
   cached_dir_t *d, *old_d;
-  smartlist_t *trusted_dirs;
   if (!cached_v2_networkstatus)
     cached_v2_networkstatus = digestmap_new();
 
@@ -1494,9 +1493,9 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus,
   }
 
   /* Now purge old entries. */
-  trusted_dirs = router_get_trusted_dir_servers();
+
   if (digestmap_size(cached_v2_networkstatus) >
-      smartlist_len(trusted_dirs) + MAX_UNTRUSTED_NETWORKSTATUSES) {
+      get_n_authorities(V2_DIRINFO) + MAX_UNTRUSTED_NETWORKSTATUSES) {
     /* We need to remove the oldest untrusted networkstatus. */
     const char *oldest = NULL;
     time_t oldest_published = TIME_MAX;
@@ -3127,7 +3126,7 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result,
       }
     } else {
       SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
-                  trusted_dir_server_t *, ds,
+                  dir_server_t *, ds,
                   if (ds->type & V2_DIRINFO)
                     smartlist_add(result, tor_memdup(ds->digest, DIGEST_LEN)));
     }

src/or/dirvote.c

@@ -2787,7 +2787,7 @@ dirvote_fetch_missing_votes(void)
   char *resource;
 
   SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
-                          trusted_dir_server_t *, ds) {
+                          dir_server_t *, ds) {
       if (!(ds->type & V3_DIRINFO))
         continue;
       if (!dirvote_get_vote(ds->v3_identity_digest,
@@ -2905,7 +2905,7 @@ list_v3_auth_ids(void)
   smartlist_t *known_v3_keys = smartlist_new();
   char *keys;
   SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
-                    trusted_dir_server_t *, ds,
+                    dir_server_t *, ds,
     if ((ds->type & V3_DIRINFO) &&
         !tor_digest_is_zero(ds->v3_identity_digest))
       smartlist_add(known_v3_keys,
@@ -2926,7 +2926,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
 {
   networkstatus_t *vote;
   networkstatus_voter_info_t *vi;
-  trusted_dir_server_t *ds;
+  dir_server_t *ds;
   pending_vote_t *pending_vote = NULL;
   const char *end_of_vote = NULL;
   int any_failed = 0;

src/or/networkstatus.c

@@ -219,8 +219,6 @@ router_reload_consensus_networkstatus(void)
 {
   char *filename;
   char *s;
-  struct stat st;
-  const or_options_t *options = get_options();
   const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
   int flav;
 
@@ -263,25 +261,6 @@ router_reload_consensus_networkstatus(void)
     tor_free(filename);
   }
 
-  if (!current_consensus ||
-      (stat(options->FallbackNetworkstatusFile, &st)==0 &&
-       st.st_mtime > current_consensus->valid_after)) {
-    s = read_file_to_str(options->FallbackNetworkstatusFile,
-                         RFTS_IGNORE_MISSING, NULL);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, "ns",
-                                              flags|NSSET_ACCEPT_OBSOLETE)) {
-        log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"",
-                 options->FallbackNetworkstatusFile);
-      } else {
-        log_notice(LD_FS,
-                   "Loaded fallback consensus networkstatus from \"%s\"",
-                   options->FallbackNetworkstatusFile);
-      }
-      tor_free(s);
-    }
-  }
-
   if (!current_consensus) {
     if (!named_server_map)
       named_server_map = strmap_new();
@@ -565,7 +544,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus,
 
   /* Now see whether we're missing any voters entirely. */
   SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
-                    trusted_dir_server_t *, ds,
+                    dir_server_t *, ds,
     {
       if ((ds->type & V3_DIRINFO) &&
           !networkstatus_get_voter_by_id(consensus, ds->v3_identity_digest))
@@ -597,7 +576,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus,
                  voter->contact?voter->contact:"n/a",
                  hex_str(voter->identity_digest, DIGEST_LEN));
       });
-    SMARTLIST_FOREACH(missing_authorities, trusted_dir_server_t *, ds,
+    SMARTLIST_FOREACH(missing_authorities, dir_server_t *, ds,
       {
         log(severity, LD_DIR, "Consensus does not include configured "
                  "authority '%s' at %s:%d (identity %s)",
@@ -739,7 +718,7 @@ router_set_networkstatus_v2(const char *s, time_t arrived_at,
   int i, found;
   time_t now;
   int skewed = 0;
-  trusted_dir_server_t *trusted_dir = NULL;
+  dir_server_t *trusted_dir = NULL;
   const char *source_desc = NULL;
   char fp[HEX_DIGEST_LEN+1];
   char published[ISO_TIME_LEN+1];
@@ -1144,7 +1123,7 @@ update_v2_networkstatus_cache_downloads(time_t now)
 
   if (authority) {
     /* An authority launches a separate connection for everybody. */
-    SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds)
+    SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds)
       {
          char resource[HEX_DIGEST_LEN+6]; /* fp/hexdigit.z\0 */
          tor_addr_t addr;
@@ -1674,9 +1653,6 @@ networkstatus_set_current_consensus(const char *consensus,
 
   if (from_cache && !accept_obsolete &&
       c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) {
-    /* XXXX If we try to make fallbackconsensus work again, we should
-     * consider taking this out. Until then, believing obsolete consensuses
-     * is causing more harm than good. See also bug 887. */
     log_info(LD_DIR, "Loaded an expired consensus. Discarding.");
     goto done;
   }
@@ -2042,7 +2018,7 @@ void
 routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
                                                    int reset_failures)
 {
-  trusted_dir_server_t *ds;
+  dir_server_t *ds;
   const or_options_t *options = get_options();
   int authdir = authdir_mode_v2(options) || authdir_mode_v3(options);
   networkstatus_t *ns = current_consensus;
@@ -2062,7 +2038,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
     /* We have a routerstatus for this router. */
     const char *digest = router->cache_info.identity_digest;
 
-    ds = router_get_trusteddirserver_by_digest(digest);
+    ds = router_get_fallback_dirserver_by_digest(digest);
 
     /* Is it the same descriptor, or only the same identity? */
     if (tor_memeq(router->cache_info.signed_descriptor_digest,

src/or/nodelist.c

@@ -1167,8 +1167,13 @@ router_set_status(const char *digest, int up)
   node_t *node;
   tor_assert(digest);
 
+  SMARTLIST_FOREACH(router_get_fallback_dir_servers(),
+                    dir_server_t *, d,
+                    if (tor_memeq(d->digest, digest, DIGEST_LEN))
+                      d->is_running = up);
+
   SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
-                    trusted_dir_server_t *, d,
+                    dir_server_t *, d,
                     if (tor_memeq(d->digest, digest, DIGEST_LEN))
                       d->is_running = up);
 

src/or/or.h

@@ -2517,6 +2517,8 @@ typedef enum {
   MICRODESC_DIRINFO=1 << 6,
 } dirinfo_type_t;
 
+#define ALL_DIRINFO ((dirinfo_type_t)((1<<7)-1))
+
 #define CRYPT_PATH_MAGIC 0x70127012u
 
 /** Holds accounting information for a single step in the layered encryption
@@ -3439,7 +3441,14 @@ typedef struct {
   /** List of configuration lines for replacement directory authorities.
    * If you just want to replace one class of authority at a time,
    * use the "Alternate*Authority" options below instead. */
-  config_line_t *DirServers;
+  config_line_t *DirAuthorities;
+
+  /** List of fallback directory servers */
+  config_line_t *FallbackDir;
+
+  /** Weight to apply to all directory authority rates if considering them
+   * along with fallbackdirs */
+  double DirAuthorityFallbackRate;
 
   /** If set, use these main (currently v3) directory authorities and
    * not the default ones. */
@@ -3708,10 +3717,6 @@ typedef struct {
    * of certain configuration options. */
   int TestingTorNetwork;
 
-  /** File to check for a consensus networkstatus, if we don't have one
-   * cached. */
-  char *FallbackNetworkstatusFile;
-
   /** If true, and we have GeoIP data, and we're a bridge, keep a per-country
    * count of how many client addresses have contacted us so that we can help
    * the bridge authority guess which countries have blocked access to us. */
@@ -4501,19 +4506,23 @@ typedef struct rend_cache_entry_t {
 
 /********************************* routerlist.c ***************************/
 
-/** Represents information about a single trusted directory server. */
-typedef struct trusted_dir_server_t {
+/** Represents information about a single trusted or fallback directory
+ * server. */
+typedef struct dir_server_t {
   char *description;
   char *nickname;
   char *address; /**< Hostname. */
   uint32_t addr; /**< IPv4 address. */
   uint16_t dir_port; /**< Directory port. */
   uint16_t or_port; /**< OR port: Used for tunneling connections. */
+  double weight; /** Weight used when selecting this node at random */
   char digest[DIGEST_LEN]; /**< Digest of identity key. */
   char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only,
                                         * high-security) identity key. */
 
   unsigned int is_running:1; /**< True iff we think this server is running. */
+  unsigned int is_authority:1; /**< True iff this is a directory authority
+                                * of some kind. */
 
   /** True iff this server has accepted the most recent server descriptor
    * we tried to upload to it. */
@@ -4532,7 +4541,7 @@ typedef struct trusted_dir_server_t {
                                * as a routerstatus_t.  Not updated by the
                                * router-status management code!
                                **/
-} trusted_dir_server_t;
+} dir_server_t;
 
 #define ROUTER_REQUIRED_MIN_BANDWIDTH (20*1024)
 

src/or/router.c

@@ -517,7 +517,7 @@ init_keys(void)
   const or_options_t *options = get_options();
   dirinfo_type_t type;
   time_t now = time(NULL);
-  trusted_dir_server_t *ds;
+  dir_server_t *ds;
   int v3_digest_set = 0;
   authority_cert_t *cert = NULL;
 
@@ -732,17 +732,18 @@ init_keys(void)
 
   ds = router_get_trusteddirserver_by_digest(digest);
   if (!ds) {
-    ds = add_trusted_dir_server(options->Nickname, NULL,
+    ds = trusted_dir_server_new(options->Nickname, NULL,
                                 router_get_advertised_dir_port(options, 0),
                                 router_get_advertised_or_port(options),
                                 digest,
                                 v3_digest,
-                                type);
+                                type, 0.0);
     if (!ds) {
       log_err(LD_GENERAL,"We want to be a directory authority, but we "
               "couldn't add ourselves to the authority list. Failing.");
       return -1;
     }
+    dir_server_add(ds);
   }
   if (ds->type != type) {
     log_warn(LD_DIR,  "Configured authority type does not match authority "

src/or/routerlist.c

@@ -45,11 +45,15 @@
 static const routerstatus_t *router_pick_directory_server_impl(
                                            dirinfo_type_t auth, int flags);
 static const routerstatus_t *router_pick_trusteddirserver_impl(
-                          dirinfo_type_t auth, int flags, int *n_busy_out);
-static void mark_all_trusteddirservers_up(void);
+                const smartlist_t *sourcelist, dirinfo_type_t auth,
+                int flags, int *n_busy_out);
+static const routerstatus_t *router_pick_dirserver_generic(
+                              smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags);
+static void mark_all_dirservers_up(smartlist_t *server_list);
 static int router_nickname_matches(const routerinfo_t *router,
                                    const char *nickname);
-static void trusted_dir_server_free(trusted_dir_server_t *ds);
+static void dir_server_free(dir_server_t *ds);
 static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
 static const char *signed_descriptor_get_body_impl(
                                               const signed_descriptor_t *desc,
@@ -72,9 +76,12 @@ DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
 
 /****************************************************************************/
 
-/** Global list of a trusted_dir_server_t object for each trusted directory
- * server. */
+/** Global list of a dir_server_t object for each directory
+ * authority. */
 static smartlist_t *trusted_dir_servers = NULL;
+/** Global list of dir_server_t objects for all directory authorities
+ * and all fallback directory servers. */
+static smartlist_t *fallback_dir_servers = NULL;
 
 /** List of for a given authority, and download status for latest certificate.
  */
@@ -119,7 +126,7 @@ get_n_authorities(dirinfo_type_t type)
   int n = 0;
   if (!trusted_dir_servers)
     return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
                     if (ds->type & type)
                       ++n);
   return n;
@@ -190,7 +197,7 @@ int
 trusted_dirs_load_certs_from_string(const char *contents, int from_store,
                                     int flush)
 {
-  trusted_dir_server_t *ds;
+  dir_server_t *ds;
   const char *s, *eos;
   int failure_code = 0;
 
@@ -530,7 +537,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now)
       } SMARTLIST_FOREACH_END(sig);
     } SMARTLIST_FOREACH_END(voter);
   }
-  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) {
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) {
     int found = 0;
     if (!(ds->type & V3_DIRINFO))
       continue;
@@ -914,11 +921,11 @@ router_reload_router_list(void)
   return 0;
 }
 
-/** Return a smartlist containing a list of trusted_dir_server_t * for all
+/** Return a smartlist containing a list of dir_server_t * for all
  * known trusted dirservers.  Callers must not modify the list or its
  * contents.
  */
-smartlist_t *
+const smartlist_t *
 router_get_trusted_dir_servers(void)
 {
   if (!trusted_dir_servers)
@@ -927,6 +934,15 @@ router_get_trusted_dir_servers(void)
   return trusted_dir_servers;
 }
 
+const smartlist_t *
+router_get_fallback_dir_servers(void)
+{
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  return fallback_dir_servers;
+}
+
 /** Try to find a running dirserver that supports operations of <b>type</b>.
  *
  * If there are no running dirservers in our routerlist and the
@@ -960,7 +976,7 @@ router_pick_directory_server(dirinfo_type_t type, int flags)
            "No reachable router entries for dirservers. "
            "Trying them all again.");
   /* mark all authdirservers as up again */
-  mark_all_trusteddirservers_up();
+  mark_all_dirservers_up(fallback_dir_servers);
   /* try again */
   choice = router_pick_directory_server_impl(type, flags);
   return choice;
@@ -1007,16 +1023,16 @@ router_get_my_share_of_directory_requests(double *v2_share_out,
   return 0;
 }
 
-/** Return the trusted_dir_server_t for the directory authority whose identity
+/** Return the dir_server_t for the directory authority whose identity
  * key hashes to <b>digest</b>, or NULL if no such authority is known.
  */
-trusted_dir_server_t *
+dir_server_t *
 router_get_trusteddirserver_by_digest(const char *digest)
 {
   if (!trusted_dir_servers)
     return NULL;
 
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
      {
        if (tor_memeq(ds->digest, digest, DIGEST_LEN))
          return ds;
@@ -1025,17 +1041,35 @@ router_get_trusteddirserver_by_digest(const char *digest)
   return NULL;
 }
 
-/** Return the trusted_dir_server_t for the directory authority whose
+/** Return the dir_server_t for the fallback dirserver whose identity
+ * key hashes to <b>digest</b>, or NULL if no such authority is known.
+ */
+dir_server_t *
+router_get_fallback_dirserver_by_digest(const char *digest)
+{
+  if (!trusted_dir_servers)
+    return NULL;
+
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
+     {
+       if (tor_memeq(ds->digest, digest, DIGEST_LEN))
+         return ds;
+     });
+
+  return NULL;
+}
+
+/** Return the dir_server_t for the directory authority whose
  * v3 identity key hashes to <b>digest</b>, or NULL if no such authority
  * is known.
  */
-trusted_dir_server_t *
+dir_server_t *
 trusteddirserver_get_by_v3_auth_digest(const char *digest)
 {
   if (!trusted_dir_servers)
     return NULL;
 
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds,
      {
        if (tor_memeq(ds->v3_identity_digest, digest, DIGEST_LEN) &&
            (ds->type & V3_DIRINFO))
@@ -1045,18 +1079,37 @@ trusteddirserver_get_by_v3_auth_digest(const char *digest)
   return NULL;
 }
 
-/** Try to find a running trusted dirserver.  Flags are as for
+/** Try to find a running directory authority. Flags are as for
  * router_pick_directory_server.
  */
 const routerstatus_t *
 router_pick_trusteddirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(trusted_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory Flags are as for
+ * router_pick_directory_server.
+ */
+const routerstatus_t *
+router_pick_fallback_dirserver(dirinfo_type_t type, int flags)
+{
+  return router_pick_dirserver_generic(fallback_dir_servers, type, flags);
+}
+
+/** Try to find a running fallback directory Flags are as for
+ * router_pick_directory_server.
+ */
+static const routerstatus_t *
+router_pick_dirserver_generic(smartlist_t *sourcelist,
+                              dirinfo_type_t type, int flags)
 {
   const routerstatus_t *choice;
   int busy = 0;
   if (get_options()->PreferTunneledDirConns)
     flags |= PDS_PREFER_TUNNELED_DIR_CONNS_;
 
-  choice = router_pick_trusteddirserver_impl(type, flags, &busy);
+  choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy);
   if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
     return choice;
   if (busy) {
@@ -1069,9 +1122,9 @@ router_pick_trusteddirserver(dirinfo_type_t type, int flags)
   }
 
   log_info(LD_DIR,
-           "No trusted dirservers are reachable. Trying them all again.");
-  mark_all_trusteddirservers_up();
-  return router_pick_trusteddirserver_impl(type, flags, NULL);
+           "No dirservers are reachable. Trying them all again.");
+  mark_all_dirservers_up(sourcelist);
+  return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL);
 }
 
 /** How long do we avoid using a directory server after it's given us a 503? */
@@ -1196,11 +1249,36 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags)
   return result ? result->rs : NULL;
 }
 
-/** Choose randomly from among the trusted dirservers that are up.  Flags
- * are as for router_pick_directory_server_impl().
+/** Pick a random element from a list of dir_server_t, weighting by their
+ * <b>weight</b> field. */
+static const dir_server_t *
+dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight)
+{
+  int n = smartlist_len(servers);
+  int i;
+  u64_dbl_t *weights;
+  const dir_server_t *ds;
+
+  weights = tor_malloc(sizeof(u64_dbl_t) * n);
+  for (i = 0; i < n; ++i) {
+    ds = smartlist_get(servers, i);
+    weights[i].dbl = ds->weight;
+    if (ds->is_authority)
+      weights[i].dbl *= authority_weight;
+  }
+
+  scale_array_elements_to_u64(weights, n, NULL);
+  i = choose_array_element_by_weight(weights, n);
+  tor_free(weights);
+  return smartlist_get(servers, i);
+}
+
+/** Choose randomly from among the dir_server_ts in sourcelist that
+ * are up. Flags are as for router_pick_directory_server_impl().
  */
 static const routerstatus_t *
-router_pick_trusteddirserver_impl(dirinfo_type_t type, int flags,
+router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
+                                  dirinfo_type_t type, int flags,
                                   int *n_busy_out)
 {
   const or_options_t *options = get_options();
@@ -1214,10 +1292,13 @@ router_pick_trusteddirserver_impl(dirinfo_type_t type, int flags,
   const int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_);
   const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
   const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH);
+  const double auth_weight = (sourcelist == fallback_dir_servers) ?
+    options->DirAuthorityFallbackRate : 1.0;
+  smartlist_t *pick_from;
   int n_busy = 0;
   int try_excluding = 1, n_excluded = 0;
 
-  if (!trusted_dir_servers)
+  if (!sourcelist)
     return NULL;
 
  retry_without_exclude:
@@ -1227,7 +1308,7 @@ router_pick_trusteddirserver_impl(dirinfo_type_t type, int flags,
   overloaded_direct = smartlist_new();
   overloaded_tunnel = smartlist_new();
 
-  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, d)
+  SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
     {
       int is_overloaded =
           d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now;
@@ -1273,23 +1354,29 @@ router_pick_trusteddirserver_impl(dirinfo_type_t type, int flags,
           d->or_port &&
           (!fascistfirewall ||
            fascist_firewall_allows_address_or(&addr, d->or_port)))
-        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel,
-                      &d->fake_status);
+        smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d);
       else if (!fascistfirewall ||
                fascist_firewall_allows_address_dir(&addr, d->dir_port))
-        smartlist_add(is_overloaded ? overloaded_direct : direct,
-                      &d->fake_status);
+        smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d);
     }
   SMARTLIST_FOREACH_END(d);
 
   if (smartlist_len(tunnel)) {
-    result = smartlist_choose(tunnel);
+    pick_from = tunnel;
   } else if (smartlist_len(overloaded_tunnel)) {
-    result = smartlist_choose(overloaded_tunnel);
+    pick_from = overloaded_tunnel;
   } else if (smartlist_len(direct)) {
-    result = smartlist_choose(direct);
+    pick_from = direct;
   } else {
-    result = smartlist_choose(overloaded_direct);
+    pick_from = overloaded_direct;
+  }
+
+  {
+    const dir_server_t *selection =
+      dirserver_choose_by_weight(pick_from, auth_weight);
+
+    if (selection)
+      result = &selection->fake_status;
   }
 
   if (n_busy_out)
@@ -1311,19 +1398,19 @@ router_pick_trusteddirserver_impl(dirinfo_type_t type, int flags,
   return result;
 }
 
-/** Go through and mark the authoritative dirservers as up. */
+/** Mark as running every dir_server_t in <b>server_list</b>. */
 static void
-mark_all_trusteddirservers_up(void)
+mark_all_dirservers_up(smartlist_t *server_list)
 {
-  SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node, {
-       if (router_digest_is_trusted_dir(node->identity))
-         node->is_running = 1;
-    });
-  if (trusted_dir_servers) {
-    SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, dir) {
+  if (server_list) {
+    SMARTLIST_FOREACH_BEGIN(server_list, dir_server_t *, dir) {
       routerstatus_t *rs;
+      node_t *node;
       dir->is_running = 1;
       download_status_reset(&dir->v2_ns_dl_status);
+      node = node_get_mutable_by_id(dir->digest);
+      if (node)
+        node->is_running = 1;
       rs = router_get_mutable_consensus_status_by_id(dir->digest);
       if (rs) {
         rs->last_dir_503_at = 0;
@@ -1348,7 +1435,7 @@ routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2)
 void
 router_reset_status_download_failures(void)
 {
-  mark_all_trusteddirservers_up();
+  mark_all_dirservers_up(fallback_dir_servers);
 }
 
 /** Given a <b>router</b>, add every node_t in its family (including the
@@ -2185,7 +2272,7 @@ router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
     return 0;
   if (authdir_mode(get_options()) && router_digest_is_me(digest))
     return 1;
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ent,
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
     if (tor_memeq(digest, ent->digest, DIGEST_LEN)) {
       return (!type) || ((type & ent->type) != 0);
     });
@@ -2199,7 +2286,7 @@ router_addr_is_trusted_dir(uint32_t addr)
 {
   if (!trusted_dir_servers)
     return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ent,
+  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
     if (ent->addr == addr)
       return 1;
     );
@@ -2910,12 +2997,10 @@ routerlist_free_all(void)
     smartlist_free(warned_nicknames);
     warned_nicknames = NULL;
   }
-  if (trusted_dir_servers) {
-    SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds,
-                      trusted_dir_server_free(ds));
-    smartlist_free(trusted_dir_servers);
-    trusted_dir_servers = NULL;
-  }
+  clear_dir_servers();
+  smartlist_free(trusted_dir_servers);
+  smartlist_free(fallback_dir_servers);
+  trusted_dir_servers = fallback_dir_servers = NULL;
   if (trusted_dir_certs) {
     DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
       SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
@@ -3722,47 +3807,47 @@ router_exit_policy_rejects_all(const routerinfo_t *router)
   return router->policy_is_reject_star;
 }
 
-/** Add to the list of authoritative directory servers one at
- * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
- * <b>address</b> is NULL, add ourself.  Return the new trusted directory
- * server entry on success or NULL if we couldn't add it. */
-trusted_dir_server_t *
-add_trusted_dir_server(const char *nickname, const char *address,
-                       uint16_t dir_port, uint16_t or_port,
-                       const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type)
+/** Create an directory server at <b>address</b>:<b>port</b>, with OR identity
+ * key <b>digest</b>.  If <b>address</b> is NULL, add ourself.  If
+ * <b>is_authority</b>, this is a directory authority.  Return the new
+ * directory server entry on success or NULL on failure. */
+static dir_server_t *
+dir_server_new(int is_authority,
+               const char *nickname,
+               const tor_addr_t *addr,
+               const char *hostname,
+               uint16_t dir_port, uint16_t or_port,
+               const char *digest, const char *v3_auth_digest,
+               dirinfo_type_t type,
+               double weight)
 {
-  trusted_dir_server_t *ent;
+  dir_server_t *ent;
   uint32_t a;
-  char *hostname = NULL;
-  if (!trusted_dir_servers)
-    trusted_dir_servers = smartlist_new();
+  char *hostname_ = NULL;
 
-  if (!address) { /* The address is us; we should guess. */
-    if (resolve_my_address(LOG_WARN, get_options(), &a, &hostname) < 0) {
-      log_warn(LD_CONFIG,
-               "Couldn't find a suitable address when adding ourself as a "
-               "trusted directory server.");
-      return NULL;
-    }
-  } else {
-    if (tor_lookup_hostname(address, &a)) {
-      log_warn(LD_CONFIG,
-               "Unable to lookup address for directory server at '%s'",
-               address);
-      return NULL;
-    }
-    hostname = tor_strdup(address);
-  }
+  if (weight < 0)
+    return NULL;
+
+  if (tor_addr_family(addr) == AF_INET)
+    a = tor_addr_to_ipv4h(addr);
+  else
+    return NULL; /*XXXX Support IPv6 */
+
+  if (!hostname)
+    hostname_ = tor_dup_addr(addr);
+  else
+    hostname_ = tor_strdup(hostname);
 
-  ent = tor_malloc_zero(sizeof(trusted_dir_server_t));
+  ent = tor_malloc_zero(sizeof(dir_server_t));
   ent->nickname = nickname ? tor_strdup(nickname) : NULL;
-  ent->address = hostname;
+  ent->address = hostname_;
   ent->addr = a;
   ent->dir_port = dir_port;
   ent->or_port = or_port;
   ent->is_running = 1;
+  ent->is_authority = is_authority;
   ent->type = type;
+  ent->weight = weight;
   memcpy(ent->digest, digest, DIGEST_LEN);
   if (v3_auth_digest && (type & V3_DIRINFO))
     memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
@@ -3784,11 +3869,77 @@ add_trusted_dir_server(const char *nickname, const char *address,
   ent->fake_status.dir_port = ent->dir_port;
   ent->fake_status.or_port = ent->or_port;
 
-  smartlist_add(trusted_dir_servers, ent);
-  router_dir_info_changed();
   return ent;
 }
 
+/** Create an authoritative directory server at
+ * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
+ * <b>address</b> is NULL, add ourself.  Return the new trusted directory
+ * server entry on success or NULL if we couldn't add it. */
+dir_server_t *
+trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight)
+{
+  uint32_t a;
+  tor_addr_t addr;
+  char *hostname=NULL;
+  dir_server_t *result;
+
+  if (!address) { /* The address is us; we should guess. */
+    if (resolve_my_address(LOG_WARN, get_options(), &a, &hostname) < 0) {
+      log_warn(LD_CONFIG,
+               "Couldn't find a suitable address when adding ourself as a "
+               "trusted directory server.");
+      return NULL;
+    }
+  } else {
+    if (tor_lookup_hostname(address, &a)) {
+      log_warn(LD_CONFIG,
+               "Unable to lookup address for directory server at '%s'",
+               address);
+      return NULL;
+    }
+    hostname = tor_strdup(address);
+  }
+  tor_addr_from_ipv4h(&addr, a);
+
+  result = dir_server_new(1, nickname, &addr, hostname,
+                          dir_port, or_port, digest,
+                          v3_auth_digest, type, weight);
+  tor_free(hostname);
+  return result;
+}
+
+/** Return a new dir_server_t for a fallback directory server at
+ * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest
+ * <b>id_digest</b> */
+dir_server_t *
+fallback_dir_server_new(const tor_addr_t *addr,
+                        uint16_t dir_port, uint16_t or_port,
+                        const char *id_digest, double weight)
+{
+  return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, id_digest,
+                        NULL, ALL_DIRINFO, weight);
+}
+
+/** Add a directory server to the global list(s). */
+void
+dir_server_add(dir_server_t *ent)
+{
+  if (!trusted_dir_servers)
+    trusted_dir_servers = smartlist_new();
+  if (!fallback_dir_servers)
+    fallback_dir_servers = smartlist_new();
+
+  if (ent->is_authority)
+    smartlist_add(trusted_dir_servers, ent);
+
+  smartlist_add(fallback_dir_servers, ent);
+  router_dir_info_changed();
+}
+
 /** Free storage held in <b>cert</b>. */
 void
 authority_cert_free(authority_cert_t *cert)
@@ -3805,7 +3956,7 @@ authority_cert_free(authority_cert_t *cert)
 
 /** Free storage held in <b>ds</b>. */
 static void
-trusted_dir_server_free(trusted_dir_server_t *ds)
+dir_server_free(dir_server_t *ds)
 {
   if (!ds)
     return;
@@ -3816,13 +3967,18 @@ trusted_dir_server_free(trusted_dir_server_t *ds)
   tor_free(ds);
 }
 
-/** Remove all members from the list of trusted dir servers. */
+/** Remove all members from the list of dir servers. */
 void
-clear_trusted_dir_servers(void)
+clear_dir_servers(void)
 {
+  if (fallback_dir_servers) {
+    SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ent,
+                      dir_server_free(ent));
+    smartlist_clear(fallback_dir_servers);
+  } else {
+    fallback_dir_servers = smartlist_new();
+  }
   if (trusted_dir_servers) {
-    SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ent,
-                      trusted_dir_server_free(ent));
     smartlist_clear(trusted_dir_servers);
   } else {
     trusted_dir_servers = smartlist_new();
@@ -4128,7 +4284,7 @@ update_router_descriptor_cache_downloads_v2(time_t now)
    */
   n_download = 0;
   SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) {
-      trusted_dir_server_t *ds;
+      dir_server_t *ds;
       smartlist_t *dl;
       dl = downloadable[ns_sl_idx] = smartlist_new();
       download_from[ns_sl_idx] = smartlist_new();
@@ -4203,7 +4359,7 @@ update_router_descriptor_cache_downloads_v2(time_t now)
   /* Now, we can actually launch our requests. */
   for (i=0; i<n; ++i) {
     networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i);
-    trusted_dir_server_t *ds =
+    dir_server_t *ds =
       router_get_trusteddirserver_by_digest(ns->identity_digest);
     smartlist_t *dl = download_from[i];
     int pds_flags = PDS_RETRY_IF_NO_SERVERS;
@@ -4256,7 +4412,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
 
   if (is_vote) {
     /* where's it from, so we know whom to ask for descriptors */
-    trusted_dir_server_t *ds;
+    dir_server_t *ds;
     networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0);
     tor_assert(voter);
     ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest);

src/or/routerlist.h

@@ -25,14 +25,19 @@ void authority_cert_dl_failed(const char *id_digest, int status);
 void authority_certs_fetch_missing(networkstatus_t *status, time_t now);
 int router_reload_router_list(void);
 int authority_cert_dl_looks_uncertain(const char *id_digest);
-smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_trusted_dir_servers(void);
+const smartlist_t *router_get_fallback_dir_servers(void);
 
 const routerstatus_t *router_pick_directory_server(dirinfo_type_t type,
                                                    int flags);
-trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
-trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d);
+dir_server_t *router_get_trusteddirserver_by_digest(const char *d);
+dir_server_t *router_get_fallback_dirserver_by_digest(
+                                                   const char *digest);
+dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d);
 const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type,
                                                    int flags);
+const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type,
+                                                     int flags);
 int router_get_my_share_of_directory_requests(double *v2_share_out,
                                               double *v3_share_out);
 void router_reset_status_download_failures(void);
@@ -127,13 +132,18 @@ void router_load_extrainfo_from_string(const char *s, const char *eos,
 void routerlist_retry_directory_downloads(time_t now);
 
 int router_exit_policy_rejects_all(const routerinfo_t *router);
-trusted_dir_server_t *add_trusted_dir_server(const char *nickname,
-                           const char *address,
-                           uint16_t dir_port, uint16_t or_port,
-                           const char *digest, const char *v3_auth_digest,
-                           dirinfo_type_t type);
+
+dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
+                       uint16_t dir_port, uint16_t or_port,
+                       const char *digest, const char *v3_auth_digest,
+                       dirinfo_type_t type, double weight);
+dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
+                                      uint16_t dir_port, uint16_t or_port,
+                                      const char *id_digest, double weight);
+void dir_server_add(dir_server_t *ent);
+
 void authority_cert_free(authority_cert_t *cert);
-void clear_trusted_dir_servers(void);
+void clear_dir_servers(void);
 int any_trusted_dir_is_v1_authority(void);
 void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
                                                   networkstatus_t *consensus);