More stuff for new directories.

- Distinguish v1 authorities (all currently trusted directories) from
  v2 authorities (all trusted directories).
    - Add configuration option for which dirs are v1 authories.
    - Add configuration option for whether to be a v1 authority.
    - Make trusted dirserver selection functions take options to
      choose which functionality we need.
    - Remove option when getting directory cache to see whether they
      support running-routers; they all do now.  Replace it with one
      to see whether caches support v2 stuff.

- Parse, cache, and serve network-status objects properly.

- Serve compressed groups of router descriptors.  The compression logic
  here could be more memory-efficient.

-


svn:r4911

doc/TODO

@@ -118,12 +118,12 @@ N     . Dirservers publish compressed network-status objects.
         - Support retrieving several-at-once
 N     . Everyone downloads network-status objects
         - From all directories, round-robin
-        . Parse them
-        - Cache them, reload on restart
+        o Parse them
+        o Cache them, reload on restart
         o Serve cached directories
-N     . Directories expose individual descriptors
+      o Directories expose individual descriptors
         X By 'if-newer-than' (Does the spec require this??)
-        - Support compression.
+        o Support compression.
 N     - Alice acts on network-status objects
         - Alice downloads descriptors as needed.
         - Alice sets descriptor status from networks-status

src/or/config.c

@@ -102,6 +102,8 @@ static config_var_t _option_vars[] = {
   VAR("AllowUnverifiedNodes",CSV,      AllowUnverifiedNodes, "middle,rendezvous"),
   VAR("AssumeReachable",     BOOL,     AssumeReachable,      "0"),
   VAR("AuthoritativeDirectory",BOOL,   AuthoritativeDir,     "0"),
+  /* XXXX 011 change this default on 0.1.1.x */
+  VAR("V1AuthoritativeDirectory",BOOL, V1AuthoritativeDir,   "1"),
   VAR("BandwidthBurst",      MEMUNIT,  BandwidthBurst,       "5 MB"),
   VAR("BandwidthRate",       MEMUNIT,  BandwidthRate,        "2 MB"),
   VAR("ClientOnly",          BOOL,     ClientOnly,           "0"),
@@ -1140,13 +1142,13 @@ add_default_trusted_dirservers(or_options_t *options)
 {
   /* moria1 */
   config_line_append(&options->DirServers, "DirServer",
-       "18.244.0.188:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441");
+     "v1 18.244.0.188:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441");
   /* moria2 */
   config_line_append(&options->DirServers, "DirServer",
-         "18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF");
+     "v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF");
   /* tor26 */
   config_line_append(&options->DirServers, "DirServer",
-     "86.59.5.130:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D");
+     "v1 86.59.5.130:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D");
 //  "tor.noreply.org:9030 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D");
 }
 
@@ -2753,6 +2755,15 @@ parse_dir_server_line(const char *line, int validate_only)
   char *addrport, *address=NULL;
   uint16_t port;
   char digest[DIGEST_LEN];
+  int supports_v1 = 1; /*XXXX011 change default when clients support v2. */
+
+  while (TOR_ISSPACE(*line))
+    ++line;
+
+  if (!strcmpstart(line, "v1 ")) {
+    line += 3;
+    supports_v1 = 1;
+  }
 
   items = smartlist_create();
   smartlist_split_string(items, line, NULL,
@@ -2785,7 +2796,7 @@ parse_dir_server_line(const char *line, int validate_only)
   if (!validate_only) {
     log_fn(LOG_DEBUG, "Trusted dirserver at %s:%d (%s)", address, (int)port,
            (char*)smartlist_get(items,1));
-    add_trusted_dir_server(address, port, digest);
+    add_trusted_dir_server(address, port, digest, supports_v1);
   }
 
   r = 0;

src/or/directory.c

@@ -170,26 +170,25 @@ directory_get_from_dirserver(uint8_t purpose, const char *resource,
   if (directconn) {
     if (fetch_fresh_first) {
       /* only ask authdirservers, and don't ask myself */
-      ds = router_pick_trusteddirserver(1, fascistfirewall,
+      ds = router_pick_trusteddirserver(1, 1, fascistfirewall,
                                         retry_if_no_servers);
     }
     if (!ds) {
       /* anybody with a non-zero dirport will do */
-      r = router_pick_directory_server(1, fascistfirewall,
-                                purpose==DIR_PURPOSE_FETCH_RUNNING_LIST,
+      r = router_pick_directory_server(1, fascistfirewall, 0,
                                        retry_if_no_servers);
       if (!r) {
         log_fn(LOG_INFO, "No router found for %s; falling back to dirserver list",
                purpose == DIR_PURPOSE_FETCH_RUNNING_LIST
                ? "status list" : "directory");
-        ds = router_pick_trusteddirserver(1, fascistfirewall,
+        ds = router_pick_trusteddirserver(1, 1, fascistfirewall,
                                           retry_if_no_servers);
       }
     }
   } else { // (purpose == DIR_PURPOSE_FETCH_RENDDESC)
     /* only ask authdirservers, any of them will do */
     /* Never use fascistfirewall; we're going via Tor. */
-    ds = router_pick_trusteddirserver(0, 0, retry_if_no_servers);
+    ds = router_pick_trusteddirserver(0, 0, 0, retry_if_no_servers);
   }
 
   if (r)
@@ -1062,24 +1061,39 @@ directory_handle_command_get(connection_t *conn, char *headers,
     /* v2 network status fetch. */
     size_t url_len = strlen(url);
     int deflated = !strcmp(url+url_len-2, ".z");
+    smartlist_t *dir_objs = smartlist_create();
     const char *key = url + strlen("/tor/status/");
     if (deflated)
       url[url_len-2] = '\0';
-    dlen = dirserv_get_networkstatus_v2(&cp, key, deflated);
+    if (dirserv_get_networkstatus_v2(dir_objs, key)) {
+      smartlist_free(dir_objs);
+      return 0;
+    }
     tor_free(url);
-    if (!dlen) { /* we failed to create/cache cp */
+    if (!smartlist_len(dir_objs)) { /* we failed to create/cache cp */
       write_http_status_line(conn, 503, "Network status object unavailable");
+      smartlist_free(dir_objs);
       /* try to get a new one now */
       // XXXX NM
       return 0;
     }
+    dlen = 0;
+    SMARTLIST_FOREACH(dir_objs, cached_dir_t *, d,
+                      dlen += deflated?d->dir_z_len:d->dir_len);
     format_rfc1123_time(date, time(NULL));
     tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\nContent-Encoding: %s\r\n\r\n",
                  date,
                  (int)dlen,
                  deflated?"deflate":"identity");
     connection_write_to_buf(tmp, strlen(tmp), conn);
-    connection_write_to_buf(cp, strlen(cp), conn);
+    SMARTLIST_FOREACH(dir_objs, cached_dir_t *, d,
+       {
+         if (deflated)
+           connection_write_to_buf(d->dir_z, d->dir_z_len, conn);
+         else
+           connection_write_to_buf(d->dir, d->dir_len, conn);
+       });
+    smartlist_free(dir_objs);
     return 0;
   }
 
@@ -1098,15 +1112,42 @@ directory_handle_command_get(connection_t *conn, char *headers,
       format_rfc1123_time(date, time(NULL));
       SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
                         len += ri->signed_descriptor_len);
-      /* XXXX We need to support deflate here. */
-      tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: application/octet-stream\r\n\r\n",
-                   date,
-                   (int)len);
-      connection_write_to_buf(tmp, strlen(tmp), conn);
-      SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
-                        connection_write_to_buf(ri->signed_descriptor,
-                                                ri->signed_descriptor_len,
-                                                conn));
+      if (deflated) {
+        size_t compressed_len;
+        char *compressed;
+        char *inp = tor_malloc(len+smartlist_len(descs)+1);
+        char *cp = inp;
+        SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
+           {
+             memcpy(cp, ri->signed_descriptor,
+                    ri->signed_descriptor_len);
+             cp += ri->signed_descriptor_len;
+             *cp++ = '\n';
+           });
+        *cp = '\0';
+        /* XXXX This could be way more efficiently handled. */
+        if (tor_gzip_compress(&compressed, &compressed_len,
+                              inp, cp-inp, ZLIB_METHOD)<0){
+          tor_free(cp);
+          smartlist_free(descs);
+          return -1;
+        }
+        tor_free(cp);
+        tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: application/octet-stream\r\n\r\n",
+                     date,
+                     (int)compressed_len);
+        connection_write_to_buf(tmp, strlen(tmp), conn);
+        connection_write_to_buf(compressed, compressed_len, conn);
+      } else {
+        tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: application/octet-stream\r\n\r\n",
+                     date,
+                     (int)len);
+        connection_write_to_buf(tmp, strlen(tmp), conn);
+        SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
+                          connection_write_to_buf(ri->signed_descriptor,
+                                                  ri->signed_descriptor_len,
+                                                  conn));
+      }
     }
     smartlist_free(descs);
     return 0;

src/or/dirserv.c

@@ -669,16 +669,6 @@ dirserv_dump_directory_to_string(char **dir_out,
   return -1;
 }
 
-/** A cached_dir_t represents a cacheable directory object, along with its
- * compressed form. */
-typedef struct cached_dir_t {
-  char *dir; /**< Contents of this object */
-  char *dir_z; /**< Compressed contents of this object. */
-  size_t dir_len; /**< Length of <b>dir</b> */
-  size_t dir_z_len; /**< Length of <b>dir_z</b> */
-  time_t published; /**< When was this object published */
-} cached_dir_t;
-
 /** Most recently generated encoded signed directory. (auth dirservers only.)*/
 static cached_dir_t the_directory = { NULL, NULL, 0, 0, 0 };
 
@@ -768,7 +758,6 @@ dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
                                     time_t published)
 {
   cached_dir_t *d;
-  char fname[512];
   if (!cached_v2_networkstatus)
     cached_v2_networkstatus = strmap_new();
 
@@ -781,14 +770,33 @@ dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
 
   tor_assert(d);
   set_cached_dir(d, tor_strdup(directory), published);
+}
 
-  if (!d->dir)
-    return;
+static cached_dir_t *
+dirserv_pick_cached_dir_obj(cached_dir_t *cache_src,
+                            cached_dir_t *auth_src,
+                            time_t dirty, int (*regenerate)(void),
+                            const char *name,
+                            int is_v1_object)
+{
+  int authority = get_options()->AuthoritativeDir &&
+    (!is_v1_object || get_options()->V1AuthoritativeDir);
 
-  tor_snprintf(fname,sizeof(fname), "%s/cached-status/%s",
-               get_options()->DataDirectory, fp);
-  if (write_str_to_file(fname, d->dir, 0)<0) {
-    log_fn(LOG_NOTICE, "Couldn't write cached network status to disk. Ignoring.");
+  if (!authority) {
+    return cache_src;
+  } else {
+    /* We're authoritative. */
+    if (regenerate != NULL) {
+      if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) {
+        if (regenerate()) {
+          log_fn(LOG_ERR, "Couldn't generate %s?", name);
+          exit(1);
+        }
+      } else {
+        log_fn(LOG_INFO, "The %s is still clean; reusing.", name);
+      }
+    }
+    return auth_src ? auth_src : cache_src;
   }
 }
 
@@ -798,30 +806,22 @@ dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
  * DIR_REGEN_SLACK_TIME seconds, call <b>regenerate</b>() to make a fresh one.
  * Yields the compressed version of the directory object if <b>compress</b> is
  * set; otherwise return the uncompressed version.  (In either case, sets
- * *<b>out</b> and returns the size of the buffer in *<b>out</b>. */
+ * *<b>out</b> and returns the size of the buffer in *<b>out</b>.
+ *
+ * DOCDOC is_v1_object
+ **/
 static size_t
 dirserv_get_obj(const char **out, int compress,
                 cached_dir_t *cache_src,
                 cached_dir_t *auth_src,
                 time_t dirty, int (*regenerate)(void),
-                const char *name)
+                const char *name,
+                int is_v1_object)
 {
-  cached_dir_t *d;
-  if (!get_options()->AuthoritativeDir || !auth_src) {
-    d = cache_src;
-  } else {
-    if (regenerate != NULL) {
-      if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) {
-        if (regenerate()) {
-          log_fn(LOG_ERR, "Couldn't generate %s?", name);
-          exit(1);
-        }
-      } else {
-        log_fn(LOG_INFO, "The %s is still clean; reusing.", name);
-      }
-    }
-    d = auth_src;
-  }
+  cached_dir_t *d = dirserv_pick_cached_dir_obj(
+      cache_src, auth_src,
+      dirty, regenerate, name, is_v1_object);
+
   if (!d)
     return 0;
   *out = compress ? d->dir_z : d->dir;
@@ -843,7 +843,7 @@ dirserv_get_directory(const char **directory, int compress)
                          &cached_directory, &the_directory,
                          the_directory_is_dirty,
                          dirserv_regenerate_directory,
-                         "server directory");
+                         "server directory", 1);
 }
 
 /**
@@ -938,7 +938,7 @@ dirserv_get_runningrouters(const char **rr, int compress)
                          &cached_runningrouters, &the_runningrouters,
                          runningrouters_is_dirty,
                          generate_runningrouters,
-                         "v1 network status list");
+                         "v1 network status list", 1);
 }
 
 /** Return true iff <b>ri</b> is "useful as an exit node." */
@@ -1132,8 +1132,7 @@ generate_v2_networkstatus(void)
   set_cached_dir(&the_v2_networkstatus, status, time(NULL));
   status = NULL; /* So it doesn't get double-freed. */
   the_v2_networkstatus_is_dirty = 0;
-  dirserv_set_cached_networkstatus_v2(the_v2_networkstatus.dir,
-                                      fingerprint, time(NULL));
+  router_set_networkstatus(the_v2_networkstatus.dir, time(NULL), 0);
 
   r = 0;
  done:
@@ -1153,27 +1152,43 @@ generate_v2_networkstatus(void)
  * nothing was found; otherwise set *<b>directory</b> to the matching network
  * status and return its length.
  */
-size_t
-dirserv_get_networkstatus_v2(const char **directory, const char *key,
-                             int compress)
+int
+dirserv_get_networkstatus_v2(smartlist_t *result,
+                             const char *key)
 {
-  *directory = NULL;
+  tor_assert(result);
+
   if (!(strcmp(key,"authority"))) {
     if (get_options()->AuthoritativeDir) {
-      return dirserv_get_obj(directory, compress, NULL,
-                             &the_v2_networkstatus,
-                             the_v2_networkstatus_is_dirty,
-                             generate_v2_networkstatus,
-                             "network status list");
+      cached_dir_t *d =
+        dirserv_pick_cached_dir_obj(NULL,
+                                    &the_v2_networkstatus,
+                                    the_v2_networkstatus_is_dirty,
+                                    generate_v2_networkstatus,
+                                    "network status list", 0);
+      if (d)
+        smartlist_add(result, d);
     }
   } else if (!strcmp(key, "all")) {
-    // XXXX NM
-    return dirserv_get_networkstatus_v2(directory, "authority", compress);
-  } else if (strlen(key)==HEX_DIGEST_LEN) {
-    cached_dir_t *cached = strmap_get(cached_v2_networkstatus, key);
-    if (cached)
-      return dirserv_get_obj(directory, compress, cached, NULL, 0, NULL,
-                             "cached network status");
+    strmap_iter_t *iter = strmap_iter_init(cached_v2_networkstatus);
+    while (!strmap_iter_done(iter)) {
+      const char *fp;
+      void *val;
+      strmap_iter_get(iter, &fp, &val);
+      smartlist_add(result, val);
+    }
+  } else if (!strcmpstart(key, "fp/")) {
+    smartlist_t *hexdigests = smartlist_create();
+    smartlist_split_string(hexdigests, key+3, "+", 0, 0);
+    SMARTLIST_FOREACH(hexdigests, char *, cp,
+        {
+          cached_dir_t *cached;
+          tor_strlower(cp);
+          /* XXXX special-case own key? */
+          cached = strmap_get(cached_v2_networkstatus, cp);
+          if (cached)
+            smartlist_add(result, cached);
+        });
   }
   return 0;
 }

src/or/main.c

@@ -711,7 +711,7 @@ run_scheduled_events(time_t now)
   }
 
   if (time_to_fetch_running_routers < now) {
-    if (!authdir_mode(options)) {
+    if (!authdir_mode(options) || !options->V1AuthoritativeDir) {
       directory_get_from_dirserver(DIR_PURPOSE_FETCH_RUNNING_LIST, NULL, 1);
     }
     time_to_fetch_running_routers = now + get_status_fetch_period(options);
@@ -972,6 +972,9 @@ do_main_loop(void)
   if (router_reload_router_list()) {
     return -1;
   }
+  if (router_reload_networkstatus()) {
+    return -1;
+  }
 
   if (authdir_mode(get_options())) {
     /* the directory is already here, run startup things */

src/or/or.h

@@ -705,6 +705,17 @@ typedef struct addr_policy_t {
   struct addr_policy_t *next; /**< Next rule in list. */
 } addr_policy_t;
 
+
+/** A cached_dir_t represents a cacheable directory object, along with its
+ * compressed form. */
+typedef struct cached_dir_t {
+  char *dir; /**< Contents of this object */
+  char *dir_z; /**< Compressed contents of this object. */
+  size_t dir_len; /**< Length of <b>dir</b> */
+  size_t dir_z_len; /**< Length of <b>dir_z</b> */
+  time_t published; /**< When was this object published */
+} cached_dir_t;
+
 /** Information about another onion router in the network. */
 typedef struct {
   char *signed_descriptor; /**< The original signed descriptor for this router*/
@@ -789,6 +800,8 @@ typedef struct routerstatus_t {
 typedef struct networkstatus_t {
   /** When did we receive the network-status document? */
   time_t received_on;
+  /** What was the digest of the document? */
+  char networkstatus_digest[DIGEST_LEN];
 
   /* These fields come from the actual network-status document.*/
   time_t published_on; /**< Declared publication date. */
@@ -1146,6 +1159,7 @@ typedef struct {
   int DirPort; /**< Port to listen on for directory connections. */
   int AssumeReachable; /**< Whether to publish our descriptor regardless. */
   int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
+  int V1AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
   int ClientOnly; /**< Boolean: should we never evolve into a server role? */
   int NoPublish; /**< Boolean: should we never publish a descriptor? */
   int ConnLimit; /**< Requested maximum number of simultaneous connections. */
@@ -1694,8 +1708,7 @@ void dirserv_set_cached_directory(const char *directory, time_t when,
                                   int is_running_routers);
 void dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
                                          time_t published);
-size_t dirserv_get_networkstatus_v2(const char **directory, const char *key,
-                                    int compress);
+int dirserv_get_networkstatus_v2(smartlist_t *result, const char *key);
 void dirserv_get_routerdescs(smartlist_t *descs_out, const char *key);
 void dirserv_orconn_tls_done(const char *address,
                              uint16_t or_port,
@@ -1995,15 +2008,18 @@ typedef struct trusted_dir_server_t {
   uint16_t dir_port;
   char digest[DIGEST_LEN];
   int is_running;
+  int supports_v1_protocol;
 } trusted_dir_server_t;
 
 int router_reload_router_list(void);
+int router_reload_networkstatus(void);
 void router_get_trusted_dir_servers(smartlist_t **outp);
 routerinfo_t *router_pick_directory_server(int requireother,
                                            int fascistfirewall,
-                                           int for_running_routers,
+                                           int for_v2_directory,
                                            int retry_if_no_servers);
-trusted_dir_server_t *router_pick_trusteddirserver(int requireother,
+trusted_dir_server_t *router_pick_trusteddirserver(int need_v1_support,
+                                                   int requireother,
                                                    int fascistfirewall,
                                                    int retry_if_no_servers);
 int all_trusted_directory_servers_down(void);
@@ -2046,6 +2062,7 @@ int router_add_to_routerlist(routerinfo_t *router, const char **msg);
 int router_load_single_router(const char *s, const char **msg);
 int router_load_routerlist_from_directory(const char *s,crypto_pk_env_t *pkey,
                                         int dir_is_recent, int dir_is_cached);
+int router_set_networkstatus(const char *s, time_t arrived_at, int is_cached);
 addr_policy_result_t router_compare_addr_to_addr_policy(uint32_t addr,
                               uint16_t port, addr_policy_t *policy);
 
@@ -2061,7 +2078,8 @@ int routers_update_status_from_entry(smartlist_t *routers,
 int router_update_status_from_smartlist(routerinfo_t *r,
                                         time_t list_time,
                                         smartlist_t *running_list);
-void add_trusted_dir_server(const char *addr, uint16_t port,const char *digest);
+void add_trusted_dir_server(const char *addr, uint16_t port,
+                            const char *digest, int supports_v1);
 void clear_trusted_dir_servers(void);
 
 /********************************* routerparse.c ************************/

src/or/router.c

@@ -368,7 +368,8 @@ init_keys(void)
   /* 6b. [authdirserver only] add own key to approved directories. */
   crypto_pk_get_digest(get_identity_key(), digest);
   if (!router_digest_is_trusted_dir(digest)) {
-    add_trusted_dir_server(NULL, (uint16_t)options->DirPort, digest);
+    add_trusted_dir_server(NULL, (uint16_t)options->DirPort, digest,
+                           options->V1AuthoritativeDir);
   }
 #if 0
   /* 7. [authdirserver only] load old directory, if it's there */

src/or/routerlist.c

@@ -21,9 +21,10 @@ static smartlist_t *trusted_dir_servers = NULL;
 /* static function prototypes */
 static routerinfo_t *
 router_pick_directory_server_impl(int requireother, int fascistfirewall,
-                                  int for_runningrouters);
+                                  int for_v2_directory);
 static trusted_dir_server_t *
-router_pick_trusteddirserver_impl(int requireother, int fascistfirewall);
+router_pick_trusteddirserver_impl(int need_v1_support,
+                                  int requireother, int fascistfirewall);
 static void mark_all_trusteddirservers_up(void);
 static int router_nickname_is_in_list(routerinfo_t *router, const char *list);
 static int router_nickname_matches(routerinfo_t *router, const char *nickname);
@@ -78,12 +79,13 @@ router_reload_router_list(void)
   return 0;
 }
 
+/** DOCDOC */
 int
 router_reload_networkstatus(void)
 {
   char filename[512];
   struct stat st;
-  smartlist_t *entries;
+  smartlist_t *entries, *bad_names;
   char *s;
   tor_assert(get_options()->DataDirectory);
   if (!networkstatus_list)
@@ -92,17 +94,24 @@ router_reload_networkstatus(void)
   tor_snprintf(filename,sizeof(filename),"%s/cached-status",
                get_options()->DataDirectory);
   entries = tor_listdir(filename);
+  bad_names = smartlist_create();
   SMARTLIST_FOREACH(entries, const char *, fn, {
+      char buf[DIGEST_LEN];
+      if (strlen(fn) != HEX_DIGEST_LEN ||
+          base16_decode(buf, sizeof(buf), fn, strlen(fn))) {
+        log_fn(LOG_INFO,
+               "Skipping cached-status file with unexpected name \"%s\"",fn);
+        continue;
+      }
       tor_snprintf(filename,sizeof(filename),"%s/cached-status/%s",
                    get_options()->DataDirectory, fn);
       s = read_file_to_str(filename, 0);
       if (s) {
-        networkstatus_t *ns;
         stat(filename, &st);
-        log_fn(LOG_INFO, "Loading cached network status from %s", filename);
-        ns = networkstatus_parse_from_string(s);
-        ns->received_on = st.st_mtime;
-        smartlist_add(networkstatus_list, ns);
+        if (router_set_networkstatus(s, st.st_mtime, 1)<0) {
+          log_fn(LOG_WARN, "Couldn't load networkstatus from \"%s\"",filename);
+        }
+        tor_free(s);
       }
     });
   return 0;
@@ -133,7 +142,7 @@ router_get_trusted_dir_servers(smartlist_t **outp)
 routerinfo_t *
 router_pick_directory_server(int requireother,
                              int fascistfirewall,
-                             int for_runningrouters,
+                             int for_v2_directory,
                              int retry_if_no_servers)
 {
   routerinfo_t *choice;
@@ -142,7 +151,7 @@ router_pick_directory_server(int requireother,
     return NULL;
 
   choice = router_pick_directory_server_impl(requireother, fascistfirewall,
-                                             for_runningrouters);
+                                             for_v2_directory);
   if (choice || !retry_if_no_servers)
     return choice;
 
@@ -151,7 +160,7 @@ router_pick_directory_server(int requireother,
   mark_all_trusteddirservers_up();
   /* try again */
   choice = router_pick_directory_server_impl(requireother, fascistfirewall,
-                                             for_runningrouters);
+                                             for_v2_directory);
   if (choice)
     return choice;
 
@@ -163,7 +172,7 @@ router_pick_directory_server(int requireother,
   }
   /* give it one last try */
   choice = router_pick_directory_server_impl(requireother, 0,
-                                             for_runningrouters);
+                                             for_v2_directory);
   return choice;
 }
 
@@ -171,21 +180,26 @@ router_pick_directory_server(int requireother,
  * trusted dirservers and <b>retry_if_no_servers</b> is non-zero,
  * set them all as running again, and try again.
  * Other args are as in router_pick_trusteddirserver_impl().
+ *
+ * DOCDOC need_v1_support
  */
 trusted_dir_server_t *
-router_pick_trusteddirserver(int requireother,
+router_pick_trusteddirserver(int need_v1_support,
+                             int requireother,
                              int fascistfirewall,
                              int retry_if_no_servers)
 {
   trusted_dir_server_t *choice;
 
-  choice = router_pick_trusteddirserver_impl(requireother, fascistfirewall);
+  choice = router_pick_trusteddirserver_impl(need_v1_support,
+                                             requireother, fascistfirewall);
   if (choice || !retry_if_no_servers)
     return choice;
 
   log_fn(LOG_INFO,"No trusted dirservers are reachable. Trying them all again.");
   mark_all_trusteddirservers_up();
-  return router_pick_trusteddirserver_impl(requireother, fascistfirewall);
+  return router_pick_trusteddirserver_impl(need_v1_support,
+                                           requireother, fascistfirewall);
 }
 
 /** Pick a random running verified directory server/mirror from our
@@ -193,13 +207,12 @@ router_pick_trusteddirserver(int requireother,
  * If <b>fascistfirewall</b> and we're not using a proxy,
  * make sure the port we pick is allowed by options-\>firewallports.
  * If <b>requireother</b>, it cannot be us.
- * If <b>for_runningrouters</b>, make sure we pick a dirserver that
- * can answer queries for running-routers (this option will become obsolete
- * once 0.0.9-rc5 is dead).
+ *
+ * DOCDOC need_v1_support, for_v2_directory
  */
 static routerinfo_t *
 router_pick_directory_server_impl(int requireother, int fascistfirewall,
-                                  int for_runningrouters)
+                                  int for_v2_directory)
 {
   int i;
   routerinfo_t *router;
@@ -223,9 +236,9 @@ router_pick_directory_server_impl(int requireother, int fascistfirewall,
       if (!fascist_firewall_allows_address(router->addr, router->dir_port))
         continue;
     }
-    /* before 0.0.9rc5-cvs, only trusted dirservers served status info. */
-    if (for_runningrouters &&
-        !(tor_version_as_new_as(router->platform,"0.0.9rc5-cvs") ||
+    /* before 0.1.1.6-alpha, only trusted dirservers served status info. */
+    if (for_v2_directory &&
+        !(tor_version_as_new_as(router->platform,"0.1.1.6-alpha") ||
           router_digest_is_trusted_dir(router->identity_digest)))
       continue;
     smartlist_add(sl, router);
@@ -242,7 +255,8 @@ router_pick_directory_server_impl(int requireother, int fascistfirewall,
  * If <b>requireother</b>, it cannot be us.
  */
 static trusted_dir_server_t *
-router_pick_trusteddirserver_impl(int requireother, int fascistfirewall)
+router_pick_trusteddirserver_impl(int need_v1_support,
+                                  int requireother, int fascistfirewall)
 {
   smartlist_t *sl;
   routerinfo_t *me;
@@ -259,6 +273,8 @@ router_pick_trusteddirserver_impl(int requireother, int fascistfirewall)
   SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, d,
     {
       if (!d->is_running) continue;
+      if (need_v1_support && !d->supports_v1_protocol)
+        continue;
       if (requireother && me &&
           !memcmp(me->identity_digest, d->digest, DIGEST_LEN))
         continue;
@@ -1166,6 +1182,79 @@ router_load_routerlist_from_directory(const char *s,
   return 0;
 }
 
+/** DOCDOC returns 0 on no problems, -1 on problems.
+ */
+int
+router_set_networkstatus(const char *s, time_t arrived_at, int is_cached)
+{
+  networkstatus_t *ns;
+  int i, found;
+  time_t now;
+  char fp[HEX_DIGEST_LEN+1];
+
+  ns = networkstatus_parse_from_string(s);
+  if (!ns) {
+    log_fn(LOG_WARN, "Couldn't parse network status.");
+    return -1;
+  }
+  if (!router_digest_is_trusted_dir(ns->identity_digest)) {
+    log_fn(LOG_INFO, "Network status was signed, but not by an authoritative directory we recognize.");
+    return -1;
+  }
+  now = time(NULL);
+  if (arrived_at > now)
+    arrived_at = now;
+
+  ns->received_on = arrived_at;
+
+  /*XXXX Check publishing skew. NM*/
+
+  if (!networkstatus_list)
+    networkstatus_list = smartlist_create();
+
+  found = 0;
+  for (i=0; i < smartlist_len(networkstatus_list); ++i) {
+    networkstatus_t *old_ns = smartlist_get(networkstatus_list, i);
+
+    if (!memcmp(old_ns->identity_digest, ns->identity_digest, DIGEST_LEN)) {
+      if (!memcmp(old_ns->networkstatus_digest,
+                  ns->networkstatus_digest, DIGEST_LEN)) {
+        networkstatus_free(ns);
+        return 0;
+      } else if (old_ns->published_on >= ns->published_on) {
+        log_fn(LOG_INFO, "Dropping network-status; we have a newer one for this authority.");
+        networkstatus_free(ns);
+        return 0;
+      } else {
+        networkstatus_free(old_ns);
+        smartlist_set(networkstatus_list, i, ns);
+        found = 1;
+        break;
+      }
+    }
+  }
+
+  if (!found)
+    smartlist_add(networkstatus_list, ns);
+
+  base16_encode(fp, HEX_DIGEST_LEN+1, ns->identity_digest, DIGEST_LEN);
+
+  if (!is_cached) {
+    const char *datadir = get_options()->DataDirectory;
+    char fp[HEX_DIGEST_LEN+1];
+    size_t len = strlen(datadir)+64;
+    char *fn = tor_malloc(len+1);
+    tor_snprintf(fn, len, "%s/cached-directory/%s",datadir,fp);
+    if (write_str_to_file(fn, s, 0)<0) {
+      log_fn(LOG_NOTICE, "Couldn't write cached network status to \"%s\"", fn);
+    }
+    tor_free(fn);
+  }
+
+  if (get_options()->DirPort)
+    dirserv_set_cached_networkstatus_v2(s, fp, ns->published_on);
+}
+
 /** Ensure that our own routerinfo is at the front, and remove duplicates
  * of our routerinfo.
  */
@@ -1571,7 +1660,8 @@ router_update_status_from_smartlist(routerinfo_t *router,
  * <b>address</b>:<b>port</b>, with identity key <b>digest</b>.  If
  * <b>address</b> is NULL, add ourself. */
 void
-add_trusted_dir_server(const char *address, uint16_t port, const char *digest)
+add_trusted_dir_server(const char *address, uint16_t port, const char *digest,
+                       int supports_v1)
 {
   trusted_dir_server_t *ent;
   uint32_t a;
@@ -1599,6 +1689,7 @@ add_trusted_dir_server(const char *address, uint16_t port, const char *digest)
   ent->addr = a;
   ent->dir_port = port;
   ent->is_running = 1;
+  ent->supports_v1_protocol = supports_v1;
   memcpy(ent->digest, digest, DIGEST_LEN);
   smartlist_add(trusted_dir_servers, ent);
 }

src/or/routerparse.c

@@ -1282,6 +1282,7 @@ networkstatus_parse_from_string(const char *s)
     goto err;
   }
   ns = tor_malloc_zero(sizeof(networkstatus_t));
+  memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN);
 
   if (!(tok = find_first_by_keyword(tokens, K_NETWORK_STATUS_VERSION))) {
     log_fn(LOG_WARN, "Couldn't find network-status-version keyword");