Start implementing the server side of the new directory protocol.

Probably very buggy, since I can't actually run an authdir.

Features
- Generate and publish new network-status format
- Code to cache and re-serve network-status objects generated by others.
- Publish individual descriptors (by fingerprint, by "all", and by
  "tell me yours.")  [Still needs compression logic]
- Publish client and server recommended versions seprately.
- Add digest of descriptor to routerinfo_t, so we can track them better, and
  length, so we can server them more easily.

Cleanups
- Unify code to sign directory-like things
- Make resolve_my_address() able to tell you which name it wound up resolving.
- Unify code to store and serve directory-like things so it all uses
  cached_dir_t.
- Unify code to set the value of cached_dir_t objects.


svn:r4835

doc/TODO

@@ -125,17 +125,28 @@ R     - check reachability as soon as you hear about a new server
       - have new people be in limbo and need to demonstrate usefulness
         before we approve them
       - other?
-    - dirservers publish router-status with all these flags.
+N   . Authoritative dirservers publish very compressed network-status objects.
+      o Generate format
+      o Publish it
+N   . Everyone downloads network-status objects
+      - From all directories, round-robin
+      - Cache them, reload on restart
+      o Serve cached directories
+      - If DirPort, act as a cache.
+N   - Directories expose individual descriptors
+      o By server ID
+      o By 'all'
+      - By 'if-newer-than' (Does the spec require this??)
+      - Support compression.
+      o Expose "own most recent descriptor".
+N   - Alice acts on network-status objects, downloading descriptors as needed.
     o Servers publish new descriptors when:
       o options change
       o when 12-24 hours have passed
       o when uptime is reset
       o When bandwidth changes a lot.
-    - alices fetch many router-statuses and update descriptors as needed.
-    - add if-newer-than fetch options
-    - dirservers allow people to lookup by N descriptors, or to fetch all.
     - alices avoid duplicate class C nodes.
-    - everybody with a dirport will give you his descriptor.
+    o everybody with a dirport will give you his descriptor.
     - config option, on by default, to cache all descriptors.
     - Compress router desc sets before transmitting them
     M Analyze how bad the partitioning is or isn't.

src/or/config.c

@@ -161,6 +161,8 @@ static config_var_t _option_vars[] = {
   VAR("PidFile",             STRING,   PidFile,              NULL),
   VAR("ReachableAddresses",  LINELIST, ReachableAddresses,   NULL),
   VAR("RecommendedVersions", LINELIST, RecommendedVersions,  NULL),
+  VAR("RecommendedClientVersions", LINELIST, RecommendedClientVersions,  NULL),
+  VAR("RecommendedServerVersions", LINELIST, RecommendedServerVersions,  NULL),
   VAR("RedirectExit",        LINELIST, RedirectExit,         NULL),
   VAR("RendExcludeNodes",    STRING,   RendExcludeNodes,     NULL),
   VAR("RendNodes",           STRING,   RendNodes,            NULL),
@@ -875,6 +877,22 @@ option_get_assignment(or_options_t *options, const char *key)
   return get_assigned_option(&options_format, options, key);
 }
 
+static config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+  config_line_t *result = NULL;
+  config_line_t **next_out = &result;
+  while (inp) {
+    *next_out = tor_malloc(sizeof(config_line_t));
+    (*next_out)->key = tor_strdup(inp->key);
+    (*next_out)->value = tor_strdup(inp->value);
+    inp = inp->next;
+    next_out = &((*next_out)->next);
+  }
+  (*next_out) = NULL;
+  return result;
+}
+
 static config_line_t *
 get_assigned_option(config_format_t *fmt, or_options_t *options, const char *key)
 {
@@ -899,17 +917,7 @@ get_assigned_option(config_format_t *fmt, or_options_t *options, const char *key
   if (var->type == CONFIG_TYPE_LINELIST ||
       var->type == CONFIG_TYPE_LINELIST_V) {
     /* Linelist requires special handling: we just copy and return it. */
-    const config_line_t *next_in = *(const config_line_t**)value;
-    config_line_t **next_out = &result;
-    while (next_in) {
-      *next_out = tor_malloc(sizeof(config_line_t));
-      (*next_out)->key = tor_strdup(next_in->key);
-      (*next_out)->value = tor_strdup(next_in->value);
-      next_in = next_in->next;
-      next_out = &((*next_out)->next);
-    }
-    (*next_out) = NULL;
-    return result;
+    return config_lines_dup(*(const config_line_t**)value);
   }
 
   result = tor_malloc_zero(sizeof(config_line_t));
@@ -1133,11 +1141,13 @@ print_usage(void)
 
 /**
  * Based on <b>options-\>Address</b>, guess our public IP address and put it
- * in *<b>addr</b>. Return 0 if all is well, or -1 if we can't find a
- * suitable public IP address.
+ * in *<b>addr_out</b>. If <b>hostname_out</b> is provided, set
+ * *<b>hostname_out</b> to a new string holding the hostname we used to get
+ * the address. Return 0 if all is well, or -1 if we can't find a suitable
+ * public IP address.
  */
 int
-resolve_my_address(or_options_t *options, uint32_t *addr)
+resolve_my_address(or_options_t *options, uint32_t *addr_out, char **hostname_out)
 {
   struct in_addr in;
   struct hostent *rent;
@@ -1147,7 +1157,7 @@ resolve_my_address(or_options_t *options, uint32_t *addr)
   static uint32_t old_addr=0;
   const char *address = options->Address;
 
-  tor_assert(addr);
+  tor_assert(addr_out);
 
   /* workaround: some people were leaving "Address  " in their torrc,
    * and they had a buggy resolver that resolved " " to 0.0.0.0. Oops.
@@ -1203,12 +1213,14 @@ resolve_my_address(or_options_t *options, uint32_t *addr)
   }
 
   log_fn(LOG_DEBUG, "Resolved Address to %s.", tmpbuf);
-  *addr = ntohl(in.s_addr);
-  if (old_addr && old_addr != *addr) {
+  *addr_out = ntohl(in.s_addr);
+  if (old_addr && old_addr != *addr_out) {
     log_fn(LOG_NOTICE,"Your IP seems to have changed. Updating.");
     server_has_changed_ip();
   }
-  old_addr = *addr;
+  old_addr = *addr_out;
+  if (hostname_out)
+    *hostname_out = tor_strdup(hostname);
   return 0;
 }
 
@@ -1582,7 +1594,7 @@ options_validate(or_options_t *options)
   if (server_mode(options)) {
     /* confirm that our address isn't broken, so we can complain now */
     uint32_t tmp;
-    if (resolve_my_address(options, &tmp) < 0)
+    if (resolve_my_address(options, &tmp, NULL) < 0)
       result = -1;
   }
 
@@ -1616,9 +1628,17 @@ options_validate(or_options_t *options)
     log(LOG_WARN, "StrictEntryNodes set, but no EntryNodes listed.");
   }
 
-  if (options->AuthoritativeDir && options->RecommendedVersions == NULL) {
-    log(LOG_WARN, "Directory servers must configure RecommendedVersions.");
-    result = -1;
+  if (options->AuthoritativeDir) {
+    if (!options->RecommendedVersions) {
+      log(LOG_WARN, "Directory servers must configure RecommendedVersions.");
+      result = -1;
+    }
+    if (!options->RecommendedClientVersions)
+      options->RecommendedClientVersions =
+        config_lines_dup(options->RecommendedVersions);
+    if (!options->RecommendedServerVersions)
+      options->RecommendedServerVersions =
+        config_lines_dup(options->RecommendedVersions);
   }
 
   if (options->AuthoritativeDir && !options->DirPort) {

src/or/directory.c

@@ -1057,6 +1057,60 @@ directory_handle_command_get(connection_t *conn, char *headers,
     return 0;
   }
 
+  if (!strcmpstart(url,"/tor/status/")) {
+    /* v2 network status fetch. */
+    size_t url_len = strlen(url);
+    int deflated = !strcmp(url+url_len-2, ".z");
+    const char *key = url + strlen("/tor/status/");
+    if (deflated)
+      url[url_len-2] = '\0';
+    dlen = dirserv_get_networkstatus_v2(&cp, key, deflated);
+    tor_free(url);
+    if (!dlen) { /* we failed to create/cache cp */
+      write_http_status_line(conn, 503, "Network status object unavailable");
+      /* try to get a new one now */
+      // XXXX NM
+      return 0;
+    }
+    format_rfc1123_time(date, time(NULL));
+    tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\nContent-Encoding: %s\r\n\r\n",
+                 date,
+                 (int)dlen,
+                 deflated?"deflate":"identity");
+    connection_write_to_buf(tmp, strlen(tmp), conn);
+    connection_write_to_buf(cp, strlen(cp), conn);
+    return 0;
+  }
+
+  if (!strcmpstart(url,"/tor/server/")) {
+    size_t url_len = strlen(url);
+    int deflated = !strcmp(url+url_len-2, ".z");
+    smartlist_t *descs = smartlist_create();
+    if (deflated)
+      url[url_len-2] = '\0';
+    dirserv_get_routerdescs(descs, url);
+    tor_free(url);
+    if (!smartlist_len(descs)) {
+      write_http_status_line(conn, 400, "Servers unavailable.");
+    } else {
+      size_t len = 0;
+      format_rfc1123_time(date, time(NULL));
+      SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
+                        len += ri->signed_descriptor_len);
+      /* XXXX We need to support deflate here. */
+      tor_snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: application/octet-stream\r\n\r\n",
+                   date,
+                   (int)len);
+      connection_write_to_buf(tmp, strlen(tmp), conn);
+      SMARTLIST_FOREACH(descs, routerinfo_t *, ri,
+                        connection_write_to_buf(ri->signed_descriptor,
+                                                ri->signed_descriptor_len,
+                                                conn));
+    }
+    smartlist_free(descs);
+    return 0;
+  }
+
   if (!strcmpstart(url,"/tor/rendezvous/") ||
       !strcmpstart(url,"/tor/rendezvous1/")) {
     /* rendezvous descriptor fetch */

src/or/dirserv.c

@@ -22,11 +22,14 @@ extern long stats_n_seconds_working;
 /** Do we need to regenerate the directory when someone asks for it? */
 static int the_directory_is_dirty = 1;
 static int runningrouters_is_dirty = 1;
+static int networkstatus_v2_is_dirty = 1;
 
 static void directory_remove_invalid(void);
 static int dirserv_regenerate_directory(void);
+static char *format_versions_list(config_line_t *ln);
 /* Should be static; exposed for testing */
 int add_fingerprint_to_dir(const char *nickname, const char *fp, smartlist_t *list);
+static int router_is_general_exit(routerinfo_t *ri);
 
 /************** Fingerprint handling code ************/
 
@@ -660,6 +663,55 @@ dirserv_remove_old_servers(int age)
   }
 }
 
+/* DOCDOC */
+static char *
+format_versions_list(config_line_t *ln)
+{
+  smartlist_t *versions;
+  char *result;
+  versions = smartlist_create();
+  for ( ; ln; ln = ln->next) {
+    smartlist_split_string(versions, ln->value, ",",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  }
+  result = smartlist_join_strings(versions,",",0,NULL);
+  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
+  smartlist_free(versions);
+  return result;
+}
+
+/* DOCDOC */
+static int
+append_signature(char *buf, size_t buf_len, const char *digest,
+                 crypto_pk_env_t *private_key)
+{
+  char signature[PK_BYTES];
+  int i;
+
+  if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) {
+
+    log_fn(LOG_WARN,"Couldn't sign digest.");
+    return -1;
+  }
+  if (strlcat(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  i = strlen(buf);
+  if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) {
+    log_fn(LOG_WARN,"couldn't base64-encode signature");
+    tor_free(buf);
+    return -1;
+  }
+
+  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
+    goto truncated;
+
+  return 0;
+ truncated:
+  log_fn(LOG_WARN,"tried to exceed string length.");
+  return -1;
+}
+
 /** Generate a new directory and write it into a newly allocated string.
  * Point *<b>dir_out</b> to the allocated string.  Sign the
  * directory with <b>private_key</b>.  Return 0 on success, -1 on
@@ -672,13 +724,11 @@ dirserv_dump_directory_to_string(char **dir_out,
   char *router_status;
   char *identity_pkey; /* Identity key, DER64-encoded. */
   char *recommended_versions;
-  char digest[20];
-  char signature[128];
-  char published[33];
+  char digest[DIGEST_LEN];
+  char published[ISO_TIME_LEN+1];
   time_t published_on;
   char *buf = NULL;
   size_t buf_len;
-  int i;
   size_t identity_pkey_len;
 
   tor_assert(dir_out);
@@ -696,18 +746,7 @@ dirserv_dump_directory_to_string(char **dir_out,
     return -1;
   }
 
-  {
-    smartlist_t *versions;
-    config_line_t *ln;
-    versions = smartlist_create();
-    for (ln = get_options()->RecommendedVersions; ln; ln = ln->next) {
-      smartlist_split_string(versions, ln->value, ",",
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    }
-    recommended_versions = smartlist_join_strings(versions,",",0,NULL);
-    SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
-    smartlist_free(versions);
-  }
+  recommended_versions = format_versions_list(get_options()->RecommendedVersions);
 
   dirserv_remove_old_servers(ROUTER_MAX_AGE);
   published_on = time(NULL);
@@ -755,26 +794,11 @@ dirserv_dump_directory_to_string(char **dir_out,
     tor_free(buf);
     return -1;
   }
-  if (crypto_pk_private_sign(private_key, signature, digest, 20) < 0) {
-    log_fn(LOG_WARN,"couldn't sign digest");
-    tor_free(buf);
-    return -1;
-  }
-  log(LOG_DEBUG,"generated directory digest begins with %s",hex_str(digest,4));
-
-  if (strlcat(buf, "-----BEGIN SIGNATURE-----\n", buf_len) >= buf_len)
-    goto truncated;
-
-  i = strlen(buf);
-  if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) {
-    log_fn(LOG_WARN,"couldn't base64-encode signature");
+  if (append_signature(buf,buf_len,digest,private_key)<0) {
     tor_free(buf);
     return -1;
   }
 
-  if (strlcat(buf, "-----END SIGNATURE-----\n", buf_len) >= buf_len)
-    goto truncated;
-
   *dir_out = buf;
   return 0;
  truncated:
@@ -783,12 +807,6 @@ dirserv_dump_directory_to_string(char **dir_out,
   return -1;
 }
 
-/** Most recently generated encoded signed directory. */
-static char *the_directory = NULL;
-static size_t the_directory_len = 0;
-static char *the_directory_z = NULL;
-static size_t the_directory_z_len = 0;
-
 /** DOCDOC */
 typedef struct cached_dir_t {
   char *dir;
@@ -798,21 +816,22 @@ typedef struct cached_dir_t {
   time_t published;
 } cached_dir_t;
 
+/** Most recently generated encoded signed directory. (auth dirservers only.)*/
+static cached_dir_t the_directory = { NULL, NULL, 0, 0, 0 };
+
 /* used only by non-auth dirservers */
 static cached_dir_t cached_directory = { NULL, NULL, 0, 0, 0 };
 static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0 };
 
-/** If we have no cached directory, or it is older than <b>when</b>, then
- * replace it with <b>directory</b>, published at <b>when</b>.
- */
-void
-dirserv_set_cached_directory(const char *directory, time_t when,
-                             int is_running_routers)
+/* Used for other dirservers' network statuses.  Map from hexdigest to
+ * cached_dir_t. */
+static strmap_t *cached_v2_networkstatus = NULL;
+
+/** DOCDOC */
+static void
+set_cached_dir(cached_dir_t *d, const char *directory, time_t when)
 {
-  time_t now;
-  cached_dir_t *d;
-  now = time(NULL);
-  d = is_running_routers ? &cached_runningrouters : &cached_directory;
+  time_t now = time(NULL);
   if (when<=d->published) {
     log_fn(LOG_INFO, "Ignoring old directory; not caching.");
   } else if (when>=now+ROUTER_MAX_AGE) {
@@ -829,13 +848,105 @@ dirserv_set_cached_directory(const char *directory, time_t when,
       log_fn(LOG_WARN,"Error compressing cached directory");
     }
     d->published = when;
-    if (!is_running_routers) {
-      char filename[512];
-      tor_snprintf(filename,sizeof(filename),"%s/cached-directory", get_options()->DataDirectory);
-      if (write_str_to_file(filename,cached_directory.dir,0) < 0) {
-        log_fn(LOG_NOTICE, "Couldn't write cached directory to disk. Ignoring.");
+  }
+}
+
+static void
+clear_cached_dir(cached_dir_t *d)
+{
+  tor_free(d->dir);
+  tor_free(d->dir_z);
+  memset(d, 0, sizeof(cached_dir_t));
+}
+
+static void
+free_cached_dir(void *_d)
+{
+  cached_dir_t *d = (cached_dir_t *)_d;
+  clear_cached_dir(d);
+  tor_free(d);
+}
+
+/** If we have no cached directory, or it is older than <b>when</b>, then
+ * replace it with <b>directory</b>, published at <b>when</b>.
+ */
+void
+dirserv_set_cached_directory(const char *directory, time_t published,
+                             int is_running_routers)
+{
+  cached_dir_t *d;
+  d = is_running_routers ? &cached_runningrouters : &cached_directory;
+  set_cached_dir(d, directory, published);
+  if (!is_running_routers) {
+    char filename[512];
+    tor_snprintf(filename,sizeof(filename),"%s/cached-directory", get_options()->DataDirectory);
+    if (write_str_to_file(filename,cached_directory.dir,0) < 0) {
+      log_fn(LOG_NOTICE, "Couldn't write cached directory to disk. Ignoring.");
+    }
+  }
+}
+
+/** DOCDOC */
+void
+dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
+                                    time_t published)
+{
+  cached_dir_t *d;
+  char fname[512];
+  if (!cached_v2_networkstatus)
+    cached_v2_networkstatus = strmap_new();
+
+  tor_assert(strlen(fp) == HEX_DIGEST_LEN);
+
+  if (!(d = strmap_get(cached_v2_networkstatus, fp))) {
+    d = tor_malloc_zero(sizeof(cached_dir_t));
+    strmap_set(cached_v2_networkstatus, fp, d);
+  }
+
+  tor_assert(d);
+  set_cached_dir(d, directory, published);
+
+  if (!d->dir)
+    return;
+
+  tor_snprintf(fname,sizeof(fname), "%s/cached-status/%s",
+               get_options()->DataDirectory, fp);
+  if (write_str_to_file(fname, d->dir, 0)<0) {
+    log_fn(LOG_NOTICE, "Couldn't write cached network status to disk. Ignoring.");
+  }
+}
+
+static size_t
+dirserv_get_obj(const char **out, int compress,
+                cached_dir_t *cache_src,
+                cached_dir_t *auth_src,
+                time_t dirty, int (*regenerate)(void),
+                const char *name)
+{
+  cached_dir_t *d;
+  if (!get_options()->AuthoritativeDir || !auth_src) {
+    d = cache_src;
+  } else {
+    if (regenerate != NULL) {
+      if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) {
+        if (regenerate()) {
+          log_fn(LOG_ERR, "Couldn't generate %s?", name);
+          exit(1);
+        }
+      } else {
+        log_fn(LOG_INFO, "The %s is still clean; reusing.", name);
       }
     }
+    d = auth_src;
+  }
+  if (!d)
+    return 0;
+  *out = compress ? d->dir_z : d->dir;
+  if (*out) {
+    return compress ? d->dir_z_len : d->dir_len;
+  } else {
+    /* not yet available. */
+    return 0;
   }
 }
 
@@ -845,25 +956,11 @@ dirserv_set_cached_directory(const char *directory, time_t when,
 size_t
 dirserv_get_directory(const char **directory, int compress)
 {
-  if (!get_options()->AuthoritativeDir) {
-    cached_dir_t *d = &cached_directory;
-    *directory = compress ? d->dir_z : d->dir;
-    if (*directory) {
-      return compress ? d->dir_z_len : d->dir_len;
-    } else {
-      /* no directory yet retrieved */
-      return 0;
-    }
-  }
-  if (the_directory_is_dirty &&
-      the_directory_is_dirty + DIR_REGEN_SLACK_TIME < time(NULL)) {
-    if (dirserv_regenerate_directory())
-      return 0;
-  } else {
-    log(LOG_INFO,"Directory still clean, reusing.");
-  }
-  *directory = compress ? the_directory_z : the_directory;
-  return compress ? the_directory_z_len : the_directory_len;
+  return dirserv_get_obj(directory, compress,
+                         &cached_directory, &the_directory,
+                         the_directory_is_dirty,
+                         dirserv_regenerate_directory,
+                         "server directory");
 }
 
 /**
@@ -880,45 +977,31 @@ dirserv_regenerate_directory(void)
     tor_free(new_directory);
     return -1;
   }
-  tor_free(the_directory);
-  the_directory = new_directory;
-  the_directory_len = strlen(the_directory);
-  log_fn(LOG_INFO,"New directory (size %d):\n%s",(int)the_directory_len,
-         the_directory);
-  tor_free(the_directory_z);
-  if (tor_gzip_compress(&the_directory_z, &the_directory_z_len,
-                        the_directory, the_directory_len,
-                        ZLIB_METHOD)) {
-    log_fn(LOG_WARN, "Error gzipping directory.");
-    return -1;
-  }
+  set_cached_dir(&the_directory, new_directory, time(NULL));
+  log_fn(LOG_INFO,"New directory (size %d):\n%s",(int)the_directory.dir_len,
+         the_directory.dir);
 
   the_directory_is_dirty = 0;
 
   /* Save the directory to disk so we re-load it quickly on startup.
    */
-  dirserv_set_cached_directory(the_directory, time(NULL), 0);
+  dirserv_set_cached_directory(the_directory.dir, time(NULL), 0);
 
   return 0;
 }
 
-static char *the_runningrouters=NULL;
-static size_t the_runningrouters_len=0;
-static char *the_runningrouters_z=NULL;
-static size_t the_runningrouters_z_len=0;
+static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0 };
 
 /** Replace the current running-routers list with a newly generated one. */
 static int
-generate_runningrouters(crypto_pk_env_t *private_key)
+generate_runningrouters(void)
 {
-  char *s=NULL, *cp;
+  char *s=NULL;
   char *router_status=NULL;
   char digest[DIGEST_LEN];
-  char signature[PK_BYTES];
-  int i;
-  char published[33];
+  char published[ISO_TIME_LEN+1];
   size_t len;
-  time_t published_on;
+  crypto_pk_env_t *private_key = get_identity_key();
   char *identity_pkey; /* Identity key, DER64-encoded. */
   size_t identity_pkey_len;
 
@@ -933,49 +1016,27 @@ generate_runningrouters(crypto_pk_env_t *private_key)
     log_fn(LOG_WARN,"write identity_pkey to string failed!");
     goto err;
   }
-  published_on = time(NULL);
-  format_iso_time(published, published_on);
+  format_iso_time(published, time(NULL));
 
   len = 2048+strlen(router_status);
   s = tor_malloc_zero(len);
-  tor_snprintf(s, len, "network-status\n"
-             "published %s\n"
-             "router-status %s\n"
-             "dir-signing-key\n%s"
-             "directory-signature %s\n"
-             "-----BEGIN SIGNATURE-----\n",
-          published, router_status, identity_pkey, get_options()->Nickname);
+  tor_snprintf(s, len,
+               "network-status\n"
+               "published %s\n"
+               "router-status %s\n"
+               "dir-signing-key\n%s"
+               "directory-signature %s\n",
+               published, router_status, identity_pkey, get_options()->Nickname);
   tor_free(router_status);
   tor_free(identity_pkey);
   if (router_get_runningrouters_hash(s,digest)) {
     log_fn(LOG_WARN,"couldn't compute digest");
     goto err;
   }
-  if (crypto_pk_private_sign(private_key, signature, digest, 20) < 0) {
-    log_fn(LOG_WARN,"couldn't sign digest");
-    goto err;
-  }
-
-  i = strlen(s);
-  cp = s+i;
-  if (base64_encode(cp, len-i, signature, 128) < 0) {
-    log_fn(LOG_WARN,"couldn't base64-encode signature");
-    goto err;
-  }
-  if (strlcat(s, "-----END SIGNATURE-----\n", len) >= len) {
+  if (append_signature(s, len, digest, private_key)<0)
     goto err;
-  }
 
-  tor_free(the_runningrouters);
-  the_runningrouters = s;
-  the_runningrouters_len = strlen(s);
-  tor_free(the_runningrouters_z);
-  if (tor_gzip_compress(&the_runningrouters_z, &the_runningrouters_z_len,
-                        the_runningrouters, the_runningrouters_len,
-                        ZLIB_METHOD)) {
-    log_fn(LOG_WARN, "Error gzipping runningrouters");
-    return -1;
-  }
+  set_cached_dir(&the_runningrouters, s, time(NULL));
   runningrouters_is_dirty = 0;
 
   /* We don't cache running-routers to disk, so there's no point in
@@ -995,25 +1056,255 @@ generate_runningrouters(crypto_pk_env_t *private_key)
 size_t
 dirserv_get_runningrouters(const char **rr, int compress)
 {
-  if (!get_options()->AuthoritativeDir) {
-    cached_dir_t *d = &cached_runningrouters;
-    *rr = compress ? d->dir_z : d->dir;
-    if (*rr) {
-      return compress ? d->dir_z_len : d->dir_len;
-    } else {
-      /* no directory yet retrieved */
-      return 0;
+  return dirserv_get_obj(rr, compress,
+                         &cached_runningrouters, &the_runningrouters,
+                         runningrouters_is_dirty,
+                         generate_runningrouters,
+                         "v1 network status list");
+}
+
+/** DOCDOC */
+static int
+router_is_general_exit(routerinfo_t *ri)
+{
+  static const int ports[] = { 80, 443, 194 };
+  int n_allowed = 3;
+  int i;
+  for (i = 0; i < 3; ++i) {
+    struct addr_policy_t *policy = ri->exit_policy;
+    for ( ; policy; policy = policy->next) {
+      if (policy->prt_min > ports[i] || policy->prt_max < ports[i])
+        continue; /* Doesn't cover our port. */
+      if ((policy->msk & 0x00fffffful) != 0)
+        continue; /* Wider than /8. */
+      if ((policy->addr & 0xff000000ul) == 0x7f000000ul)
+        continue; /* 127.x */
+      /* We have a match that is wider than /24. */
+      if (policy->policy_type != ADDR_POLICY_ACCEPT)
+        --n_allowed;
+      break;
     }
   }
-  if (runningrouters_is_dirty &&
-      runningrouters_is_dirty + DIR_REGEN_SLACK_TIME < time(NULL)) {
-    if (generate_runningrouters(get_identity_key())) {
-      log_fn(LOG_ERR, "Couldn't generate running-routers list?");
-      return 0;
+  return n_allowed > 0;
+}
+
+static cached_dir_t the_v2_networkstatus = { NULL, NULL, 0, 0, 0 };
+
+static int
+generate_v2_networkstatus(void)
+{
+#define BASE64_DIGEST_LEN 29
+#define LONGEST_STATUS_FLAG_NAME_LEN 7
+#define N_STATUS_FLAGS 6
+#define RS_ENTRY_LEN                                                    \
+  ( /* first line */                                                    \
+   MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \
+   5*2 /* ports */ + 10 /* punctuation */ +                             \
+   /* second line */                                                    \
+   (LONGEST_STATUS_FLAG_NAME_LEN+1)*N_STATUS_FLAGS + 2)
+
+  int r = -1;
+  size_t len, identity_pkey_len;
+  char *status = NULL, *client_versions = NULL, *server_versions = NULL,
+    *identity_pkey = NULL, *hostname = NULL;
+  char *outp, *endp;
+  or_options_t *options = get_options();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char ipaddr[INET_NTOA_BUF_LEN+1];
+  char published[ISO_TIME_LEN];
+  char digest[DIGEST_LEN];
+  struct in_addr in;
+  uint32_t addr;
+  crypto_pk_env_t *private_key = get_identity_key();
+
+  if (resolve_my_address(options, &addr, &hostname)<0) {
+    log_fn(LOG_WARN, "Couldn't resolve my hostname");
+    goto done;
+  }
+  in.s_addr = htonl(addr);
+  tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
+
+  format_iso_time(published, time(NULL));
+
+  client_versions = format_versions_list(options->RecommendedClientVersions);
+  server_versions = format_versions_list(options->RecommendedServerVersions);
+
+  if (crypto_pk_write_public_key_to_string(private_key, &identity_pkey,
+                                           &identity_pkey_len)<0) {
+    log_fn(LOG_WARN,"Writing public key to string failed.");
+    goto done;
+  }
+
+  if (crypto_pk_get_fingerprint(private_key, fingerprint, 0)<0) {
+    log_fn(LOG_ERR, "Error computing fingerprint");
+    return -1;
+  }
+
+  len = 2048+strlen(client_versions)+strlen(server_versions)+identity_pkey_len*2;
+  len += (RS_ENTRY_LEN)*smartlist_len(descriptor_list) ;
+
+  status = tor_malloc(len);
+  tor_snprintf(status, len,
+               "network-status-version 2\n"
+               "dir-source %s %s %d\n"
+               "dir-fingerprint %s\n"
+               "published %s\n"
+               "dir-options %s\n"
+               "client-versions %s\n"
+               "server-versions %s\n"
+               "dir-signing-key\n%s\n",
+               hostname, ipaddr, (int)options->DirPort,
+               fingerprint,
+               published,
+               "Names",
+               client_versions,
+               server_versions,
+               identity_pkey);
+  outp = status + strlen(status);
+  endp = status + len;
+
+  SMARTLIST_FOREACH(descriptor_list, routerinfo_t *, ri, {
+      int f_exit = router_is_general_exit(ri);
+      int f_stable = !router_is_unreliable(ri, 1, 0);
+      int f_fast = !router_is_unreliable(ri, 0, 1);
+      int f_running;
+      int f_named = ri->is_verified;
+      int f_valid = f_named;
+      char identity64[128];
+      char digest64[128];
+
+      if (options->AuthoritativeDir) {
+        connection_t *conn = connection_get_by_identity_digest(
+                                         ri->identity_digest, CONN_TYPE_OR);
+        f_running = (router_is_me(ri) && !we_are_hibernating()) ||
+          (conn && conn->state == OR_CONN_STATE_OPEN);
+      } else {
+        f_running = ri->is_running;
+      }
+
+      format_iso_time(published, ri->published_on);
+
+      if (base64_encode(identity64, sizeof(identity64),
+                        ri->identity_digest, DIGEST_LEN)<0)
+        goto done;
+      if (base64_encode(digest64, sizeof(digest64),
+                        ri->signed_descriptor_digest, DIGEST_LEN)<0)
+        goto done;
+      identity64[BASE64_DIGEST_LEN] = '\0';
+      digest64[BASE64_DIGEST_LEN] = '\0';
+
+      in.s_addr = htonl(ri->addr);
+      tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
+
+      if (tor_snprintf(outp, endp-outp,
+                       "r %s %s %s %s %s %d %d\n"
+                       "s%s%s%s%s%s%s\n",
+                       ri->nickname,
+                       identity64,
+                       digest64,
+                       published,
+                       ipaddr,
+                       ri->or_port,
+                       ri->dir_port,
+                       f_exit?" Exit":"",
+                       f_stable?" Stable":"",
+                       f_fast?" Fast":"",
+                       f_running?" Running":"",
+                       f_named?" Named":"",
+                       f_valid?" Valid":"")<0) {
+        log_fn(LOG_WARN, "Unable to print router status.");
+        goto done;
+      }
+      outp += strlen(outp);
+    });
+
+  if (tor_snprintf(outp, endp-outp, "directory-signature %s\n",
+                   get_options()->Nickname)<0) {
+    log_fn(LOG_WARN, "Unable to write signature line.");
+    goto done;
+  }
+
+  if (router_get_networkstatus_v2_hash(status, digest)<0) {
+    log_fn(LOG_WARN, "Unable to hash network status");
+    goto done;
+  }
+
+  if (append_signature(outp,endp-outp,digest,private_key)<0)
+    goto done;
+
+  set_cached_dir(&the_v2_networkstatus, status, time(NULL));
+  dirserv_set_cached_networkstatus_v2(status, fingerprint, time(NULL));
+
+  r = 0;
+ done:
+  tor_free(client_versions);
+  tor_free(server_versions);
+  tor_free(status);
+  tor_free(hostname);
+  tor_free(identity_pkey);
+  return r;
+}
+
+size_t
+dirserv_get_networkstatus_v2(const char **directory, const char *key,
+                             int compress)
+{
+  *directory = NULL;
+  if (!(strcmp(key,"authority"))) {
+    if (get_options()->AuthoritativeDir) {
+      return dirserv_get_obj(directory, compress, NULL,
+                             &the_v2_networkstatus,
+                             networkstatus_v2_is_dirty,
+                             generate_v2_networkstatus,
+                             "network status list");
     }
+  } else if (!strcmp(key, "all")) {
+    // XXXX NM
+    return dirserv_get_networkstatus_v2(directory, "authority", compress);
+  } else if (strlen(key)==HEX_DIGEST_LEN) {
+    cached_dir_t *cached = strmap_get(cached_v2_networkstatus, key);
+    if (cached)
+      return dirserv_get_obj(directory, compress, cached, NULL, 0, NULL,
+                             "cached network status");
+  }
+  return 0;
+}
+
+void
+dirserv_get_routerdescs(smartlist_t *descs_out, const char *key)
+{
+
+  if (!strcmp(key, "/tor/server/all")) {
+    smartlist_add_all(descs_out, descriptor_list);
+  } else if (!strcmp(key, "/tor/server/authority")) {
+    routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(descs_out, ri);
+  } else if (!strcmpstart(key, "/tor/server/fp/")) {
+    smartlist_t *hexdigests = smartlist_create();
+    smartlist_t *digests = smartlist_create();
+    key += strlen("/tor/server/fp/");
+    smartlist_split_string(hexdigests, key, "+", 0, 0);
+    SMARTLIST_FOREACH(hexdigests, char *, cp,
+                      {
+                        char *d;
+                        if (strlen(cp) != HEX_DIGEST_LEN)
+                          continue;
+                        d = tor_malloc_zero(DIGEST_LEN);
+                        base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN);
+                        tor_free(cp);
+                        smartlist_add(digests, d);
+                      });
+    smartlist_free(hexdigests);
+    SMARTLIST_FOREACH(descriptor_list, routerinfo_t *, ri,
+                      SMARTLIST_FOREACH(digests, const char *, d,
+                        if (!memcmp(d,ri->identity_digest,DIGEST_LEN)) {
+                          smartlist_add(descs_out,ri);
+                          break;
+                        }));
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
   }
-  *rr = compress ? the_runningrouters_z : the_runningrouters;
-  return compress ? the_runningrouters_z_len : the_runningrouters_len;
 }
 
 /** Called when a TLS handshake has completed successfully with a
@@ -1087,19 +1378,11 @@ dirserv_free_all(void)
     smartlist_free(descriptor_list);
     descriptor_list = NULL;
   }
-  tor_free(the_directory);
-  tor_free(the_directory_z);
-  the_directory_len = 0;
-  the_directory_z_len = 0;
-  tor_free(the_runningrouters);
-  tor_free(the_runningrouters_z);
-  the_runningrouters_len = 0;
-  the_runningrouters_z_len = 0;
-  tor_free(cached_directory.dir);
-  tor_free(cached_directory.dir_z);
-  tor_free(cached_runningrouters.dir);
-  tor_free(cached_runningrouters.dir_z);
-  memset(&cached_directory, 0, sizeof(cached_directory));
-  memset(&cached_runningrouters, 0, sizeof(cached_runningrouters));
+  clear_cached_dir(&the_directory);
+  clear_cached_dir(&the_runningrouters);
+  clear_cached_dir(&cached_directory);
+  clear_cached_dir(&cached_runningrouters);
+  strmap_free(cached_v2_networkstatus, free_cached_dir);
+  cached_v2_networkstatus = NULL;
 }
 

src/or/or.h

@@ -708,6 +708,8 @@ typedef struct addr_policy_t {
 /** Information about another onion router in the network. */
 typedef struct {
   char *signed_descriptor; /**< The original signed descriptor for this router*/
+  size_t signed_descriptor_len; /**< The length of signed_descriptor */
+  char signed_descriptor_digest[DIGEST_LEN]; /**< The digest of the signed descriptor. */
 
   char *address; /**< Location of OR: either a hostname or an IP address. */
   char *nickname; /**< Human-readable OR name. */
@@ -1078,6 +1080,8 @@ typedef struct {
   /** Directory server only: which versions of
    * Tor should we tell users to run? */
   config_line_t *RecommendedVersions;
+  config_line_t *RecommendedClientVersions;
+  config_line_t *RecommendedServerVersions;
   /** Whether dirservers refuse router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   char *User; /**< Name of user to run Tor as. */
@@ -1360,7 +1364,8 @@ const char *safe_str(const char *address);
 int config_get_lines(char *string, config_line_t **result);
 void config_free_lines(config_line_t *front);
 int options_trial_assign(config_line_t *list, int reset);
-int resolve_my_address(or_options_t *options, uint32_t *addr);
+int resolve_my_address(or_options_t *options, uint32_t *addr,
+                       char **hostname_out);
 void options_init(or_options_t *options);
 int options_init_from_torrc(int argc, char **argv);
 int options_init_logs(or_options_t *options, int validate_only);
@@ -1635,6 +1640,11 @@ size_t dirserv_get_directory(const char **cp, int compress);
 size_t dirserv_get_runningrouters(const char **rr, int compress);
 void dirserv_set_cached_directory(const char *directory, time_t when,
                                   int is_running_routers);
+void dirserv_set_cached_networkstatus_v2(const char *directory, const char *fp,
+                                         time_t published);
+size_t dirserv_get_networkstatus_v2(const char **directory, const char *key,
+                                    int compress);
+void dirserv_get_routerdescs(smartlist_t *descs_out, const char *key);
 void dirserv_orconn_tls_done(const char *address,
                              uint16_t or_port,
                              const char *digest_rcvd,
@@ -2023,6 +2033,7 @@ typedef struct tor_version_t {
 int router_get_router_hash(const char *s, char *digest);
 int router_get_dir_hash(const char *s, char *digest);
 int router_get_runningrouters_hash(const char *s, char *digest);
+int router_get_networkstatus_v2_hash(const char *s, char *digest);
 int router_parse_list_from_string(const char **s,
                                   routerlist_t **dest,
                                   smartlist_t *good_nickname_list,

src/or/router.c

@@ -728,7 +728,7 @@ router_rebuild_descriptor(int force)
   if (desc_clean_since && !force)
     return 0;
 
-  if (resolve_my_address(options, &addr) < 0) {
+  if (resolve_my_address(options, &addr, NULL) < 0) {
     log_fn(LOG_WARN,"options->Address didn't resolve into an IP.");
     return -1;
   }
@@ -774,6 +774,9 @@ router_rebuild_descriptor(int force)
     log_fn(LOG_WARN, "Couldn't dump router to string.");
     return -1;
   }
+  ri->signed_descriptor_len = strlen(ri->signed_descriptor);
+  crypto_digest(ri->signed_descriptor_digest,
+                ri->signed_descriptor, ri->signed_descriptor_len);
 
   if (desc_routerinfo)
     routerinfo_free(desc_routerinfo);

src/or/routerparse.c

@@ -179,6 +179,14 @@ router_get_runningrouters_hash(const char *s, char *digest)
                               "network-status","\ndirectory-signature");
 }
 
+/** DOCDOC */
+int
+router_get_networkstatus_v2_hash(const char *s, char *digest)
+{
+  return router_get_hash_impl(s,digest,
+                              "network-status-version","\ndirectory-signature");
+}
+
 /**
  * Find the first instance of "recommended-software ...\n" at the start of
  * a line; return a newly allocated string containing the "..." portion.
@@ -857,6 +865,8 @@ router_parse_entry_from_string(const char *s, const char *end)
 
   router = tor_malloc_zero(sizeof(routerinfo_t));
   router->signed_descriptor = tor_strndup(s, end-s);
+  router->signed_descriptor_len = end-s;
+  crypto_digest(router->signed_descriptor_digest, s, end-s);
   ports_set = bw_set = 0;
 
   if (tok->n_args == 2 || tok->n_args == 5 || tok->n_args == 6) {