Stop downloading directories and download routers instead. This still needs some work, but at last clients are finally on the new architecture. Next comes the tuning and bugfixing.

svn:r5070

doc/TODO

@@ -133,30 +133,37 @@ R     - check reachability as soon as you hear about a new server
         X By 'if-newer-than' (Does the spec require this??)
         o Support compression.
 N     - Alice acts on network-status objects
-        - Alice downloads descriptors as needed.
+        . Alice downloads descriptors as needed.
           o Figure out what's needed
-          - Download it
-          - Store it
+          o Store it
             o Implement store
-            - Implement reload-from-store
-            - Store downloaded descriptors
-          - Retry descriptors on failure for a while
+            o Implement reload-from-store
+            o Store downloaded descriptors
+          o Download it
+            o As-needed if we have 2 network-status objs.
+            o Download "all" if we have less than 2 network-status objs.
+              (This has vulnerabilities if we're not careful)
+            o Call directory_has_arrived as needed; rename it.
+            o Set has_fetched_directory properly.
+          o Retry descriptors on failure
+          - Give up after a while.
         o Alice sets descriptor status from network-status
           o Implement
           o Use
+      - Call dirport_is_reachable from somewhere else.
 
     - Security
       - Alices avoid duplicate class C nodes.
       - Analyze how bad the partitioning is or isn't.
 
-N   - Naming and validation:
-      - Separate naming from validation in authdirs.
+N   . Naming and validation:
+      o Separate naming from validation in authdirs.
       - Authdirs need to be able to decline to validate based on
         IP range and key
       - Authdirs need to be able to decline to include baased on
         IP range and key.
       - Clients choose names based on network-status options.
-      - Names are remembered in client status.
+      - Names are remembered in client state
 
   - packaging and ui stuff:
     . multiple sample torrc files

src/or/directory.c

@@ -52,7 +52,9 @@ static int purpose_is_private(uint8_t purpose);
 static char *http_get_header(const char *headers, const char *which);
 static char *http_get_origin(const char *headers, connection_t *conn);
 static void connection_dir_download_networkstatus_failed(connection_t *conn);
+static void connection_dir_download_routerdesc_failed(connection_t *conn);
 static void dir_networkstatus_download_failed(smartlist_t *failed);
+static void dir_routerdesc_download_failed(smartlist_t *failed);
 
 /********* START VARIABLES **********/
 
@@ -282,6 +284,10 @@ connection_dir_request_failed(connection_t *conn)
     log_fn(LOG_INFO, "Giving up on directory server at '%s'; retrying",
            conn->address);
     connection_dir_download_networkstatus_failed(conn);
+  } else if (conn->purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS) {
+    log_fn(LOG_INFO, "Giving up on directory server at '%s'; retrying",
+           conn->address);
+    connection_dir_download_routerdesc_failed(conn);
   }
 }
 
@@ -314,6 +320,16 @@ connection_dir_download_networkstatus_failed(connection_t *conn)
   }
 }
 
+/** Called when an attempt to download one or network status documents
+ * on connection <b>conn</b> failed.
+ */
+static void
+connection_dir_download_routerdesc_failed(connection_t *conn)
+{
+  /* try again. */
+  update_router_descriptor_downloads(time(NULL));
+}
+
 /** Helper for directory_initiate_command_(router|trusted_dir): send the
  * command to a server whose address is <b>address</b>, whose IP is
  * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version is
@@ -892,14 +908,18 @@ connection_dir_client_reached_eof(connection_t *conn)
       tor_free(body); tor_free(headers); tor_free(reason);
       return -1;
     }
+    if (router_parse_directory(body) < 0) {
+      log_fn(LOG_NOTICE,"I failed to parse the directory I fetched from '%s:%d'. Ignoring.", conn->address, conn->port);
+    }
+#if 0
     if (router_load_routerlist_from_directory(body, NULL, !skewed, 0) < 0) {
       log_fn(LOG_NOTICE,"I failed to parse the directory I fetched from '%s:%d'. Ignoring.", conn->address, conn->port);
       tor_free(body); tor_free(headers); tor_free(reason);
       return -1;
     }
-    log_fn(LOG_INFO,"updated routers.");
     /* do things we've been waiting to do */
     directory_has_arrived(time(NULL), conn->identity_digest);
+#endif
   }
 
   if (conn->purpose == DIR_PURPOSE_FETCH_RUNNING_LIST) {
@@ -969,7 +989,8 @@ connection_dir_client_reached_eof(connection_t *conn)
       else
         break;
     }
-    routers_update_all_from_networkstatus();
+    routers_update_all_from_networkstatus();/*launches router downloads*/
+    directory_info_has_arrived(time(NULL),0);
     if (which) {
       if (smartlist_len(which)) {
         dir_networkstatus_download_failed(which);
@@ -980,8 +1001,36 @@ connection_dir_client_reached_eof(connection_t *conn)
   }
 
   if (conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC) {
+    smartlist_t *which = NULL;
     /* XXXX NM implement this. */
-    log_fn(LOG_WARN, "Somehow, we requested some individual server descriptors. Skipping.");
+    log_fn(LOG_INFO,"Received server info (size %d) from server '%s:%d'",
+           (int)body_len, conn->address, conn->port);
+    if (status_code != 200) {
+      log_fn(LOG_WARN,"Received http status code %d (\"%s\") from server '%s:%d' while fetching \"/tor/server/%s\". I'll try again soon.",
+             status_code, reason, conn->address, conn->port,
+             conn->requested_resource);
+      tor_free(body); tor_free(headers); tor_free(reason);
+      connection_dir_download_routerdesc_failed(conn);
+      return -1;
+    }
+    if (conn->requested_resource &&
+        !strcmpstart(conn->requested_resource,"fp/")) {
+      int n;
+      which = smartlist_create();
+      smartlist_split_string(which, conn->requested_resource+3, "+", 0, -1);
+      n = smartlist_len(which);
+      if (n && strlen(smartlist_get(which,n-1))==HEX_DIGEST_LEN+2)
+        ((char*)smartlist_get(which,n-1))[HEX_DIGEST_LEN] = '\0';
+    }
+    router_load_routers_from_string(body, 0, which);
+    directory_info_has_arrived(time(NULL),0);
+    if (which) {
+      if (smartlist_len(which)) {
+        dir_routerdesc_download_failed(which);
+      }
+      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+      smartlist_free(which);
+    }
   }
 
   if (conn->purpose == DIR_PURPOSE_UPLOAD_DIR) {
@@ -1524,3 +1573,10 @@ dir_networkstatus_download_failed(smartlist_t *failed)
   });
 }
 
+/* DOCDOC */
+static void
+dir_routerdesc_download_failed(smartlist_t *failed)
+{
+  /* XXXX writeme!  Give up after a while! */
+}
+

src/or/dirserv.c

@@ -39,6 +39,8 @@ int add_fingerprint_to_dir(const char *nickname, const char *fp, smartlist_t *li
 static int router_is_general_exit(routerinfo_t *ri);
 static router_status_t dirserv_router_get_status(const routerinfo_t *router,
                                                  const char **msg);
+static int dirserv_thinks_router_is_reachable(routerinfo_t *router,
+                                              time_t now);
 
 /************** Fingerprint handling code ************/
 
@@ -283,7 +285,8 @@ dirserv_router_has_valid_address(routerinfo_t *ri)
 }
 
 /** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
- * return 0, and set its is_valid and is_named fields.  Otherwise, return -1.
+ * return 0, and set its is_valid,named,running fields.  Otherwise, return -1.
+ *
  * DOCDOC msg
  */
 int
@@ -366,7 +369,7 @@ dirserv_add_descriptor(const char *desc, const char **msg)
     *msg = "Rejected: Couldn't parse server descriptor.";
     return -2;
   }
-  if ((r = router_add_to_routerlist(ri, msg))<0) {
+  if ((r = router_add_to_routerlist(ri, msg, 0))<0) {
     return r == -1 ? 0 : -1;
   } else {
     smartlist_t *changed = smartlist_create();
@@ -801,13 +804,6 @@ dirserv_set_cached_directory(const char *directory, time_t published,
   cached_dir_t *d;
   d = is_running_routers ? &cached_runningrouters : &cached_directory;
   set_cached_dir(d, tor_strdup(directory), published);
-  if (!is_running_routers) {
-    char filename[512];
-    tor_snprintf(filename,sizeof(filename),"%s/cached-directory", get_options()->DataDirectory);
-    if (write_str_to_file(filename,cached_directory.dir,0) < 0) {
-      log_fn(LOG_NOTICE, "Couldn't write cached directory to disk. Ignoring.");
-    }
-  }
 }
 
 /** We've just received a v2 network-status for an authoritative directory
@@ -1077,6 +1073,7 @@ generate_v2_networkstatus(void)
   uint32_t addr;
   crypto_pk_env_t *private_key = get_identity_key();
   smartlist_t *descriptor_list = get_descriptor_list();
+  time_t now = time(NULL);
   const char *contact;
 
   if (!descriptor_list) {
@@ -1147,15 +1144,9 @@ generate_v2_networkstatus(void)
       char digest64[128];
 
       if (options->AuthoritativeDir) {
-        connection_t *conn = connection_get_by_identity_digest(
-                                         ri->identity_digest, CONN_TYPE_OR);
-        f_running = (router_is_me(ri) && !we_are_hibernating()) ||
-          (conn && conn->state == OR_CONN_STATE_OPEN);
-        /* Update router status in routerinfo_t. */
-        ri->is_running = f_running;
-      } else {
-        f_running = ri->is_running;
+        ri->is_running = dirserv_thinks_router_is_reachable(ri, now);
       }
+      f_running = ri->is_running;
 
       format_iso_time(published, ri->published_on);
 

src/or/main.c

@@ -527,43 +527,32 @@ get_status_fetch_period(or_options_t *options)
     return 30*60;
 }
 
-/** This function is called whenever we successfully pull down a directory.
- * If <b>identity_digest</b> is defined, it contains the digest of the
- * router that just gave us this directory. */
+/** This function is called whenever we successfully pull down some directory
+ * information. */
 void
-directory_has_arrived(time_t now, char *identity_digest)
+directory_info_has_arrived(time_t now, int from_cache)
 {
   or_options_t *options = get_options();
   /* XXXX011 NM Update this to reflect new directories.  In particular, we
    * can't start building circuits until we have descriptors and networkstatus
    * docs.*/
 
-  log_fn(LOG_INFO, "A directory has arrived.");
-
-  has_fetched_directory=1;
-  /* Don't try to upload or download anything for a while
-   * after the directory we had when we started.
-   */
-  if (!time_to_fetch_directory)
-    time_to_fetch_directory = now + get_dir_fetch_period(options);
-
-  if (!time_to_fetch_running_routers)
-    time_to_fetch_running_routers = now + get_status_fetch_period(options);
-
-  if (identity_digest) /* if it's fresh */
-    helper_nodes_set_status_from_directory();
+  if (!router_have_minimum_dir_info()) {
+    log_fn(LOG_NOTICE, "I know too little.");
+    return;
+  }
 
-  if (server_mode(options) && identity_digest) {
-    /* if this is us, then our dirport is reachable */
-    if (router_digest_is_me(identity_digest))
-      router_dirport_found_reachable();
+  if (!has_fetched_directory) {
+    log_fn(LOG_NOTICE, "We have enough directory information to build circuits.");
   }
 
+  has_fetched_directory=1;
+
   if (server_mode(options) &&
       !we_are_hibernating()) { /* connect to the appropriate routers */
     if (!authdir_mode(options))
       router_retry_connections(0);
-    if (identity_digest) /* we got a fresh directory */
+    if (!from_cache)
       consider_testing_reachability();
   }
 }
@@ -691,7 +680,6 @@ run_scheduled_events(time_t now)
    * new running-routers list, and/or force-uploading our descriptor
    * (if we've passed our internal checks). */
   if (time_to_fetch_directory < now) {
-    time_t next_status_fetch;
     /* purge obsolete entries */
     routerlist_remove_old_routers(ROUTER_MAX_AGE);
 
@@ -701,13 +689,16 @@ run_scheduled_events(time_t now)
       }
     }
 
-    directory_get_from_dirserver(DIR_PURPOSE_FETCH_DIR, NULL, 1);
-    time_to_fetch_directory = now + get_dir_fetch_period(options);
-    next_status_fetch = now + get_status_fetch_period(options);
-    if (time_to_fetch_running_routers < next_status_fetch) {
-      time_to_fetch_running_routers = next_status_fetch;
+    /* Only caches actually need to fetch directories now. */
+    if (options->DirPort && !options->V1AuthoritativeDir) {
+      directory_get_from_dirserver(DIR_PURPOSE_FETCH_DIR, NULL, 1);
     }
 
+    /* Try to get any routers we don't have. */
+    update_router_descriptor_downloads(now);
+
+    time_to_fetch_directory = now + get_dir_fetch_period(options);
+
     /* Also, take this chance to remove old information from rephist
      * and the rend cache. */
     rep_history_clean(now - options->RephistTrackTime);
@@ -986,10 +977,13 @@ do_main_loop(void)
   if (router_reload_router_list()) {
     return -1;
   }
-  /* load the networkstatuses. */
+  /* load the networkstatuses. (This launches a download for new routers as
+   * appropriate.)
+   */
   if (router_reload_networkstatus()) {
     return -1;
   }
+  directory_info_has_arrived(time(NULL),1);
 
   if (authdir_mode(get_options())) {
     /* the directory is already here, run startup things */

src/or/or.h

@@ -1789,7 +1789,7 @@ void connection_stop_writing(connection_t *conn);
 void connection_start_writing(connection_t *conn);
 
 void directory_all_unreachable(time_t now);
-void directory_has_arrived(time_t now, char *identity_digest);
+void directory_info_has_arrived(time_t now, int from_cache);
 
 int control_signal_act(int the_signal);
 void handle_signals(int is_parent);
@@ -2089,10 +2089,15 @@ void routerlist_free_all(void);
 routerinfo_t *routerinfo_copy(const routerinfo_t *router);
 void router_mark_as_down(const char *digest);
 void routerlist_remove_old_routers(int age);
-int router_add_to_routerlist(routerinfo_t *router, const char **msg);
+int router_add_to_routerlist(routerinfo_t *router, const char **msg,
+                             int from_cache);
 int router_load_single_router(const char *s, const char **msg);
+void router_load_routers_from_string(const char *s, int from_cache,
+                                     smartlist_t *requested_fingerprints);
+#if 0
 int router_load_routerlist_from_directory(const char *s,crypto_pk_env_t *pkey,
                                         int dir_is_recent, int dir_is_cached);
+#endif
 typedef enum { NS_FROM_CACHE, NS_FROM_DIR, NS_GENERATED} networkstatus_source_t;
 int router_set_networkstatus(const char *s, time_t arrived_at,
                              networkstatus_source_t source,
@@ -2120,9 +2125,11 @@ void clear_trusted_dir_servers(void);
 networkstatus_t *networkstatus_get_by_digest(const char *digest);
 void update_networkstatus_cache_downloads(time_t now);
 void update_networkstatus_client_downloads(time_t now);
+void update_router_descriptor_downloads(time_t now);
 void routers_update_all_from_networkstatus(void);
 void routers_update_status_from_networkstatus(smartlist_t *routers);
 smartlist_t *router_list_superseded(void);
+int router_have_minimum_dir_info(void);
 
 /********************************* routerparse.c ************************/
 
@@ -2153,19 +2160,21 @@ int router_get_networkstatus_v2_hash(const char *s, char *digest);
 int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
                                    crypto_pk_env_t *private_key);
 int router_parse_list_from_string(const char **s,
-                                  smartlist_t *dest,
-                                  time_t published);
+                                  smartlist_t *dest);
 int router_parse_routerlist_from_directory(const char *s,
                                            routerlist_t **dest,
                                            crypto_pk_env_t *pkey,
                                            int check_version,
                                            int write_to_cache);
 int router_parse_runningrouters(const char *str);
+int router_parse_directory(const char *str);
 routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
 int router_add_exit_policy_from_string(routerinfo_t *router, const char *s);
 addr_policy_t *router_parse_addr_policy_from_string(const char *s,
                                                     int assume_action);
+#if 0
 int check_software_version_against_directory(const char *directory);
+#endif
 int tor_version_parse(const char *s, tor_version_t *out);
 int tor_version_as_new_as(const char *platform, const char *cutoff);
 int tor_version_compare(tor_version_t *a, tor_version_t *b);

src/or/routerlist.c

@@ -29,7 +29,6 @@ static trusted_dir_server_t *router_pick_trusteddirserver_impl(
 static void mark_all_trusteddirservers_up(void);
 static int router_nickname_is_in_list(routerinfo_t *router, const char *list);
 static int router_nickname_matches(routerinfo_t *router, const char *nickname);
-static void router_normalize_routerlist(routerlist_t *rl);
 
 /****************************************************************************/
 
@@ -53,38 +52,6 @@ static smartlist_t *networkstatus_list = NULL;
  */
 static int networkstatus_list_has_changed = 0;
 
-/**
- * Reload the most recent cached directory (if present).
- */
-int
-router_reload_router_list(void)
-{
-  char filename[512];
-  int is_recent;
-  struct stat st;
-  char *s;
-  tor_assert(get_options()->DataDirectory);
-
-  tor_snprintf(filename,sizeof(filename),"%s/cached-directory",
-               get_options()->DataDirectory);
-  s = read_file_to_str(filename,0);
-  if (s) {
-    stat(filename, &st); /* if s is true, stat probably worked */
-    log_fn(LOG_INFO, "Loading cached directory from %s", filename);
-    is_recent = st.st_mtime > time(NULL) - 60*15;
-    if (router_load_routerlist_from_directory(s, NULL, is_recent, 1) < 0) {
-      log_fn(LOG_WARN, "Cached directory at '%s' was unparseable; ignoring.", filename);
-    }
-    if (routerlist &&
-        ((routerlist->published_on_xx > time(NULL) - MIN_ONION_KEY_LIFETIME/2)
-         || is_recent)) {
-      directory_has_arrived(st.st_mtime, NULL); /* do things we've been waiting to do */
-    }
-    tor_free(s);
-  }
-  return 0;
-}
-
 /** Repopulate our list of network_status_t objects from the list cached on
  * disk.  Return 0 on success, -1 on failure. */
 int
@@ -129,10 +96,10 @@ router_reload_networkstatus(void)
  *
  * Routerdescs are stored in a big file, named "cached-routers".  As new
  * routerdescs arrive, we append them to a journal file named
- * "cached-routers.jrn".
+ * "cached-routers.new".
  *
  * From time to time, we replace "cached-routers" with a new file containing
- * only the live, non-superseded descriptors, and clear cached-routers.log.
+ * only the live, non-superseded descriptors, and clear cached-routers.new.
  *
  * On startup, we read both files.
  */
@@ -198,6 +165,9 @@ router_rebuild_store(int force)
   if (!routerlist)
     return 0;
 
+  /* Don't save deadweight. */
+  routerlist_remove_old_routers(ROUTER_MAX_AGE);
+
   options = get_options();
   fname_len = strlen(options->DataDirectory)+32;
   fname = tor_malloc(fname_len);
@@ -240,6 +210,50 @@ router_rebuild_store(int force)
   return r;
 }
 
+/* DOCDOC */
+int
+router_reload_router_list(void)
+{
+  or_options_t *options = get_options();
+  size_t fname_len = strlen(options->DataDirectory)+32;
+  char *fname = tor_malloc(fname_len);
+  struct stat st;
+  int j;
+
+  if (!routerlist) {
+    routerlist = tor_malloc_zero(sizeof(routerlist_t));
+    routerlist->routers = smartlist_create();
+  }
+
+  router_journal_len = router_store_len = 0;
+
+  for (j = 0; j < 2; ++j) {
+    char *contents;
+    tor_snprintf(fname, fname_len,
+                 (j==0)?"%s/cached-routers":"%s/cached-routers.new",
+                 options->DataDirectory);
+    contents = read_file_to_str(fname, 0);
+    if (contents) {
+      stat(fname, &st);
+      if (j==0)
+        router_store_len = st.st_size;
+      else
+        router_journal_len = st.st_size;
+      router_load_routers_from_string(contents, 1, NULL);
+      tor_free(contents);
+    }
+  }
+
+  /* Don't cache expired routers. */
+  routerlist_remove_old_routers(ROUTER_MAX_AGE);
+
+  if (router_journal_len) {
+    /* Always clear the journal on startup.*/
+    router_rebuild_store(1);
+  }
+  return 0;
+}
+
 /** Set *<b>outp</b> to a smartlist containing a list of
  * trusted_dir_server_t * for all known trusted dirservers.  Callers
  * must not modify the list or its contents.
@@ -1072,10 +1086,15 @@ router_mark_as_down(const char *digest)
  * routerinfo was accepted, but we should notify the generator of the
  * descriptor using the message *<b>msg</b>.
  *
- * DOCDOC very changed.  Also, MUST call update_status_from_networkstatus first.
+ * DOCDOC very changed.  Also, MUST call update_status_from_networkstatus
+ * first, and should call router_rebuild_store and
+ * control_event_descriptors_changed after.
+ *
+ * XXXX never replace your own descriptor.
  */
 int
-router_add_to_routerlist(routerinfo_t *router, const char **msg)
+router_add_to_routerlist(routerinfo_t *router, const char **msg,
+                         int from_cache)
 {
   int i;
   char id_digest[DIGEST_LEN];
@@ -1138,6 +1157,9 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg)
         }
         routerinfo_free(old_router);
         smartlist_set(routerlist->routers, i, router);
+        if (!from_cache)
+          router_append_to_journal(router->signed_descriptor,
+                                   router->signed_descriptor_len);
         directory_set_dirty();
         *msg = unreachable ? "Dirserver believes your ORPort is unreachable" :
                authdir_verified ? "Verified server updated" :
@@ -1176,6 +1198,9 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg)
   /* We haven't seen a router with this name before.  Add it to the end of
    * the list. */
   smartlist_add(routerlist->routers, router);
+  if (!from_cache)
+    router_append_to_journal(router->signed_descriptor,
+                             router->signed_descriptor_len);
   directory_set_dirty();
   return 0;
 }
@@ -1244,7 +1269,7 @@ router_load_single_router(const char *s, const char **msg)
 #endif
   /* XXXX011 update router status from networkstatus!! */
 
-  if (router_add_to_routerlist(ri, msg)<0) {
+  if (router_add_to_routerlist(ri, msg, 0)<0) {
     log_fn(LOG_WARN, "Couldn't add router to list: %s Dropping.",
            *msg?*msg:"(No message).");
     /* we've already assigned to *msg now, and ri is already freed */
@@ -1260,6 +1285,48 @@ router_load_single_router(const char *s, const char **msg)
   return 1;
 }
 
+/* DOCDOC */
+void
+router_load_routers_from_string(const char *s, int from_cache,
+                                smartlist_t *requested_fingerprints)
+{
+  smartlist_t *routers = smartlist_create(), *changed = smartlist_create();
+  char fp[HEX_DIGEST_LEN+1];
+  const char *msg;
+
+  router_parse_list_from_string(&s, routers);
+
+  routers_update_status_from_networkstatus(routers);
+
+  SMARTLIST_FOREACH(routers, routerinfo_t *, ri,
+  {
+    base16_encode(fp, sizeof(fp), ri->identity_digest, DIGEST_LEN);
+    if (requested_fingerprints) {
+      if (smartlist_string_isin(requested_fingerprints, fp)) {
+        smartlist_string_remove(requested_fingerprints, fp);
+      } else {
+        char *requested =
+          smartlist_join_strings(requested_fingerprints," ",0,NULL);
+        log_fn(LOG_WARN, "We received a router descriptor with a fingerprint (%s) that we never requested. (We asked for: %s.) Dropping.", fp, requested);
+        tor_free(requested);
+        routerinfo_free(ri);
+        continue;
+      }
+    }
+
+    if (router_add_to_routerlist(ri, &msg, from_cache) >= 0)
+      smartlist_add(changed, ri);
+  });
+
+  control_event_descriptors_changed(changed);
+
+  router_rebuild_store(0);
+
+  smartlist_free(routers);
+  smartlist_free(changed);
+}
+
+#if 0
 /** Add to the current routerlist each router stored in the
  * signed directory <b>s</b>.  If pkey is provided, check the signature
  * against pkey; else check against the pkey of the signing directory
@@ -1291,7 +1358,7 @@ router_load_routerlist_from_directory(const char *s,
     SMARTLIST_FOREACH(new_list->routers, routerinfo_t *, r,
     {
       const char *msg;
-      if (router_add_to_routerlist(r,&msg)>=0)
+      if (router_add_to_routerlist(r,&msg,0)>=0)
         smartlist_add(changed, r);
     });
     smartlist_clear(new_list->routers);
@@ -1306,6 +1373,7 @@ router_load_routerlist_from_directory(const char *s,
   router_normalize_routerlist(routerlist);
   return 0;
 }
+#endif
 
 /** Helper: return a newly allocated string containing the name of the filename
  * where we plan to cache <b>ns</b>. */
@@ -1319,7 +1387,6 @@ networkstatus_get_cache_filename(const networkstatus_t *ns)
   base16_encode(fp, HEX_DIGEST_LEN+1, ns->identity_digest, DIGEST_LEN);
   tor_snprintf(fn, len, "%s/cached-status/%s",datadir,fp);
   return fn;
-
 }
 
 /** Helper for smartlist_sort: Compare two networkstatus objects by
@@ -1696,28 +1763,6 @@ update_networkstatus_client_downloads(time_t now)
   tor_free(resource);
 }
 
-/** Ensure that our own routerinfo is at the front, and remove duplicates
- * of our routerinfo.
- */
-static void
-router_normalize_routerlist(routerlist_t *rl)
-{
-  int i=0;
-  routerinfo_t *r;
-  if ((r = router_get_my_routerinfo())) {
-    smartlist_insert(rl->routers, 0, routerinfo_copy(r));
-    ++i;
-  }
-
-  for ( ; i < smartlist_len(rl->routers); ++i) {
-    r = smartlist_get(rl->routers,i);
-    if (router_is_me(r)) {
-      routerinfo_free(r);
-      smartlist_del_keeporder(rl->routers, i--);
-    }
-  }
-}
-
 /** Decide whether a given addr:port is definitely accepted,
  * definitely rejected, probably accepted, or probably rejected by a
  * given policy.  If <b>addr</b> is 0, we don't know the IP of the
@@ -2192,6 +2237,8 @@ routers_update_all_from_networkstatus(void)
 
   helper_nodes_set_status_from_directory();
 
+  update_router_descriptor_downloads(time(NULL));
+
   networkstatus_list_has_changed = 0;
 }
 
@@ -2287,20 +2334,21 @@ routers_update_status_from_networkstatus(smartlist_t *routers)
   });
 }
 
-/** Return new list of ID digests for superseded routers.  A router is
+/** Return new list of ID fingerprints for superseded routers.  A router is
  * superseded if any network-status has a router with a different digest
  * published more recently, or it it is listed in the network-status but not
  * in the router list.
  */
 smartlist_t *
-router_list_superseded(void)
+router_list_downloadable(void)
 {
   smartlist_t *superseded = smartlist_create();
   strmap_t *most_recent = NULL;
   char fp[HEX_DIGEST_LEN+1];
   routerstatus_t *rs_old;
+  strmap_iter_t *iter;
 
-  if (!routerlist || !networkstatus_list)
+  if (!networkstatus_list)
     return superseded;
 
   /* Build a map from fingerprint to most recent routerstatus_t. If this
@@ -2313,29 +2361,93 @@ router_list_superseded(void)
     {
       base16_encode(fp, sizeof(fp), rs->identity_digest, DIGEST_LEN);
       rs_old = strmap_get(most_recent, fp);
-      if (!rs_old || rs_old->published_on < rs->published_on)
+      if (!rs_old || rs_old->published_on < rs->published_on) {
         strmap_set(most_recent, fp, rs);
+      }
     });
   });
 
   /* Compare each router to the most recent routerstatus. */
-  SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
-  {
-    routerstatus_t *rs;
-    base16_encode(fp, sizeof(fp), ri->identity_digest, DIGEST_LEN);
-    rs = strmap_get(most_recent, fp);
-    if (!rs)
-      continue;
-    if (memcmp(ri->signed_descriptor_digest,rs->descriptor_digest,DIGEST_LEN)
-        && rs->published_on > ri->published_on) {
-      char *d = tor_malloc(DIGEST_LEN);
-      memcpy(d, ri->identity_digest, DIGEST_LEN);
-      smartlist_add(superseded, d);
-      break;
-    }
-  });
+  if (routerlist) {
+    SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
+    {
+      routerstatus_t *rs;
+      base16_encode(fp, sizeof(fp), ri->identity_digest, DIGEST_LEN);
+      rs = strmap_get(most_recent, fp);
+      if (!rs)
+        continue;
+      if (memcmp(ri->signed_descriptor_digest,rs->descriptor_digest,DIGEST_LEN)
+          && rs->published_on > ri->published_on) {
+        char *d = tor_malloc(HEX_DIGEST_LEN+1);
+        base16_encode(d, HEX_DIGEST_LEN+1, ri->identity_digest, DIGEST_LEN);
+        smartlist_add(superseded, d);
+        break;
+      }
+      strmap_remove(most_recent, fp);
+    });
+  }
+
+  /* Anything left over, we don't even know about yet. */
+  for (iter = strmap_iter_init(most_recent); !strmap_iter_done(iter);
+       iter = strmap_iter_next(most_recent, iter)) {
+    const char *key;
+    void *val;
+    strmap_iter_get(iter, &key, &val);
+    smartlist_add(superseded, tor_strdup(key));
+  }
+
   strmap_free(most_recent, NULL);
 
   return superseded;
 }
 
+/* DOCDOC */
+void
+update_router_descriptor_downloads(time_t now)
+{
+  char *resource = NULL;
+
+  if (connection_get_by_type_purpose(CONN_TYPE_DIR,
+                                     DIR_PURPOSE_FETCH_SERVERDESC))
+    return;
+
+  if (!networkstatus_list || smartlist_len(networkstatus_list)<2) {
+    resource = tor_strdup("all.z");
+  } else {
+    smartlist_t *downloadable = router_list_downloadable();
+    if (smartlist_len(downloadable)) {
+      char *dl = smartlist_join_strings(downloadable,"+",0,NULL);
+      size_t r_len = smartlist_len(downloadable)*(DIGEST_LEN+1)+16;
+      /* Damn, that's an ugly way to do this. XXXX011 NM */
+      resource = tor_malloc(r_len);
+      tor_snprintf(resource, r_len, "fp/%s.z", dl);
+      tor_free(dl);
+    }
+    SMARTLIST_FOREACH(downloadable, char *, c, tor_free(c));
+    smartlist_free(downloadable);
+  }
+
+  if (!resource) {
+    log_fn(LOG_NOTICE, "No routers to download.");
+    return;
+  }
+
+  log_fn(LOG_NOTICE, "Launching request for routers: %s", resource);
+  directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,resource,1);
+  tor_free(resource);
+}
+
+/* DOCDOC */
+int
+router_have_minimum_dir_info(void)
+{
+  int tot = 0, avg;
+  if (!networkstatus_list || smartlist_len(networkstatus_list)<2 ||
+      !routerlist)
+    return 0;
+  SMARTLIST_FOREACH(networkstatus_list, networkstatus_t *, ns,
+                    tot += smartlist_len(ns->entries));
+  avg = tot / smartlist_len(networkstatus_list);
+  return smartlist_len(routerlist->routers) > (avg/4);
+}
+

src/or/routerparse.c

@@ -247,6 +247,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest,
   return -1;
 }
 
+#if 0
 /**
  * Find the first instance of "recommended-software ...\n" at the start of
  * a line; return a newly allocated string containing the "..." portion.
@@ -273,6 +274,7 @@ get_recommended_software_from_directory(const char *str)
   return tor_strndup(cp, eol-cp);
 #undef REC
 }
+#endif
 
 /** Return 1 if <b>myversion</b> is not in <b>versionlist</b>, and if at least
  * one version of Tor on <b>versionlist</b> is newer than <b>myversion</b>.
@@ -373,6 +375,7 @@ get_recommended_software_from_directory(const char *str)
   return ret;
 }
 
+#if 0
 /* Return 0 if myversion is supported; else warn and return -1. */
 int
 check_software_version_against_directory(const char *directory)
@@ -394,38 +397,24 @@ check_software_version_against_directory(const char *directory)
   tor_free(v);
   return -1;
 }
+#endif
 
-/** Parse a directory from <b>str</b> and, when done, store the
- * resulting routerlist in *<b>dest</b>, freeing the old value if
- * necessary.
- *
- * If <b>pkey</b> is provided, we check the directory signature with pkey.
- *
- * If <b>check_version</b> is non-zero, then examine the
- * Recommended-versions * line in the directory, and warn or quit
- * as needed.
- *
- * If <b>write_to_cache</b> is non-zero, then store this directory in
- * memory and/or disk as well.
+/** Read a signed directory from <b>str</b>.  If it's well-formed, return 0.
+ * Otherwise, return -1.  If we're a directory cache, cache it.
  */
-int /* Should be static; exposed for unit tests */
-router_parse_routerlist_from_directory(const char *str,
-                                       routerlist_t **dest,
-                                       crypto_pk_env_t *pkey,
-                                       int check_version,
-                                       int write_to_cache)
+int
+router_parse_directory(const char *str)
 {
   directory_token_t *tok;
   char digest[DIGEST_LEN];
-  routerlist_t *new_dir = NULL;
-  char *versions = NULL;
   time_t published_on;
   int r;
   const char *end, *cp;
   smartlist_t *tokens = NULL;
-  char dirnickname[MAX_NICKNAME_LEN+1];
   crypto_pk_env_t *declared_key = NULL;
 
+  /* XXXX011 This could be simplified a lot! NM */
+
   if (router_get_dir_hash(str, digest)) {
     log_fn(LOG_WARN, "Unable to compute digest of directory");
     goto err;
@@ -452,24 +441,13 @@ router_parse_routerlist_from_directory(const char *str,
     log_fn(LOG_WARN,"Expected a single directory signature"); goto err;
   }
   declared_key = find_dir_signing_key(str);
-  if (check_directory_signature(digest, tok, pkey, declared_key, 1)<0)
+  if (check_directory_signature(digest, tok, NULL, declared_key, 1)<0)
     goto err;
 
-  /* now we know tok->n_args == 1, so it's safe to access tok->args[0] */
-  if (!is_legal_nickname(tok->args[0])) {
-    log_fn(LOG_WARN, "Directory nickname '%s' is misformed", tok->args[0]);
-    goto err;
-  }
-  strlcpy(dirnickname, tok->args[0], sizeof(dirnickname));
-
   SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
   smartlist_free(tokens);
   tokens = NULL;
 
-  /* Now that we know the signature is okay, check the version. */
-  if (check_version)
-    check_software_version_against_directory(str);
-
   /* Now try to parse the first part of the directory. */
   if ((end = strstr(str,"\nrouter "))) {
     ++end;
@@ -483,20 +461,6 @@ router_parse_routerlist_from_directory(const char *str,
   if (tokenize_string(str,end,tokens,DIR)) {
     log_fn(LOG_WARN, "Error tokenizing directory"); goto err;
   }
-  if (smartlist_len(tokens) < 1) {
-    log_fn(LOG_WARN, "Impossibly short directory header"); goto err;
-  }
-  if ((tok = find_first_by_keyword(tokens, _UNRECOGNIZED))) {
-    log_fn(LOG_WARN, "Unrecognized keyword \"%s\" in directory header; can't parse directory.",
-           tok->args[0]);
-    goto err;
-  }
-
-  tok = smartlist_get(tokens,0);
-  if (tok->tp != K_SIGNED_DIRECTORY) {
-    log_fn(LOG_WARN, "Directory doesn't start with signed-directory.");
-    goto err;
-  }
 
   if (!(tok = find_first_by_keyword(tokens, K_PUBLISHED))) {
     log_fn(LOG_WARN, "Missing published time on directory.");
@@ -510,54 +474,13 @@ router_parse_routerlist_from_directory(const char *str,
 
   /* Now that we know the signature is okay, and we have a
    * publication time, cache the directory. */
-  if (!get_options()->AuthoritativeDir && write_to_cache)
+  if (get_options()->DirPort && !get_options()->V1AuthoritativeDir)
     dirserv_set_cached_directory(str, published_on, 0);
 
-  if (!(tok = find_first_by_keyword(tokens, K_RECOMMENDED_SOFTWARE))) {
-    log_fn(LOG_WARN, "Missing recommended-software line from directory.");
-    goto err;
-  }
-  if (tok->n_args > 1) {
-    log_fn(LOG_WARN, "Invalid recommended-software line");
-    goto err;
-  }
-  versions = tok->n_args ? tor_strdup(tok->args[0]) : tor_strdup("");
-
-  /* Prefer router-status, then running-routers. */
-  if (!(tok = find_first_by_keyword(tokens, K_ROUTER_STATUS))) {
-    log_fn(LOG_WARN,
-           "Missing router-status line from directory.");
-    goto err;
-  }
-
-  /* Read the router list from s, advancing s up past the end of the last
-   * router. */
-  str = end;
-  new_dir = tor_malloc_zero(sizeof(routerlist_t));
-  new_dir->routers = smartlist_create();
-  if (router_parse_list_from_string(&str, new_dir->routers,
-                                    published_on)) {
-    log_fn(LOG_WARN, "Error reading routers from directory");
-    goto err;
-  }
-
-  new_dir->published_on_xx = published_on;
-
-  SMARTLIST_FOREACH(tokens, directory_token_t *, tok, token_free(tok));
-  smartlist_free(tokens);
-  tokens = NULL;
-
-  if (*dest)
-    routerlist_free(*dest);
-  *dest = new_dir;
-
   r = 0;
   goto done;
  err:
   r = -1;
-  if (new_dir)
-    routerlist_free(new_dir);
-  tor_free(versions);
  done:
   if (declared_key) crypto_free_pk_env(declared_key);
   if (tokens) {
@@ -567,8 +490,9 @@ router_parse_routerlist_from_directory(const char *str,
   return r;
 }
 
-/** Read a signed router status statement from <b>str</b>.  If it's well-formed,
- * return 0.  Otherwise, return -1.  If we're a directory cache, cache it.*/
+/** Read a signed router status statement from <b>str</b>.  If it's
+ * well-formed, return 0.  Otherwise, return -1.  If we're a directory cache,
+ * cache it.*/
 int
 router_parse_runningrouters(const char *str)
 {
@@ -761,8 +685,7 @@ check_directory_signature(const char *digest,
  * following the last router entry.  Returns 0 on success and -1 on failure.
  */
 int
-router_parse_list_from_string(const char **s, smartlist_t *dest,
-                              time_t published_on)
+router_parse_list_from_string(const char **s, smartlist_t *dest)
 {
   routerinfo_t *router;
   smartlist_t *routers;