More work towards a sane digest-based liveness testing.

svn:r5238

doc/TODO

@@ -43,9 +43,22 @@ R - are dirservers auto-verifying duplicate nicknames?
 
   o controller libs should support resetconf command.
 N . Additional controller features
-      - Find a way to make event info more extensible
+      o Find a way to make event info more extensible
       - change circuit status events to give more details, like purpose,
-        whether they're internal, etc.
+        whether they're internal, when they become dirty, when they become
+        too dirty for further circuits, etc.
+R       - What do we want here, exactly?
+N       - Specify and implement it.
+      - Change stream status events analogously.
+R       - What do we want here, exactly?
+N       - Specify and implement it.
+      - Make other events "better".
+      - Change stream status events analogously.
+R       - What do we want here, exactly?
+N       - Specify and implement it.
+      - Make other events "better" analogously
+R       - What do we want here, exactly?
+N       - Specify and implement it.
       . Expose more information via getinfo:
         - import and export rendezvous descriptors
         - Review all static fields for additional candidates

doc/control-spec.txt

@@ -167,7 +167,7 @@ $Id$
   Request the server to inform the client about interesting events.  The
   syntax is:
 
-     "SETEVENTS" *(SP EventCode) CRLF
+     "SETEVENTS" [SP "EXTENDED"] *(SP EventCode) CRLF
 
      EventCode = "CIRC" / "STREAM" / "ORCONN" / "BW" / "DEBUG" /
          "INFO" / "NOTICE" / "WARN" / "ERR" / "NEWDESC" / "ADDRMAP"
@@ -179,6 +179,12 @@ $Id$
   Unrecognized event" reply if one of the event codes isn't recognized.  (On
   error, the list of active event codes isn't changed.)
 
+  If the flag string "EXTENDED" is provided, Tor may provide extra
+  information with events for this connection; see 4.1 for more information.
+  NOTE: All events on a given connection will be provided in extended format,
+  or none.
+  NOTE: "EXTENDED" is only supported in Tor 0.1.1.9-alpha or later.
+
 3.5. AUTHENTICATE
 
   Sent from the client to the server.  The syntax is:
@@ -579,10 +585,19 @@ $Id$
   expected.  For instance, a client that expects a CIRC message like:
       650 CIRC 1000 EXTENDED moria1,moria2
   should tolerate:
-      650+CIRC 1000 EXTENDED moria1,moria2 0xBEEF
+      650-CIRC 1000 EXTENDED moria1,moria2 0xBEEF
       650-EXTRAMAGIC=99
       650 ANONYMITY=high
 
+  If clients ask for extended events, then each event line as specified below
+  will be followed by additional extensions.  Clients that do so MUST
+  tolerate additional arguments and lines.  Additional lines will be of the
+  form
+      "650" ("-"/" ") KEYWORD ["=" ARGUMENTS] CRLF
+  Additional arguments will be of the form
+      SP KEYWORD ["=" ( QuoutedString / * NonSpDquote ) ]
+  Such clients MUST tolerate lines with keywords they do not recognize.
+
 4.1.1. Circuit status changed
 
    The syntax is:

src/or/config.c

@@ -2800,6 +2800,32 @@ config_parse_addr_policy(config_line_t *cfg,
   return r;
 }
 
+/** Compare two provided address policies, and return -1, 0, or 1 if the first
+ * is less than, equal to, or greater than the second. */
+int
+config_cmp_addr_policies(addr_policy_t *a, addr_policy_t *b)
+{
+  int r;
+  while (a && b) {
+    if (r=((int)a->addr - (int)b->addr))
+      return r;
+    if (r=((int)a->msk - (int)b->msk))
+      return r;
+    if (r=((int)a->prt_min - (int)b->prt_min))
+      return r;
+    if (r=((int)a->prt_max - (int)b->prt_max))
+      return r;
+    a = a->next;
+    b = b->next;
+  }
+  if (!a && !b)
+    return 0;
+  if (a)
+    return -1;
+  if (b)
+    return 1;
+}
+
 /** Release all storage held by <b>p</b> */
 void
 addr_policy_free(addr_policy_t *p)

src/or/or.h

@@ -1489,6 +1489,7 @@ int options_init_logs(or_options_t *options, int validate_only);
 int config_parse_addr_policy(config_line_t *cfg,
                              addr_policy_t **dest,
                              int assume_action);
+int config_cmp_addr_policies(addr_policy_t *a, addr_policy_t *b);
 void options_append_default_exit_policy(addr_policy_t **policy);
 void addr_policy_free(addr_policy_t *p);
 int option_is_recognized(const char *key);
@@ -2165,12 +2166,14 @@ void update_networkstatus_downloads(time_t now);
 void update_router_descriptor_downloads(time_t now);
 void routers_update_all_from_networkstatus(void);
 void routers_update_status_from_networkstatus(smartlist_t *routers,
-                                              int reset_failures);
+                                              int reset_failures,
+                                              int assume_recognized);
 smartlist_t *router_list_superseded(void);
 int router_have_minimum_dir_info(void);
 void networkstatus_list_update_recent(time_t now);
 void router_reset_descriptor_download_failures(void);
 void router_reset_status_download_failures(void);
+int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2);
 
 /********************************* routerparse.c ************************/
 

src/or/routerlist.c

@@ -1237,6 +1237,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
     if (authdir_wants_to_reject_router(router, msg))
       return -2;
     authdir_verified = router->is_verified;
+    /*
+  } else {
+    if (! router->xx_is_recognized) {
+      log_fn(LOG_WARN, "Dropping unrecognized descriptor for router '%s'",
+             router->nickname);
+      return -1;
+    }
+    */
   }
 
   /* If we have a router with this name, and the identity key is the same,
@@ -1386,7 +1394,7 @@ router_load_single_router(const char *s, const char **msg)
 
   lst = smartlist_create();
   smartlist_add(lst, ri);
-  routers_update_status_from_networkstatus(lst, 0);
+  routers_update_status_from_networkstatus(lst, 0, 1);
 
   if (router_add_to_routerlist(ri, msg, 0)<0) {
     log_fn(LOG_WARN, "Couldn't add router to list: %s Dropping.",
@@ -1418,17 +1426,14 @@ router_load_routers_from_string(const char *s, int from_cache,
   smartlist_t *routers = smartlist_create(), *changed = smartlist_create();
   char fp[HEX_DIGEST_LEN+1];
   const char *msg;
-  int xx_n_unrecognized = 0;
 
   router_parse_list_from_string(&s, routers);
 
-  routers_update_status_from_networkstatus(routers, !from_cache);
+  routers_update_status_from_networkstatus(routers, !from_cache, from_cache);
 
   SMARTLIST_FOREACH(routers, routerinfo_t *, ri,
   {
     base16_encode(fp, sizeof(fp), ri->identity_digest, DIGEST_LEN);
-    if (! ri->xx_is_recognized)
-      ++xx_n_unrecognized;
     if (requested_fingerprints) {
       if (smartlist_string_isin(requested_fingerprints, fp)) {
         smartlist_string_remove(requested_fingerprints, fp);
@@ -1446,12 +1451,6 @@ router_load_routers_from_string(const char *s, int from_cache,
       smartlist_add(changed, ri);
   });
 
-  if (xx_n_unrecognized && !from_cache) {
-    log_fn(LOG_WARN, "Under proposed rules I would reject %d of the %d desc(s)"
-           " I just downloaded because no networkstatus can confirm their"
-           " digest(s).", xx_n_unrecognized, smartlist_len(routers));
-  }
-
   control_event_descriptors_changed(changed);
 
   router_rebuild_store(0);
@@ -2181,7 +2180,7 @@ routers_update_all_from_networkstatus(void)
   if (networkstatus_list_has_changed)
     routerstatus_list_update_from_networkstatus(now);
 
-  routers_update_status_from_networkstatus(routerlist->routers, 0);
+  routers_update_status_from_networkstatus(routerlist->routers, 0, 0);
 
   me = router_get_my_routerinfo();
   if (me && !have_warned_about_unverified_status) {
@@ -2522,7 +2521,7 @@ routerstatus_list_update_from_networkstatus(time_t now)
  * is_named, is_verified, and is_running fields according to our current
  * networkstatus_t documents. */
 void
-routers_update_status_from_networkstatus(smartlist_t *routers, int reset_failures)
+routers_update_status_from_networkstatus(smartlist_t *routers, int reset_failures, int assume_recognized)
 {
   trusted_dir_server_t *ds;
   local_routerstatus_t *rs;
@@ -2534,6 +2533,8 @@ routers_update_status_from_networkstatus(smartlist_t *routers, int reset_failure
   if (!routerstatus_list)
     return;
 
+  log_fn(LOG_NOTICE, "Here, %d %d", reset_failures, assume_recognized);
+
   SMARTLIST_FOREACH(routers, routerinfo_t *, router,
   {
     rs = router_get_combined_status_by_digest(router->identity_digest);
@@ -2542,11 +2543,6 @@ routers_update_status_from_networkstatus(smartlist_t *routers, int reset_failure
     if (!rs)
       continue;
 
-    if (reset_failures) {
-      rs->n_download_failures = 0;
-      rs->next_attempt_at = 0;
-    }
-
     if (!namingdir)
       router->is_named = rs->status.is_named;
 
@@ -2558,11 +2554,19 @@ routers_update_status_from_networkstatus(smartlist_t *routers, int reset_failure
     if (router->is_running && ds) {
       ds->n_networkstatus_failures = 0;
     }
-    if (!router->xx_is_recognized) {
-      router->xx_is_recognized = routerdesc_digest_is_recognized(
+    if (assume_recognized) {
+      router->xx_is_recognized = 1;
+    } else {
+      if (!router->xx_is_recognized) {
+        router->xx_is_recognized = routerdesc_digest_is_recognized(
                 router->identity_digest, router->signed_descriptor_digest);
+      }
+      router->xx_is_extra_new = router->published_on > rs->status.published_on;
+    }
+    if (reset_failures && router->xx_is_recognized) {
+      rs->n_download_failures = 0;
+      rs->next_attempt_at = 0;
     }
-    router->xx_is_extra_new = router->published_on > rs->status.published_on;
   });
 }
 
@@ -2854,3 +2858,53 @@ router_reset_descriptor_download_failures(void)
   last_routerdesc_download_attempted = 0;
 }
 
+/** Return true iff the only differences between r1 and r2 are such that
+ * would not cause a recent (post 0.1.1.6) direserver to republish.
+ */
+int
+router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2)
+{
+  /* post-0.1.1.6 servers know what they're doing. */
+  if (tor_version_as_new_as(r1->platform, "0.1.1.6-alpha") ||
+      tor_version_as_new_as(r1->platform, "0.1.1.6-alpha"))
+    return 0;
+
+  /* If any key fields differ, they're different. */
+  if (strcasecmp(r1->address, r2->address) ||
+      strcasecmp(r1->nickname, r2->nickname) ||
+      r1->or_port != r2->or_port ||
+      r1->dir_port != r2->dir_port ||
+      crypto_pk_cmp_keys(r1->onion_key, r2->onion_key) ||
+      crypto_pk_cmp_keys(r1->identity_pkey, r2->onion_pkey) ||
+      strcasecmp(r1->platform, r2->platform) ||
+      strcasecmp(r1->contact_info, r2->contact_info) ||
+      r1->is_hibernating != r2->is_hibernating ||
+      config_cmp_addr_policies(r1->exit_policy, r2->exit_policy))
+    return 0;
+  if ((r1->declared_family == NULL) != (r2->declared_family == NULL))
+    return 0;
+  if (r1->declared_family && r2->declared_family) {
+    int i, n;
+    if (smartlist_len(r1->declared_family)!=smartlist_len(r2->declared_family))
+      return 0;
+    n = smartlist_len(r1->declared_family);
+    for (i=0; i < n; ++i) {
+      if (strcasecmp(smartlist_get(r1->declared_family, i),
+                     smartlist_get(r2->declared_family, i)))
+        return 0;
+    }
+  }
+
+  /* Did bandwidth change a lot? */
+  if ((r1->bandwidthcapacity < r2->bandwidthcapacity/2) ||
+      (r2->bandwidthcapacity < r1->bandwidthcapacity/2))
+    return 0;
+
+  /* Did more than 6 hours pass? */
+  if (r1->published_on + 6*60*60 < r2->published_on ||
+      r2->published_on + 6*60*60 < r1->published_on)
+    return 0;
+
+  /* Otherwise, the difference is cosmetic. */
+  return 1;
+}