Merge branch 'bug2286_unit_test_squashed'

changes/bug2286

@@ -0,0 +1,5 @@
+  o Major features (directory authority):
+    - Directory authorities now support a new consensus method (17)
+      where they cap the published bandwidth of servers for which
+      insufficient bandwidth measurements exist. Fixes part of bug
+      2286.

src/or/dirserv.c

@@ -2101,13 +2101,15 @@ version_from_platform(const char *platform)
  *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
  *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
  *        consensus entry.
- *   NS_V3_VOTE - Output a complete V3 NS vote
+ *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
+ *        it contains additional information for the vote.
  *   NS_CONTROL_PORT - Output a NS document for the control port
  */
 int
 routerstatus_format_entry(char *buf, size_t buf_len,
                           const routerstatus_t *rs, const char *version,
-                          routerstatus_format_type_t format)
+                          routerstatus_format_type_t format,
+                          const vote_routerstatus_t *vrs)
 {
   int r;
   char *cp;
@@ -2253,10 +2255,10 @@ routerstatus_format_entry(char *buf, size_t buf_len,
       return -1;
     }
     cp += strlen(cp);
-    if (format == NS_V3_VOTE && rs->has_measured_bw) {
+    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
       *--cp = '\0'; /* Kill "\n" */
       r = tor_snprintf(cp, buf_len - (cp-buf),
-                       " Measured=%d\n", rs->measured_bw);
+                       " Measured=%d\n", vrs->measured_bw);
       if (r<0) {
         log_warn(LD_BUG, "Not enough space in buffer for weight line.");
         return -1;
@@ -2639,12 +2641,12 @@ int
 measured_bw_line_apply(measured_bw_line_t *parsed_line,
                        smartlist_t *routerstatuses)
 {
-  routerstatus_t *rs = NULL;
+  vote_routerstatus_t *rs = NULL;
   if (!routerstatuses)
     return 0;
 
   rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
-                         compare_digest_to_routerstatus_entry);
+                         compare_digest_to_vote_routerstatus_entry);
 
   if (rs) {
     rs->has_measured_bw = 1;
@@ -2659,7 +2661,7 @@ measured_bw_line_apply(measured_bw_line_t *parsed_line,
 
 /**
  * Read the measured bandwidth file and apply it to the list of
- * routerstatuses. Returns -1 on error, 0 otherwise.
+ * vote_routerstatus_t. Returns -1 on error, 0 otherwise.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
@@ -2701,7 +2703,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   }
 
   if (routerstatuses)
-    smartlist_sort(routerstatuses, compare_routerstatus_entries);
+    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
 
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
@@ -3052,7 +3054,8 @@ generate_v2_networkstatus_opinion(void)
       if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
         clear_status_flags_on_sybil(&rs);
 
-      if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) {
+      if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2,
+                                    NULL)) {
         log_warn(LD_BUG, "Unable to print router status.");
         tor_free(version);
         goto done;

src/or/dirserv.h

@@ -131,7 +131,8 @@ size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed);
 
 int routerstatus_format_entry(char *buf, size_t buf_len,
                               const routerstatus_t *rs, const char *platform,
-                              routerstatus_format_type_t format);
+                              routerstatus_format_type_t format,
+                              const vote_routerstatus_t *vrs);
 void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
 cached_dir_t *new_cached_dir(char *s, time_t published);

src/or/dirvote.c

@@ -212,7 +212,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                           vrs) {
     vote_microdesc_hash_t *h;
     if (routerstatus_format_entry(outp, endp-outp, &vrs->status,
-                                  vrs->version, NS_V3_VOTE) < 0) {
+                                  vrs->version, NS_V3_VOTE, vrs) < 0) {
       log_warn(LD_BUG, "Unable to print router status.");
       goto err;
     }
@@ -1388,6 +1388,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   char *client_versions = NULL, *server_versions = NULL;
   smartlist_t *flags;
   const char *flavor_name;
+  uint32_t max_unmeasured_bw = DEFAULT_MAX_UNMEASURED_BW;
   int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */
   const routerstatus_format_type_t rs_format =
     flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
@@ -1586,6 +1587,30 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_free(dir_sources);
   }
 
+  if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) {
+    char *max_unmeasured_param = NULL;
+    /* XXXX Extract this code into a common function */
+    if (params) {
+      if (strcmpstart(params, "maxunmeasuredbw=") == 0)
+        max_unmeasured_param = params;
+      else
+        max_unmeasured_param = strstr(params, " maxunmeasuredbw=");
+    }
+    if (max_unmeasured_param) {
+      int ok = 0;
+      char *eq = strchr(max_unmeasured_param, '=');
+      if (eq) {
+        max_unmeasured_bw = (uint32_t)
+          tor_parse_ulong(eq+1, 10, 1, UINT32_MAX, &ok, NULL);
+        if (!ok) {
+          log_warn(LD_DIR, "Bad element '%s' in max unmeasured bw param",
+                   escaped(max_unmeasured_param));
+          max_unmeasured_bw = DEFAULT_MAX_UNMEASURED_BW;
+        }
+      }
+    }
+  }
+
   /* Add the actual router entries. */
   {
     int *index; /* index[j] is the current index into votes[j]. */
@@ -1612,6 +1637,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     int *named_flag; /* Index of the flag "Named" for votes[j] */
     int *unnamed_flag; /* Index of the flag "Unnamed" for votes[j] */
     int chosen_named_idx;
+    int n_authorities_measuring_bandwidth;
 
     strmap_t *name_to_id_map = strmap_new();
     char conflict[DIGEST_LEN];
@@ -1700,6 +1726,14 @@ networkstatus_compute_consensus(smartlist_t *votes,
       } SMARTLIST_FOREACH_END(v);
     }
 
+    /* We need to know how many votes measure bandwidth. */
+    n_authorities_measuring_bandwidth = 0;
+    SMARTLIST_FOREACH(votes, networkstatus_t *, v,
+       if (v->has_measured_bws) {
+         ++n_authorities_measuring_bandwidth;
+       }
+    );
+
     /* Now go through all the votes */
     flag_counts = tor_malloc(sizeof(int) * smartlist_len(flags));
     while (1) {
@@ -1769,8 +1803,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
         }
 
         /* count bandwidths */
-        if (rs->status.has_measured_bw)
-          measured_bws[num_mbws++] = rs->status.measured_bw;
+        if (rs->has_measured_bw)
+          measured_bws[num_mbws++] = rs->measured_bw;
 
         if (rs->status.has_bandwidth)
           bandwidths[num_bandwidths++] = rs->status.bandwidth;
@@ -1863,10 +1897,19 @@ networkstatus_compute_consensus(smartlist_t *votes,
       /* Pick a bandwidth */
       if (consensus_method >= 6 && num_mbws > 2) {
         rs_out.has_bandwidth = 1;
+        rs_out.bw_is_unmeasured = 0;
         rs_out.bandwidth = median_uint32(measured_bws, num_mbws);
       } else if (consensus_method >= 5 && num_bandwidths > 0) {
         rs_out.has_bandwidth = 1;
+        rs_out.bw_is_unmeasured = 1;
         rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths);
+        if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW &&
+            n_authorities_measuring_bandwidth > 2) {
+          /* Cap non-measured bandwidths. */
+          if (rs_out.bandwidth > max_unmeasured_bw) {
+            rs_out.bandwidth = max_unmeasured_bw;
+          }
+        }
       }
 
       /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */
@@ -1987,7 +2030,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         /* Okay!! Now we can write the descriptor... */
         /*     First line goes into "buf". */
         routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL,
-                                  rs_format);
+                                  rs_format, NULL);
         smartlist_add(chunks, tor_strdup(buf));
       }
       /*     Now an m line, if applicable. */
@@ -2008,7 +2051,10 @@ networkstatus_compute_consensus(smartlist_t *votes,
       smartlist_add(chunks, tor_strdup("\n"));
       /*     Now the weight line. */
       if (rs_out.has_bandwidth) {
-        smartlist_add_asprintf(chunks, "w Bandwidth=%d\n", rs_out.bandwidth);
+        int unmeasured = rs_out.bw_is_unmeasured &&
+          consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW;
+        smartlist_add_asprintf(chunks, "w Bandwidth=%d%s\n", rs_out.bandwidth,
+                               unmeasured?" Unmeasured=1":"");
       }
 
       /*     Now the exitpolicy summary line. */
@@ -2051,6 +2097,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     // Parse params, extract BW_WEIGHT_SCALE if present
     // DO NOT use consensus_param_bw_weight_scale() in this code!
     // The consensus is not formed yet!
+    /* XXXX Extract this code into a common function */
     if (params) {
       if (strcmpstart(params, "bwweightscale=") == 0)
         bw_weight_param = params;

src/or/dirvote.h

@@ -20,7 +20,7 @@
 #define MIN_VOTE_INTERVAL 300
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 16
+#define MAX_SUPPORTED_CONSENSUS_METHOD 17
 
 /** Lowest consensus method that contains a 'directory-footer' marker */
 #define MIN_METHOD_FOR_FOOTER 9
@@ -52,6 +52,14 @@
  * line */
 #define MIN_METHOD_FOR_NTOR_KEY 16
 
+/** Lowest consensus method that ensures that authorities output an
+ * Unmeasured=1 flag for unmeasured bandwidths */
+#define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17
+
+/** Default bandwidth to clip unmeasured bandwidths to using method >=
+ * MIN_METHOD_TO_CLIP_UNMEASURED_BW */
+#define DEFAULT_MAX_UNMEASURED_BW 20
+
 void dirvote_free_all(void);
 
 /* vote manipulation */

src/or/networkstatus.c

@@ -937,6 +937,17 @@ compare_digest_to_routerstatus_entry(const void *_key, const void **_member)
   return tor_memcmp(key, rs->identity_digest, DIGEST_LEN);
 }
 
+/** Helper for bsearching a list of routerstatus_t pointers: compare a
+ * digest in the key to the identity digest of a routerstatus_t. */
+int
+compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const vote_routerstatus_t *vrs = *_member;
+  return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN);
+}
+
 /** As networkstatus_v2_find_entry, but do not return a const pointer */
 routerstatus_t *
 networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, const char *digest)
@@ -2117,7 +2128,7 @@ char *
 networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 {
   char buf[RS_ENTRY_LEN+1];
-  routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT);
+  routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT, NULL);
   return tor_strdup(buf);
 }
 

src/or/networkstatus.h

@@ -38,6 +38,8 @@ int router_set_networkstatus_v2(const char *s, time_t arrived_at,
 void networkstatus_v2_list_clean(time_t now);
 int compare_digest_to_routerstatus_entry(const void *_key,
                                          const void **_member);
+int compare_digest_to_vote_routerstatus_entry(const void *_key,
+                                              const void **_member);
 const routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns,
                                          const char *digest);
 const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,

src/or/or.h

@@ -2091,9 +2091,8 @@ typedef struct routerstatus_t {
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
-  unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */
-
-  uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */
+  unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
+                                    * the Unmeasured flag set. */
 
   uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in
                        * the vote/consensus, in kilobytes/sec. */
@@ -2337,6 +2336,8 @@ typedef struct vote_routerstatus_t {
                    * networkstatus_t.known_flags. */
   char *version; /**< The version that the authority says this router is
                   * running. */
+  unsigned int has_measured_bw:1; /**< The vote had a measured bw */
+  uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */
   /** The hash or hashes that the authority claims this microdesc has. */
   vote_microdesc_hash_t *microdesc;
 } vote_routerstatus_t;
@@ -2402,6 +2403,9 @@ typedef enum {
 typedef struct networkstatus_t {
   ENUM_BF(networkstatus_type_t) type : 8; /**< Vote, consensus, or opinion? */
   ENUM_BF(consensus_flavor_t) flavor : 8; /**< If a consensus, what kind? */
+  unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains
+                                     * measured= bandwidth values. */
+
   time_t published; /**< Vote only: Time when vote was written. */
   time_t valid_after; /**< Time after which this vote or consensus applies. */
   time_t fresh_until; /**< Time before which this is the most recent vote or

src/or/routerparse.c

@@ -1974,9 +1974,9 @@ routerstatus_parse_entry_from_string(memarea_t *area,
           goto err;
         }
         rs->has_bandwidth = 1;
-      } else if (!strcmpstart(tok->args[i], "Measured=")) {
+      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
         int ok;
-        rs->measured_bw =
+        vote_rs->measured_bw =
             (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
                                       10, 0, UINT32_MAX, &ok, NULL);
         if (!ok) {
@@ -1984,7 +1984,10 @@ routerstatus_parse_entry_from_string(memarea_t *area,
                    escaped(tok->args[i]));
           goto err;
         }
-        rs->has_measured_bw = 1;
+        vote_rs->has_measured_bw = 1;
+        vote->has_measured_bws = 1;
+      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
+        rs->bw_is_unmeasured = 1;
       }
     }
   }
@@ -2062,6 +2065,14 @@ compare_routerstatus_entries(const void **_a, const void **_b)
   return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN);
 }
 
+int
+compare_vote_routerstatus_entries(const void **_a, const void **_b)
+{
+  const vote_routerstatus_t *a = *_a, *b = *_b;
+  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
+                     DIGEST_LEN);
+}
+
 /** Helper: used in call to _smartlist_uniq to clear out duplicate entries. */
 static void
 free_duplicate_routerstatus_entry_(void *e)

src/or/routerparse.h

@@ -52,6 +52,7 @@ void assert_addr_policy_ok(smartlist_t *t);
 void dump_distinct_digest_count(int severity);
 
 int compare_routerstatus_entries(const void **_a, const void **_b);
+int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s);
 int networkstatus_verify_bw_weights(networkstatus_t *ns);
 networkstatus_t *networkstatus_parse_vote_from_string(const char *s,