|
@@ -65,7 +65,7 @@ static int compute_weighted_bandwidths(const smartlist_t *sl,
|
|
|
bandwidth_weight_rule_t rule,
|
|
|
u64_dbl_t **bandwidths_out);
|
|
|
static const routerstatus_t *router_pick_directory_server_impl(
|
|
|
- dirinfo_type_t auth, int flags);
|
|
|
+ dirinfo_type_t auth, int flags, int *n_busy_out);
|
|
|
static const routerstatus_t *router_pick_trusteddirserver_impl(
|
|
|
const smartlist_t *sourcelist, dirinfo_type_t auth,
|
|
|
int flags, int *n_busy_out);
|
|
@@ -1288,22 +1288,32 @@ router_get_fallback_dir_servers(void)
|
|
|
const routerstatus_t *
|
|
|
router_pick_directory_server(dirinfo_type_t type, int flags)
|
|
|
{
|
|
|
+ int busy = 0;
|
|
|
const routerstatus_t *choice;
|
|
|
|
|
|
if (!routerlist)
|
|
|
return NULL;
|
|
|
|
|
|
- choice = router_pick_directory_server_impl(type, flags);
|
|
|
+ choice = router_pick_directory_server_impl(type, flags, &busy);
|
|
|
if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS))
|
|
|
return choice;
|
|
|
|
|
|
+ if (busy) {
|
|
|
+ /* If the reason that we got no server is that servers are "busy",
|
|
|
+ * we must be excluding good servers because we already have serverdesc
|
|
|
+ * fetches with them. Do not mark down servers up because of this. */
|
|
|
+ tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
|
|
|
+ PDS_NO_EXISTING_MICRODESC_FETCH)));
|
|
|
+ return NULL;
|
|
|
+ }
|
|
|
+
|
|
|
log_info(LD_DIR,
|
|
|
"No reachable router entries for dirservers. "
|
|
|
"Trying them all again.");
|
|
|
/* mark all authdirservers as up again */
|
|
|
mark_all_dirservers_up(fallback_dir_servers);
|
|
|
/* try again */
|
|
|
- choice = router_pick_directory_server_impl(type, flags);
|
|
|
+ choice = router_pick_directory_server_impl(type, flags, NULL);
|
|
|
return choice;
|
|
|
}
|
|
|
|
|
@@ -1413,11 +1423,15 @@ router_pick_dirserver_generic(smartlist_t *sourcelist,
|
|
|
#define DIR_503_TIMEOUT (60*60)
|
|
|
|
|
|
/** Pick a random running valid directory server/mirror from our
|
|
|
- * routerlist. Arguments are as for router_pick_directory_server(), except
|
|
|
- * that RETRY_IF_NO_SERVERS is ignored.
|
|
|
+ * routerlist. Arguments are as for router_pick_directory_server(), except:
|
|
|
+ *
|
|
|
+ * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of
|
|
|
+ * directories that we excluded for no other reason than
|
|
|
+ * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH.
|
|
|
*/
|
|
|
static const routerstatus_t *
|
|
|
-router_pick_directory_server_impl(dirinfo_type_t type, int flags)
|
|
|
+router_pick_directory_server_impl(dirinfo_type_t type, int flags,
|
|
|
+ int *n_busy_out)
|
|
|
{
|
|
|
const or_options_t *options = get_options();
|
|
|
const node_t *result;
|
|
@@ -1426,10 +1440,12 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags)
|
|
|
smartlist_t *overloaded_direct, *overloaded_tunnel;
|
|
|
time_t now = time(NULL);
|
|
|
const networkstatus_t *consensus = networkstatus_get_latest_consensus();
|
|
|
- int requireother = ! (flags & PDS_ALLOW_SELF);
|
|
|
- int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
|
|
|
- int for_guard = (flags & PDS_FOR_GUARD);
|
|
|
- int try_excluding = 1, n_excluded = 0;
|
|
|
+ const int requireother = ! (flags & PDS_ALLOW_SELF);
|
|
|
+ const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
|
|
|
+ const int no_serverdesc_fetching = (flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
|
|
|
+ const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
|
|
|
+ const int for_guard = (flags & PDS_FOR_GUARD);
|
|
|
+ int try_excluding = 1, n_excluded = 0, n_busy = 0;
|
|
|
|
|
|
if (!consensus)
|
|
|
return NULL;
|
|
@@ -1476,7 +1492,24 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags)
|
|
|
}
|
|
|
|
|
|
/* XXXX IP6 proposal 118 */
|
|
|
- tor_addr_from_ipv4h(&addr, node->rs->addr);
|
|
|
+ tor_addr_from_ipv4h(&addr, status->addr);
|
|
|
+
|
|
|
+ if (no_serverdesc_fetching && (
|
|
|
+ connection_get_by_type_addr_port_purpose(
|
|
|
+ CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)
|
|
|
+ || connection_get_by_type_addr_port_purpose(
|
|
|
+ CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_EXTRAINFO)
|
|
|
+ )) {
|
|
|
+ ++n_busy;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (no_microdesc_fetching && connection_get_by_type_addr_port_purpose(
|
|
|
+ CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_MICRODESC)
|
|
|
+ ) {
|
|
|
+ ++n_busy;
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
|
|
|
|
|
@@ -1521,9 +1554,13 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags)
|
|
|
* not set, try again without excluding nodes. */
|
|
|
try_excluding = 0;
|
|
|
n_excluded = 0;
|
|
|
+ n_busy = 0;
|
|
|
goto retry_without_exclude;
|
|
|
}
|
|
|
|
|
|
+ if (n_busy_out)
|
|
|
+ *n_busy_out = n_busy;
|
|
|
+
|
|
|
return result ? result->rs : NULL;
|
|
|
}
|
|
|
|
|
@@ -4188,51 +4225,56 @@ list_pending_fpsk_downloads(fp_pair_map_t *result)
|
|
|
* range.) If <b>source</b> is given, download from <b>source</b>;
|
|
|
* otherwise, download from an appropriate random directory server.
|
|
|
*/
|
|
|
-static void
|
|
|
-initiate_descriptor_downloads(const routerstatus_t *source,
|
|
|
- int purpose,
|
|
|
- smartlist_t *digests,
|
|
|
- int lo, int hi, int pds_flags)
|
|
|
+MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
|
|
|
+ (const routerstatus_t *source, int purpose, smartlist_t *digests,
|
|
|
+ int lo, int hi, int pds_flags))
|
|
|
{
|
|
|
- int i, n = hi-lo;
|
|
|
char *resource, *cp;
|
|
|
- size_t r_len;
|
|
|
-
|
|
|
- int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN;
|
|
|
- char sep = '+';
|
|
|
- int b64_256 = 0;
|
|
|
+ int digest_len, enc_digest_len;
|
|
|
+ const char *sep;
|
|
|
+ int b64_256;
|
|
|
+ smartlist_t *tmp;
|
|
|
|
|
|
if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
|
|
|
/* Microdescriptors are downloaded by "-"-separated base64-encoded
|
|
|
* 256-bit digests. */
|
|
|
digest_len = DIGEST256_LEN;
|
|
|
- enc_digest_len = BASE64_DIGEST256_LEN;
|
|
|
- sep = '-';
|
|
|
+ enc_digest_len = BASE64_DIGEST256_LEN + 1;
|
|
|
+ sep = "-";
|
|
|
b64_256 = 1;
|
|
|
+ } else {
|
|
|
+ digest_len = DIGEST_LEN;
|
|
|
+ enc_digest_len = HEX_DIGEST_LEN + 1;
|
|
|
+ sep = "+";
|
|
|
+ b64_256 = 0;
|
|
|
}
|
|
|
|
|
|
- if (n <= 0)
|
|
|
- return;
|
|
|
if (lo < 0)
|
|
|
lo = 0;
|
|
|
if (hi > smartlist_len(digests))
|
|
|
hi = smartlist_len(digests);
|
|
|
|
|
|
- r_len = 8 + (enc_digest_len+1)*n;
|
|
|
- cp = resource = tor_malloc(r_len);
|
|
|
- memcpy(cp, "d/", 2);
|
|
|
- cp += 2;
|
|
|
- for (i = lo; i < hi; ++i) {
|
|
|
+ if (hi-lo <= 0)
|
|
|
+ return;
|
|
|
+
|
|
|
+ tmp = smartlist_new();
|
|
|
+
|
|
|
+ for (; lo < hi; ++lo) {
|
|
|
+ cp = tor_malloc(enc_digest_len);
|
|
|
if (b64_256) {
|
|
|
- digest256_to_base64(cp, smartlist_get(digests, i));
|
|
|
+ digest256_to_base64(cp, smartlist_get(digests, lo));
|
|
|
} else {
|
|
|
- base16_encode(cp, r_len-(cp-resource),
|
|
|
- smartlist_get(digests,i), digest_len);
|
|
|
+ base16_encode(cp, enc_digest_len, smartlist_get(digests, lo), digest_len);
|
|
|
}
|
|
|
- cp += enc_digest_len;
|
|
|
- *cp++ = sep;
|
|
|
+ smartlist_add(tmp, cp);
|
|
|
}
|
|
|
- memcpy(cp-1, ".z", 3);
|
|
|
+
|
|
|
+ cp = smartlist_join_strings(tmp, sep, 0, NULL);
|
|
|
+ tor_asprintf(&resource, "d/%s.z", cp);
|
|
|
+
|
|
|
+ SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1));
|
|
|
+ smartlist_free(tmp);
|
|
|
+ tor_free(cp);
|
|
|
|
|
|
if (source) {
|
|
|
/* We know which authority we want. */
|
|
@@ -4247,14 +4289,28 @@ initiate_descriptor_downloads(const routerstatus_t *source,
|
|
|
tor_free(resource);
|
|
|
}
|
|
|
|
|
|
-/** Max amount of hashes to download per request.
|
|
|
- * Since squid does not like URLs >= 4096 bytes we limit it to 96.
|
|
|
- * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058
|
|
|
- * 4058/41 (40 for the hash and 1 for the + that separates them) => 98
|
|
|
- * So use 96 because it's a nice number.
|
|
|
+/** Return the max number of hashes to put in a URL for a given request.
|
|
|
*/
|
|
|
-#define MAX_DL_PER_REQUEST 96
|
|
|
-#define MAX_MICRODESC_DL_PER_REQUEST 92
|
|
|
+static int
|
|
|
+max_dl_per_request(const or_options_t *options, int purpose)
|
|
|
+{
|
|
|
+ /* Since squid does not like URLs >= 4096 bytes we limit it to 96.
|
|
|
+ * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058
|
|
|
+ * 4058/41 (40 for the hash and 1 for the + that separates them) => 98
|
|
|
+ * So use 96 because it's a nice number.
|
|
|
+ */
|
|
|
+ int max = 96;
|
|
|
+ if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
|
|
|
+ max = 92;
|
|
|
+ }
|
|
|
+ /* If we're going to tunnel our connections, we can ask for a lot more
|
|
|
+ * in a request. */
|
|
|
+ if (!directory_fetches_from_authorities(options)) {
|
|
|
+ max = 500;
|
|
|
+ }
|
|
|
+ return max;
|
|
|
+}
|
|
|
+
|
|
|
/** Don't split our requests so finely that we are requesting fewer than
|
|
|
* this number per server. */
|
|
|
#define MIN_DL_PER_REQUEST 4
|
|
@@ -4276,92 +4332,90 @@ launch_descriptor_downloads(int purpose,
|
|
|
smartlist_t *downloadable,
|
|
|
const routerstatus_t *source, time_t now)
|
|
|
{
|
|
|
- int should_delay = 0, n_downloadable;
|
|
|
const or_options_t *options = get_options();
|
|
|
const char *descname;
|
|
|
+ const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC);
|
|
|
+ int n_downloadable = smartlist_len(downloadable);
|
|
|
|
|
|
- tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
|
|
|
- purpose == DIR_PURPOSE_FETCH_MICRODESC);
|
|
|
+ int i, n_per_request, max_dl_per_req;
|
|
|
+ const char *req_plural = "", *rtr_plural = "";
|
|
|
+ int pds_flags = PDS_RETRY_IF_NO_SERVERS;
|
|
|
|
|
|
- descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ?
|
|
|
- "routerdesc" : "microdesc";
|
|
|
+ tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC);
|
|
|
+ descname = fetch_microdesc ? "microdesc" : "routerdesc";
|
|
|
+
|
|
|
+ if (!n_downloadable)
|
|
|
+ return;
|
|
|
|
|
|
- n_downloadable = smartlist_len(downloadable);
|
|
|
if (!directory_fetches_dir_info_early(options)) {
|
|
|
if (n_downloadable >= MAX_DL_TO_DELAY) {
|
|
|
log_debug(LD_DIR,
|
|
|
"There are enough downloadable %ss to launch requests.",
|
|
|
descname);
|
|
|
- should_delay = 0;
|
|
|
} else {
|
|
|
- should_delay = (last_descriptor_download_attempted +
|
|
|
- options->TestingClientMaxIntervalWithoutRequest) > now;
|
|
|
- if (!should_delay && n_downloadable) {
|
|
|
- if (last_descriptor_download_attempted) {
|
|
|
- log_info(LD_DIR,
|
|
|
- "There are not many downloadable %ss, but we've "
|
|
|
- "been waiting long enough (%d seconds). Downloading.",
|
|
|
- descname,
|
|
|
- (int)(now-last_descriptor_download_attempted));
|
|
|
- } else {
|
|
|
- log_info(LD_DIR,
|
|
|
- "There are not many downloadable %ss, but we haven't "
|
|
|
- "tried downloading descriptors recently. Downloading.",
|
|
|
- descname);
|
|
|
- }
|
|
|
+
|
|
|
+ /* should delay */
|
|
|
+ if ((last_descriptor_download_attempted +
|
|
|
+ options->TestingClientMaxIntervalWithoutRequest) > now)
|
|
|
+ return;
|
|
|
+
|
|
|
+ if (last_descriptor_download_attempted) {
|
|
|
+ log_info(LD_DIR,
|
|
|
+ "There are not many downloadable %ss, but we've "
|
|
|
+ "been waiting long enough (%d seconds). Downloading.",
|
|
|
+ descname,
|
|
|
+ (int)(now-last_descriptor_download_attempted));
|
|
|
+ } else {
|
|
|
+ log_info(LD_DIR,
|
|
|
+ "There are not many downloadable %ss, but we haven't "
|
|
|
+ "tried downloading descriptors recently. Downloading.",
|
|
|
+ descname);
|
|
|
}
|
|
|
+
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- if (! should_delay && n_downloadable) {
|
|
|
- int i, n_per_request;
|
|
|
- const char *req_plural = "", *rtr_plural = "";
|
|
|
- int pds_flags = PDS_RETRY_IF_NO_SERVERS;
|
|
|
- if (! authdir_mode_any_nonhidserv(options)) {
|
|
|
- /* If we wind up going to the authorities, we want to only open one
|
|
|
- * connection to each authority at a time, so that we don't overload
|
|
|
- * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
|
|
|
- * regardless of whether we're a cache or not; it gets ignored if we're
|
|
|
- * not calling router_pick_trusteddirserver.
|
|
|
- *
|
|
|
- * Setting this flag can make initiate_descriptor_downloads() ignore
|
|
|
- * requests. We need to make sure that we do in fact call
|
|
|
- * update_router_descriptor_downloads() later on, once the connections
|
|
|
- * have succeeded or failed.
|
|
|
- */
|
|
|
- pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ?
|
|
|
- PDS_NO_EXISTING_MICRODESC_FETCH :
|
|
|
- PDS_NO_EXISTING_SERVERDESC_FETCH;
|
|
|
- }
|
|
|
+ if (!authdir_mode_any_nonhidserv(options)) {
|
|
|
+ /* If we wind up going to the authorities, we want to only open one
|
|
|
+ * connection to each authority at a time, so that we don't overload
|
|
|
+ * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
|
|
|
+ * regardless of whether we're a cache or not.
|
|
|
+ *
|
|
|
+ * Setting this flag can make initiate_descriptor_downloads() ignore
|
|
|
+ * requests. We need to make sure that we do in fact call
|
|
|
+ * update_router_descriptor_downloads() later on, once the connections
|
|
|
+ * have succeeded or failed.
|
|
|
+ */
|
|
|
+ pds_flags |= fetch_microdesc ?
|
|
|
+ PDS_NO_EXISTING_MICRODESC_FETCH :
|
|
|
+ PDS_NO_EXISTING_SERVERDESC_FETCH;
|
|
|
+ }
|
|
|
|
|
|
- n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
|
|
|
- if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
|
|
|
- if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST)
|
|
|
- n_per_request = MAX_MICRODESC_DL_PER_REQUEST;
|
|
|
- } else {
|
|
|
- if (n_per_request > MAX_DL_PER_REQUEST)
|
|
|
- n_per_request = MAX_DL_PER_REQUEST;
|
|
|
- }
|
|
|
- if (n_per_request < MIN_DL_PER_REQUEST)
|
|
|
- n_per_request = MIN_DL_PER_REQUEST;
|
|
|
-
|
|
|
- if (n_downloadable > n_per_request)
|
|
|
- req_plural = rtr_plural = "s";
|
|
|
- else if (n_downloadable > 1)
|
|
|
- rtr_plural = "s";
|
|
|
-
|
|
|
- log_info(LD_DIR,
|
|
|
- "Launching %d request%s for %d %s%s, %d at a time",
|
|
|
- CEIL_DIV(n_downloadable, n_per_request), req_plural,
|
|
|
- n_downloadable, descname, rtr_plural, n_per_request);
|
|
|
- smartlist_sort_digests(downloadable);
|
|
|
- for (i=0; i < n_downloadable; i += n_per_request) {
|
|
|
- initiate_descriptor_downloads(source, purpose,
|
|
|
- downloadable, i, i+n_per_request,
|
|
|
- pds_flags);
|
|
|
- }
|
|
|
- last_descriptor_download_attempted = now;
|
|
|
+ n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS);
|
|
|
+ max_dl_per_req = max_dl_per_request(options, purpose);
|
|
|
+
|
|
|
+ if (n_per_request > max_dl_per_req)
|
|
|
+ n_per_request = max_dl_per_req;
|
|
|
+
|
|
|
+ if (n_per_request < MIN_DL_PER_REQUEST)
|
|
|
+ n_per_request = MIN_DL_PER_REQUEST;
|
|
|
+
|
|
|
+ if (n_downloadable > n_per_request)
|
|
|
+ req_plural = rtr_plural = "s";
|
|
|
+ else if (n_downloadable > 1)
|
|
|
+ rtr_plural = "s";
|
|
|
+
|
|
|
+ log_info(LD_DIR,
|
|
|
+ "Launching %d request%s for %d %s%s, %d at a time",
|
|
|
+ CEIL_DIV(n_downloadable, n_per_request), req_plural,
|
|
|
+ n_downloadable, descname, rtr_plural, n_per_request);
|
|
|
+ smartlist_sort_digests(downloadable);
|
|
|
+ for (i=0; i < n_downloadable; i += n_per_request) {
|
|
|
+ initiate_descriptor_downloads(source, purpose,
|
|
|
+ downloadable, i, i+n_per_request,
|
|
|
+ pds_flags);
|
|
|
}
|
|
|
+ last_descriptor_download_attempted = now;
|
|
|
}
|
|
|
|
|
|
/** For any descriptor that we want that's currently listed in
|
|
@@ -4541,7 +4595,7 @@ update_extrainfo_downloads(time_t now)
|
|
|
routerlist_t *rl;
|
|
|
smartlist_t *wanted;
|
|
|
digestmap_t *pending;
|
|
|
- int old_routers, i;
|
|
|
+ int old_routers, i, max_dl_per_req;
|
|
|
int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0;
|
|
|
if (! options->DownloadExtraInfo)
|
|
|
return;
|
|
@@ -4597,9 +4651,11 @@ update_extrainfo_downloads(time_t now)
|
|
|
n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted));
|
|
|
|
|
|
smartlist_shuffle(wanted);
|
|
|
- for (i = 0; i < smartlist_len(wanted); i += MAX_DL_PER_REQUEST) {
|
|
|
+
|
|
|
+ max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO);
|
|
|
+ for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) {
|
|
|
initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO,
|
|
|
- wanted, i, i + MAX_DL_PER_REQUEST,
|
|
|
+ wanted, i, i+max_dl_per_req,
|
|
|
PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH);
|
|
|
}
|
|
|
|