|
@@ -53,6 +53,9 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
|
|
|
# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
|
|
|
PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
|
|
|
|
|
|
+# Output fallback name, flags, and ContactInfo in a C comment?
|
|
|
+OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
|
|
|
+
|
|
|
# Output matching ContactInfo in fallbacks list or the blacklist?
|
|
|
# Useful if you're trying to contact operators
|
|
|
CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
|
|
@@ -933,8 +936,9 @@ class Candidate(object):
|
|
|
def is_running(self):
|
|
|
return 'Running' in self._data['flags']
|
|
|
|
|
|
+ # report how long it takes to download a consensus from dirip:dirport
|
|
|
@staticmethod
|
|
|
- def fallback_consensus_dl_speed(dirip, dirport, nickname, max_time):
|
|
|
+ def fallback_consensus_download_speed(dirip, dirport, nickname, max_time):
|
|
|
download_failed = False
|
|
|
downloader = DescriptorDownloader()
|
|
|
start = datetime.datetime.utcnow()
|
|
@@ -970,47 +974,60 @@ class Candidate(object):
|
|
|
dirip, dirport, max_time)
|
|
|
return download_failed
|
|
|
|
|
|
- def fallback_consensus_dl_check(self):
|
|
|
+ # does this fallback download the consensus fast enough?
|
|
|
+ def check_fallback_download_consensus(self):
|
|
|
# include the relay if we're not doing a check, or we can't check (IPv6)
|
|
|
ipv4_failed = False
|
|
|
ipv6_failed = False
|
|
|
if PERFORM_IPV4_DIRPORT_CHECKS:
|
|
|
- ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip,
|
|
|
+ ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
|
|
|
self.dirport,
|
|
|
self._data['nickname'],
|
|
|
CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
if self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
# Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
|
|
|
- ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr,
|
|
|
- self.dirport,
|
|
|
- self._data['nickname'],
|
|
|
- CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
- # Now retry the relay if it took too long the first time
|
|
|
- if (PERFORM_IPV4_DIRPORT_CHECKS and ipv4_failed
|
|
|
- and CONSENSUS_DOWNLOAD_RETRY):
|
|
|
- ipv4_failed = Candidate.fallback_consensus_dl_speed(self.dirip,
|
|
|
- self.dirport,
|
|
|
- self._data['nickname'],
|
|
|
- CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
- if (self.ipv6addr is not None and PERFORM_IPV6_DIRPORT_CHECKS
|
|
|
- and ipv6_failed and CONSENSUS_DOWNLOAD_RETRY):
|
|
|
- ipv6_failed = Candidate.fallback_consensus_dl_speed(self.ipv6addr,
|
|
|
+ ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
|
|
|
self.dirport,
|
|
|
self._data['nickname'],
|
|
|
CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
return ((not ipv4_failed) and (not ipv6_failed))
|
|
|
|
|
|
- def fallbackdir_line(self, dl_speed_ok, fallbacks, prefilter_fallbacks):
|
|
|
+ # if this fallback has not passed a download check, try it again,
|
|
|
+ # and record the result, available in get_fallback_download_consensus
|
|
|
+ def try_fallback_download_consensus(self):
|
|
|
+ if not self.get_fallback_download_consensus():
|
|
|
+ self._data['download_check'] = self.check_fallback_download_consensus()
|
|
|
+
|
|
|
+ # did this fallback pass the download check?
|
|
|
+ def get_fallback_download_consensus(self):
|
|
|
+ # if we're not performing checks, return True
|
|
|
+ if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
+ return True
|
|
|
+ # if we are performing checks, but haven't done one, return False
|
|
|
+ if not self._data.has_key('download_check'):
|
|
|
+ return False
|
|
|
+ return self._data['download_check']
|
|
|
+
|
|
|
+ # output an optional header comment and info for this fallback
|
|
|
+ # try_fallback_download_consensus before calling this
|
|
|
+ def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
|
|
|
+ s = ''
|
|
|
+ if OUTPUT_COMMENTS:
|
|
|
+ s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
|
|
|
+ # if the download speed is ok, output a C string
|
|
|
+ # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
|
|
|
+ if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
|
|
|
+ s += self.fallbackdir_info(self.get_fallback_download_consensus())
|
|
|
+ return s
|
|
|
+
|
|
|
+ # output a header comment for this fallback
|
|
|
+ def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
|
|
|
# /*
|
|
|
# nickname
|
|
|
# flags
|
|
|
# [contact]
|
|
|
# [identical contact counts]
|
|
|
# */
|
|
|
- # "address:dirport orport=port id=fingerprint"
|
|
|
- # "[ipv6=addr:orport]"
|
|
|
- # "weight=FALLBACK_OUTPUT_WEIGHT",
|
|
|
- #
|
|
|
# Multiline C comment
|
|
|
s = '/*'
|
|
|
s += '\n'
|
|
@@ -1040,9 +1057,26 @@ class Candidate(object):
|
|
|
s += '\n'
|
|
|
s += '*/'
|
|
|
s += '\n'
|
|
|
+
|
|
|
+ # output the fallback info C string for this fallback
|
|
|
+ # this is the text that would go after FallbackDir in a torrc
|
|
|
+ # if this relay failed the download test and we OUTPUT_COMMENTS,
|
|
|
+ # comment-out the returned string
|
|
|
+ def fallbackdir_info(self, dl_speed_ok):
|
|
|
+ # "address:dirport orport=port id=fingerprint"
|
|
|
+ # "[ipv6=addr:orport]"
|
|
|
+ # "weight=FALLBACK_OUTPUT_WEIGHT",
|
|
|
+ #
|
|
|
+ # Do we want a C string, or a commented-out string?
|
|
|
+ c_string = dl_speed_ok
|
|
|
+ comment_string = not dl_speed_ok and OUTPUT_COMMENTS
|
|
|
+ # If we don't want either kind of string, bail
|
|
|
+ if not c_string and not comment_string:
|
|
|
+ return ''
|
|
|
+ s = ''
|
|
|
# Comment out the fallback directory entry if it's too slow
|
|
|
# See the debug output for which address and port is failing
|
|
|
- if not dl_speed_ok:
|
|
|
+ if comment_string:
|
|
|
s += '/* Consensus download failed or was too slow:\n'
|
|
|
# Multi-Line C string with trailing comma (part of a string list)
|
|
|
# This makes it easier to diff the file, and remove IPv6 lines using grep
|
|
@@ -1057,7 +1091,7 @@ class Candidate(object):
|
|
|
cleanse_c_string(self.ipv6addr), cleanse_c_string(self.ipv6orport))
|
|
|
s += '\n'
|
|
|
s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT)
|
|
|
- if not dl_speed_ok:
|
|
|
+ if comment_string:
|
|
|
s += '\n'
|
|
|
s += '*/'
|
|
|
return s
|
|
@@ -1129,13 +1163,19 @@ class CandidateList(dict):
|
|
|
# lowest to highest
|
|
|
# used to find the median cw_to_bw_factor()
|
|
|
def sort_fallbacks_by_cw_to_bw_factor(self):
|
|
|
- self.fallbacks.sort(key=lambda x: self[x].cw_to_bw_factor())
|
|
|
+ self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor(), self.fallbacks)
|
|
|
|
|
|
# sort fallbacks by their measured bandwidth, highest to lowest
|
|
|
# calculate_measured_bandwidth before calling this
|
|
|
+ # this is useful for reviewing candidates in priority order
|
|
|
def sort_fallbacks_by_measured_bandwidth(self):
|
|
|
- self.fallbacks.sort(key=lambda x: self[x].self._data['measured_bandwidth'],
|
|
|
- reverse=True)
|
|
|
+ self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
|
|
|
+ self.fallbacks, reverse=True)
|
|
|
+
|
|
|
+ # sort fallbacks by their fingerprint, lowest to highest
|
|
|
+ # this is useful for stable diffs of fallback lists
|
|
|
+ def sort_fallbacks_by_fingerprint(self):
|
|
|
+ self.fallbacks.sort(key=lambda f: self[f]._fpr, self.fallbacks)
|
|
|
|
|
|
@staticmethod
|
|
|
def load_relaylist(file_name):
|
|
@@ -1301,6 +1341,39 @@ class CandidateList(dict):
|
|
|
else:
|
|
|
return None
|
|
|
|
|
|
+ # try a download check on each fallback candidate in order
|
|
|
+ # stop after max_count successful downloads
|
|
|
+ # but don't remove any candidates from the array
|
|
|
+ def try_download_consensus_checks(self, max_count):
|
|
|
+ dl_ok_count = 0
|
|
|
+ for f in self.fallbacks:
|
|
|
+ f.try_fallback_download_consensus()
|
|
|
+ if f.get_fallback_download_consensus():
|
|
|
+ # this fallback downloaded a consensus ok
|
|
|
+ dl_ok_count += 1
|
|
|
+ if dl_ok_count >= max_count:
|
|
|
+ # we have enough fallbacks
|
|
|
+ return
|
|
|
+
|
|
|
+ # put max_count successful candidates in the fallbacks array:
|
|
|
+ # - perform download checks on each fallback candidate
|
|
|
+ # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
|
|
|
+ # - eliminate failed candidates
|
|
|
+ # - if there are more than max_count candidates, eliminate lowest bandwidth
|
|
|
+ # - if there are fewer than max_count candidates, leave only successful
|
|
|
+ def perform_download_consensus_checks(self, max_count):
|
|
|
+ self.sort_fallbacks_by_measured_bandwidth()
|
|
|
+ self.try_download_consensus_checks(max_count)
|
|
|
+ if CONSENSUS_DOWNLOAD_RETRY:
|
|
|
+ # try unsuccessful candidates again
|
|
|
+ # we could end up with more than max_count successful candidates here
|
|
|
+ self.try_download_consensus_checks(max_count)
|
|
|
+ # now we have at least max_count successful candidates,
|
|
|
+ # or we've tried them all
|
|
|
+ self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
|
|
|
+ self.fallbacks)
|
|
|
+ self.fallbacks = self.fallbacks[:max_count]
|
|
|
+
|
|
|
def summarise_fallbacks(self, eligible_count, guard_count, target_count,
|
|
|
max_count):
|
|
|
# Report:
|
|
@@ -1393,9 +1466,6 @@ def list_fallbacks():
|
|
|
# then remove low-bandwidth relays
|
|
|
candidates.calculate_measured_bandwidth()
|
|
|
candidates.remove_low_bandwidth_relays()
|
|
|
- # make sure the list is sorted by bandwidth when we output it
|
|
|
- # so that we include the active fallbacks with the greatest bandwidth
|
|
|
- candidates.sort_fallbacks_by_measured_bandwidth()
|
|
|
|
|
|
# print the raw fallback list
|
|
|
#for x in candidates.fallbacks:
|
|
@@ -1412,17 +1482,18 @@ def list_fallbacks():
|
|
|
for s in fetch_source_list():
|
|
|
print describe_fetch_source(s)
|
|
|
|
|
|
- active_count = 0
|
|
|
+ # check if each candidate can serve a consensus
|
|
|
+ candidates.perform_download_consensus_checks(max_count)
|
|
|
+
|
|
|
+ # if we're outputting the final fallback list, sort by fingerprint
|
|
|
+ # this makes diffs much more stable
|
|
|
+ # otherwise, leave sorted by bandwidth, which allows operators to be
|
|
|
+ # contacted in priority order
|
|
|
+ if not OUTPUT_CANDIDATES:
|
|
|
+ candidates.sort_fallbacks_by_fingerprint()
|
|
|
+
|
|
|
for x in candidates.fallbacks:
|
|
|
- dl_speed_ok = x.fallback_consensus_dl_check()
|
|
|
- print x.fallbackdir_line(dl_speed_ok, candidates.fallbacks,
|
|
|
- prefilter_fallbacks)
|
|
|
- if dl_speed_ok:
|
|
|
- # this fallback is included in the list
|
|
|
- active_count += 1
|
|
|
- if active_count >= max_count:
|
|
|
- # we have enough fallbacks
|
|
|
- break
|
|
|
+ print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
list_fallbacks()
|