|
|
@@ -1,2383 +0,0 @@
|
|
|
-#!/usr/bin/env python
|
|
|
-
|
|
|
-# Usage:
|
|
|
-#
|
|
|
-# Regenerate the list:
|
|
|
-# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log
|
|
|
-#
|
|
|
-# Check the existing list:
|
|
|
-# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
|
|
|
-# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc
|
|
|
-#
|
|
|
-# This script should be run from a stable, reliable network connection,
|
|
|
-# with no other network activity (and not over tor).
|
|
|
-# If this is not possible, please disable:
|
|
|
-# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
|
|
|
-#
|
|
|
-# Needs dateutil, stem, and potentially other python packages.
|
|
|
-# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
|
|
|
-# for netblock analysis.
|
|
|
-#
|
|
|
-# After running this script, read the logs to make sure the fallbacks aren't
|
|
|
-# dominated by a single netblock or port.
|
|
|
-
|
|
|
-# Script by weasel, April 2015
|
|
|
-# Portions by gsathya & karsten, 2013
|
|
|
-# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py
|
|
|
-# Modifications by teor, 2015
|
|
|
-
|
|
|
-import StringIO
|
|
|
-import string
|
|
|
-import re
|
|
|
-import datetime
|
|
|
-import gzip
|
|
|
-import os.path
|
|
|
-import json
|
|
|
-import math
|
|
|
-import sys
|
|
|
-import urllib
|
|
|
-import urllib2
|
|
|
-import hashlib
|
|
|
-import dateutil.parser
|
|
|
-import copy
|
|
|
-import re
|
|
|
-
|
|
|
-from stem.descriptor import DocumentHandler
|
|
|
-from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
|
|
|
-
|
|
|
-import logging
|
|
|
-logging.root.name = ''
|
|
|
-
|
|
|
-HAVE_IPADDRESS = False
|
|
|
-try:
|
|
|
- # python 3 builtin, or install package py2-ipaddress
|
|
|
- # there are several ipaddress implementations for python 2
|
|
|
- # with slightly different semantics with str typed text
|
|
|
- # fortunately, all our IP addresses are in unicode
|
|
|
- import ipaddress
|
|
|
- HAVE_IPADDRESS = True
|
|
|
-except ImportError:
|
|
|
- # if this happens, we avoid doing netblock analysis
|
|
|
- logging.warning('Unable to import ipaddress, please install py2-ipaddress.' +
|
|
|
- ' A fallback list will be created, but optional netblock' +
|
|
|
- ' analysis will not be performed.')
|
|
|
-
|
|
|
-## Top-Level Configuration
|
|
|
-
|
|
|
-# We use semantic versioning: https://semver.org
|
|
|
-# In particular:
|
|
|
-# * major changes include removing a mandatory field, or anything else that
|
|
|
-# would break an appropriately tolerant parser,
|
|
|
-# * minor changes include adding a field,
|
|
|
-# * patch changes include changing header comments or other unstructured
|
|
|
-# content
|
|
|
-FALLBACK_FORMAT_VERSION = '2.0.0'
|
|
|
-SECTION_SEPARATOR_BASE = '====='
|
|
|
-SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
|
|
|
-
|
|
|
-# Output all candidate fallbacks, or only output selected fallbacks?
|
|
|
-OUTPUT_CANDIDATES = False
|
|
|
-
|
|
|
-# Perform DirPort checks over IPv4?
|
|
|
-# Change this to False if IPv4 doesn't work for you, or if you don't want to
|
|
|
-# download a consensus for each fallback
|
|
|
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
|
|
|
-PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
|
|
|
-
|
|
|
-# Perform DirPort checks over IPv6?
|
|
|
-# If you know IPv6 works for you, set this to True
|
|
|
-# This will exclude IPv6 relays without an IPv6 DirPort configured
|
|
|
-# So it's best left at False until #18394 is implemented
|
|
|
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
|
|
|
-PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
|
|
|
-
|
|
|
-# Must relays be running now?
|
|
|
-MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
|
|
|
- or PERFORM_IPV6_DIRPORT_CHECKS)
|
|
|
-
|
|
|
-# Clients have been using microdesc consensuses by default for a while now
|
|
|
-DOWNLOAD_MICRODESC_CONSENSUS = True
|
|
|
-
|
|
|
-# If a relay delivers an invalid consensus, if it will become valid less than
|
|
|
-# this many seconds in the future, or expired less than this many seconds ago,
|
|
|
-# accept the relay as a fallback. For the consensus expiry check to be
|
|
|
-# accurate, the machine running this script needs an accurate clock.
|
|
|
-#
|
|
|
-# Relays on 0.3.0 and later return a 404 when they are about to serve a
|
|
|
-# consensus that expired more than 24 hours ago. 0.2.9 and earlier relays
|
|
|
-# will serve consensuses that are very old.
|
|
|
-#
|
|
|
-# Relays on 0.3.5.6-rc? and later return a 404 when they are about to serve a
|
|
|
-# consensus that will become valid more than 24 hours in the future. Older
|
|
|
-# relays don't serve future consensuses.
|
|
|
-#
|
|
|
-# A 404 makes relays fail the download check. We use a tolerance of 24 hours,
|
|
|
-# so that 0.2.9 relays also fail the download check if they serve a consensus
|
|
|
-# that is not reasonably live.
|
|
|
-#
|
|
|
-# REASONABLY_LIVE_TIME should never be more than Tor's REASONABLY_LIVE_TIME,
|
|
|
-# (24 hours), because clients reject consensuses that are older than that.
|
|
|
-# Clients on 0.3.5.5-alpha? and earlier also won't select guards from
|
|
|
-# consensuses that have expired, but can bootstrap if they already have guards
|
|
|
-# in their state file.
|
|
|
-REASONABLY_LIVE_TIME = 24*60*60
|
|
|
-
|
|
|
-# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
|
|
|
-OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
|
|
|
-
|
|
|
-# Output matching ContactInfo in fallbacks list?
|
|
|
-# Useful if you're trying to contact operators
|
|
|
-CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
|
|
|
-
|
|
|
-# How the list should be sorted:
|
|
|
-# fingerprint: is useful for stable diffs of fallback lists
|
|
|
-# measured_bandwidth: is useful when pruning the list based on bandwidth
|
|
|
-# contact: is useful for contacting operators once the list has been pruned
|
|
|
-OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
|
|
|
-
|
|
|
-## OnionOO Settings
|
|
|
-
|
|
|
-ONIONOO = 'https://onionoo.torproject.org/'
|
|
|
-#ONIONOO = 'https://onionoo.thecthulhu.com/'
|
|
|
-
|
|
|
-# Don't bother going out to the Internet, just use the files available locally,
|
|
|
-# even if they're very old
|
|
|
-LOCAL_FILES_ONLY = False
|
|
|
-
|
|
|
-## Whitelist / Blacklist Filter Settings
|
|
|
-
|
|
|
-# The whitelist contains entries that are included if all attributes match
|
|
|
-# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport)
|
|
|
-
|
|
|
-# What happens to entries not in whitelist?
|
|
|
-# When True, they are included, when False, they are excluded
|
|
|
-INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
|
|
|
-
|
|
|
-WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
|
|
|
-FALLBACK_FILE_NAME = 'src/app/config/fallback_dirs.inc'
|
|
|
-
|
|
|
-# The number of bytes we'll read from a filter file before giving up
|
|
|
-MAX_LIST_FILE_SIZE = 1024 * 1024
|
|
|
-
|
|
|
-## Eligibility Settings
|
|
|
-
|
|
|
-# Require fallbacks to have the same address and port for a set amount of time
|
|
|
-# We used to have this at 1 week, but that caused many fallback failures, which
|
|
|
-# meant that we had to rebuild the list more often. We want fallbacks to be
|
|
|
-# stable for 2 years, so we set it to a few months.
|
|
|
-#
|
|
|
-# If a relay changes address or port, that's it, it's not useful any more,
|
|
|
-# because clients can't find it
|
|
|
-ADDRESS_AND_PORT_STABLE_DAYS = 90
|
|
|
-# We ignore relays that have been down for more than this period
|
|
|
-MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
|
|
|
-# FallbackDirs must have a time-weighted-fraction that is greater than or
|
|
|
-# equal to:
|
|
|
-# Mirrors that are down half the time are still useful half the time
|
|
|
-CUTOFF_RUNNING = .50
|
|
|
-CUTOFF_V2DIR = .50
|
|
|
-# Guard flags are removed for some time after a relay restarts, so we ignore
|
|
|
-# the guard flag.
|
|
|
-CUTOFF_GUARD = .00
|
|
|
-# FallbackDirs must have a time-weighted-fraction that is less than or equal
|
|
|
-# to:
|
|
|
-# .00 means no bad exits
|
|
|
-PERMITTED_BADEXIT = .00
|
|
|
-
|
|
|
-# older entries' weights are adjusted with ALPHA^(age in days)
|
|
|
-AGE_ALPHA = 0.99
|
|
|
-
|
|
|
-# this factor is used to scale OnionOO entries to [0,1]
|
|
|
-ONIONOO_SCALE_ONE = 999.
|
|
|
-
|
|
|
-## Fallback Count Limits
|
|
|
-
|
|
|
-# The target for these parameters is 20% of the guards in the network
|
|
|
-# This is around 200 as of October 2015
|
|
|
-_FB_POG = 0.2
|
|
|
-FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
|
|
|
-
|
|
|
-# Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
|
|
|
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
|
|
|
-# Emit a C #error if the number of fallbacks is less than expected
|
|
|
-MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
|
|
|
-
|
|
|
-# The maximum number of fallbacks on the same address, contact, or family
|
|
|
-#
|
|
|
-# With 150 fallbacks, this means each operator sees 5% of client bootstraps.
|
|
|
-# For comparison:
|
|
|
-# - We try to limit guard and exit operators to 5% of the network
|
|
|
-# - The directory authorities used to see 11% of client bootstraps each
|
|
|
-#
|
|
|
-# We also don't want too much of the list to go down if a single operator
|
|
|
-# has to move all their relays.
|
|
|
-MAX_FALLBACKS_PER_IP = 1
|
|
|
-MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
|
|
|
-MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
|
|
|
-MAX_FALLBACKS_PER_CONTACT = 7
|
|
|
-MAX_FALLBACKS_PER_FAMILY = 7
|
|
|
-
|
|
|
-## Fallback Bandwidth Requirements
|
|
|
-
|
|
|
-# Any fallback with the Exit flag has its bandwidth multiplied by this fraction
|
|
|
-# to make sure we aren't further overloading exits
|
|
|
-# (Set to 1.0, because we asked that only lightly loaded exits opt-in,
|
|
|
-# and the extra load really isn't that much for large relays.)
|
|
|
-EXIT_BANDWIDTH_FRACTION = 1.0
|
|
|
-
|
|
|
-# If a single fallback's bandwidth is too low, it's pointless adding it
|
|
|
-# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
|
|
|
-# Make sure they can support fifty times the expected extra load
|
|
|
-#
|
|
|
-# We convert this to a consensus weight before applying the filter,
|
|
|
-# because all the bandwidth amounts are specified by the relay
|
|
|
-MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
|
|
|
-
|
|
|
-# Clients will time out after 30 seconds trying to download a consensus
|
|
|
-# So allow fallback directories half that to deliver a consensus
|
|
|
-# The exact download times might change based on the network connection
|
|
|
-# running this script, but only by a few seconds
|
|
|
-# There is also about a second of python overhead
|
|
|
-CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
|
|
|
-# If the relay fails a consensus check, retry the download
|
|
|
-# This avoids delisting a relay due to transient network conditions
|
|
|
-CONSENSUS_DOWNLOAD_RETRY = True
|
|
|
-
|
|
|
-## Parsing Functions
|
|
|
-
|
|
|
-def parse_ts(t):
|
|
|
- return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
|
|
|
-
|
|
|
-def remove_bad_chars(raw_string, bad_char_list):
|
|
|
- # Remove each character in the bad_char_list
|
|
|
- cleansed_string = raw_string
|
|
|
- for c in bad_char_list:
|
|
|
- cleansed_string = cleansed_string.replace(c, '')
|
|
|
- return cleansed_string
|
|
|
-
|
|
|
-def cleanse_unprintable(raw_string):
|
|
|
- # Remove all unprintable characters
|
|
|
- cleansed_string = ''
|
|
|
- for c in raw_string:
|
|
|
- if c in string.printable:
|
|
|
- cleansed_string += c
|
|
|
- return cleansed_string
|
|
|
-
|
|
|
-def cleanse_whitespace(raw_string):
|
|
|
- # Replace all whitespace characters with a space
|
|
|
- cleansed_string = raw_string
|
|
|
- for c in string.whitespace:
|
|
|
- cleansed_string = cleansed_string.replace(c, ' ')
|
|
|
- return cleansed_string
|
|
|
-
|
|
|
-def cleanse_c_multiline_comment(raw_string):
|
|
|
- cleansed_string = raw_string
|
|
|
- # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
|
|
|
- cleansed_string = cleanse_whitespace(cleansed_string)
|
|
|
- # ContactInfo and Version can be arbitrary binary data
|
|
|
- cleansed_string = cleanse_unprintable(cleansed_string)
|
|
|
- # Prevent a malicious / unanticipated string from breaking out
|
|
|
- # of a C-style multiline comment
|
|
|
- # This removes '/*' and '*/' and '//'
|
|
|
- bad_char_list = '*/'
|
|
|
- # Prevent a malicious string from using C nulls
|
|
|
- bad_char_list += '\0'
|
|
|
- # Avoid confusing parsers by making sure there is only one comma per fallback
|
|
|
- bad_char_list += ','
|
|
|
- # Avoid confusing parsers by making sure there is only one equals per field
|
|
|
- bad_char_list += '='
|
|
|
- # Be safer by removing bad characters entirely
|
|
|
- cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
|
|
|
- # Some compilers may further process the content of comments
|
|
|
- # There isn't much we can do to cover every possible case
|
|
|
- # But comment-based directives are typically only advisory
|
|
|
- return cleansed_string
|
|
|
-
|
|
|
-def cleanse_c_string(raw_string):
|
|
|
- cleansed_string = raw_string
|
|
|
- # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
|
|
|
- cleansed_string = cleanse_whitespace(cleansed_string)
|
|
|
- # ContactInfo and Version can be arbitrary binary data
|
|
|
- cleansed_string = cleanse_unprintable(cleansed_string)
|
|
|
- # Prevent a malicious address/fingerprint string from breaking out
|
|
|
- # of a C-style string
|
|
|
- bad_char_list = '"'
|
|
|
- # Prevent a malicious string from using escapes
|
|
|
- bad_char_list += '\\'
|
|
|
- # Prevent a malicious string from using C nulls
|
|
|
- bad_char_list += '\0'
|
|
|
- # Avoid confusing parsers by making sure there is only one comma per fallback
|
|
|
- bad_char_list += ','
|
|
|
- # Avoid confusing parsers by making sure there is only one equals per field
|
|
|
- bad_char_list += '='
|
|
|
- # Be safer by removing bad characters entirely
|
|
|
- cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
|
|
|
- # Some compilers may further process the content of strings
|
|
|
- # There isn't much we can do to cover every possible case
|
|
|
- # But this typically only results in changes to the string data
|
|
|
- return cleansed_string
|
|
|
-
|
|
|
-## OnionOO Source Functions
|
|
|
-
|
|
|
-# a dictionary of source metadata for each onionoo query we've made
|
|
|
-fetch_source = {}
|
|
|
-
|
|
|
-# register source metadata for 'what'
|
|
|
-# assumes we only retrieve one document for each 'what'
|
|
|
-def register_fetch_source(what, url, relays_published, version):
|
|
|
- fetch_source[what] = {}
|
|
|
- fetch_source[what]['url'] = url
|
|
|
- fetch_source[what]['relays_published'] = relays_published
|
|
|
- fetch_source[what]['version'] = version
|
|
|
-
|
|
|
-# list each registered source's 'what'
|
|
|
-def fetch_source_list():
|
|
|
- return sorted(fetch_source.keys())
|
|
|
-
|
|
|
-# given 'what', provide a multiline C comment describing the source
|
|
|
-def describe_fetch_source(what):
|
|
|
- desc = '/*'
|
|
|
- desc += '\n'
|
|
|
- desc += 'Onionoo Source: '
|
|
|
- desc += cleanse_c_multiline_comment(what)
|
|
|
- desc += ' Date: '
|
|
|
- desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published'])
|
|
|
- desc += ' Version: '
|
|
|
- desc += cleanse_c_multiline_comment(fetch_source[what]['version'])
|
|
|
- desc += '\n'
|
|
|
- desc += 'URL: '
|
|
|
- desc += cleanse_c_multiline_comment(fetch_source[what]['url'])
|
|
|
- desc += '\n'
|
|
|
- desc += '*/'
|
|
|
- return desc
|
|
|
-
|
|
|
-## File Processing Functions
|
|
|
-
|
|
|
-def write_to_file(str, file_name, max_len):
|
|
|
- try:
|
|
|
- with open(file_name, 'w') as f:
|
|
|
- f.write(str[0:max_len])
|
|
|
- except EnvironmentError, error:
|
|
|
- logging.error('Writing file %s failed: %d: %s'%
|
|
|
- (file_name,
|
|
|
- error.errno,
|
|
|
- error.strerror)
|
|
|
- )
|
|
|
-
|
|
|
-def read_from_file(file_name, max_len):
|
|
|
- try:
|
|
|
- if os.path.isfile(file_name):
|
|
|
- with open(file_name, 'r') as f:
|
|
|
- return f.read(max_len)
|
|
|
- except EnvironmentError, error:
|
|
|
- logging.info('Loading file %s failed: %d: %s'%
|
|
|
- (file_name,
|
|
|
- error.errno,
|
|
|
- error.strerror)
|
|
|
- )
|
|
|
- return None
|
|
|
-
|
|
|
-def parse_fallback_file(file_name):
|
|
|
- file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
|
|
|
- file_data = cleanse_unprintable(file_data)
|
|
|
- file_data = remove_bad_chars(file_data, '\n"\0')
|
|
|
- file_data = re.sub('/\*.*?\*/', '', file_data)
|
|
|
- file_data = file_data.replace(',', '\n')
|
|
|
- file_data = file_data.replace(' weight=10', '')
|
|
|
- return file_data
|
|
|
-
|
|
|
-def load_possibly_compressed_response_json(response):
|
|
|
- if response.info().get('Content-Encoding') == 'gzip':
|
|
|
- buf = StringIO.StringIO( response.read() )
|
|
|
- f = gzip.GzipFile(fileobj=buf)
|
|
|
- return json.load(f)
|
|
|
- else:
|
|
|
- return json.load(response)
|
|
|
-
|
|
|
-def load_json_from_file(json_file_name):
|
|
|
- # An exception here may be resolved by deleting the .last_modified
|
|
|
- # and .json files, and re-running the script
|
|
|
- try:
|
|
|
- with open(json_file_name, 'r') as f:
|
|
|
- return json.load(f)
|
|
|
- except EnvironmentError, error:
|
|
|
- raise Exception('Reading not-modified json file %s failed: %d: %s'%
|
|
|
- (json_file_name,
|
|
|
- error.errno,
|
|
|
- error.strerror)
|
|
|
- )
|
|
|
-
|
|
|
-## OnionOO Functions
|
|
|
-
|
|
|
-def datestr_to_datetime(datestr):
|
|
|
- # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT
|
|
|
- if datestr is not None:
|
|
|
- dt = dateutil.parser.parse(datestr)
|
|
|
- else:
|
|
|
- # Never modified - use start of epoch
|
|
|
- dt = datetime.datetime.utcfromtimestamp(0)
|
|
|
- # strip any timezone out (in case they're supported in future)
|
|
|
- dt = dt.replace(tzinfo=None)
|
|
|
- return dt
|
|
|
-
|
|
|
-def onionoo_fetch(what, **kwargs):
|
|
|
- params = kwargs
|
|
|
- params['type'] = 'relay'
|
|
|
- #params['limit'] = 10
|
|
|
- params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
|
|
|
- params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
|
|
|
- params['flag'] = 'V2Dir'
|
|
|
- url = ONIONOO + what + '?' + urllib.urlencode(params)
|
|
|
-
|
|
|
- # Unfortunately, the URL is too long for some OS filenames,
|
|
|
- # but we still don't want to get files from different URLs mixed up
|
|
|
- base_file_name = what + '-' + hashlib.sha1(url).hexdigest()
|
|
|
-
|
|
|
- full_url_file_name = base_file_name + '.full_url'
|
|
|
- MAX_FULL_URL_LENGTH = 1024
|
|
|
-
|
|
|
- last_modified_file_name = base_file_name + '.last_modified'
|
|
|
- MAX_LAST_MODIFIED_LENGTH = 64
|
|
|
-
|
|
|
- json_file_name = base_file_name + '.json'
|
|
|
-
|
|
|
- if LOCAL_FILES_ONLY:
|
|
|
- # Read from the local file, don't write to anything
|
|
|
- response_json = load_json_from_file(json_file_name)
|
|
|
- else:
|
|
|
- # store the full URL to a file for debugging
|
|
|
- # no need to compare as long as you trust SHA-1
|
|
|
- write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH)
|
|
|
-
|
|
|
- request = urllib2.Request(url)
|
|
|
- request.add_header('Accept-encoding', 'gzip')
|
|
|
-
|
|
|
- # load the last modified date from the file, if it exists
|
|
|
- last_mod_date = read_from_file(last_modified_file_name,
|
|
|
- MAX_LAST_MODIFIED_LENGTH)
|
|
|
- if last_mod_date is not None:
|
|
|
- request.add_header('If-modified-since', last_mod_date)
|
|
|
-
|
|
|
- # Parse last modified date
|
|
|
- last_mod = datestr_to_datetime(last_mod_date)
|
|
|
-
|
|
|
- # Not Modified and still recent enough to be useful
|
|
|
- # Onionoo / Globe used to use 6 hours, but we can afford a day
|
|
|
- required_freshness = datetime.datetime.utcnow()
|
|
|
- # strip any timezone out (to match dateutil.parser)
|
|
|
- required_freshness = required_freshness.replace(tzinfo=None)
|
|
|
- required_freshness -= datetime.timedelta(hours=24)
|
|
|
-
|
|
|
- # Make the OnionOO request
|
|
|
- response_code = 0
|
|
|
- try:
|
|
|
- response = urllib2.urlopen(request)
|
|
|
- response_code = response.getcode()
|
|
|
- except urllib2.HTTPError, error:
|
|
|
- response_code = error.code
|
|
|
- if response_code == 304: # not modified
|
|
|
- pass
|
|
|
- else:
|
|
|
- raise Exception("Could not get " + url + ": "
|
|
|
- + str(error.code) + ": " + error.reason)
|
|
|
-
|
|
|
- if response_code == 200: # OK
|
|
|
- last_mod = datestr_to_datetime(response.info().get('Last-Modified'))
|
|
|
-
|
|
|
- # Check for freshness
|
|
|
- if last_mod < required_freshness:
|
|
|
- if last_mod_date is not None:
|
|
|
- # This check sometimes fails transiently, retry the script if it does
|
|
|
- date_message = "Outdated data: last updated " + last_mod_date
|
|
|
- else:
|
|
|
- date_message = "No data: never downloaded "
|
|
|
- raise Exception(date_message + " from " + url)
|
|
|
-
|
|
|
- # Process the data
|
|
|
- if response_code == 200: # OK
|
|
|
-
|
|
|
- response_json = load_possibly_compressed_response_json(response)
|
|
|
-
|
|
|
- with open(json_file_name, 'w') as f:
|
|
|
- # use the most compact json representation to save space
|
|
|
- json.dump(response_json, f, separators=(',',':'))
|
|
|
-
|
|
|
- # store the last modified date in its own file
|
|
|
- if response.info().get('Last-modified') is not None:
|
|
|
- write_to_file(response.info().get('Last-Modified'),
|
|
|
- last_modified_file_name,
|
|
|
- MAX_LAST_MODIFIED_LENGTH)
|
|
|
-
|
|
|
- elif response_code == 304: # Not Modified
|
|
|
-
|
|
|
- response_json = load_json_from_file(json_file_name)
|
|
|
-
|
|
|
- else: # Unexpected HTTP response code not covered in the HTTPError above
|
|
|
- raise Exception("Unexpected HTTP response code to " + url + ": "
|
|
|
- + str(response_code))
|
|
|
-
|
|
|
- register_fetch_source(what,
|
|
|
- url,
|
|
|
- response_json['relays_published'],
|
|
|
- response_json['version'])
|
|
|
-
|
|
|
- return response_json
|
|
|
-
|
|
|
-def fetch(what, **kwargs):
|
|
|
- #x = onionoo_fetch(what, **kwargs)
|
|
|
- # don't use sort_keys, as the order of or_addresses is significant
|
|
|
- #print json.dumps(x, indent=4, separators=(',', ': '))
|
|
|
- #sys.exit(0)
|
|
|
-
|
|
|
- return onionoo_fetch(what, **kwargs)
|
|
|
-
|
|
|
-## Fallback Candidate Class
|
|
|
-
|
|
|
-class Candidate(object):
|
|
|
- CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow()
|
|
|
- - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS))
|
|
|
-
|
|
|
- def __init__(self, details):
|
|
|
- for f in ['fingerprint', 'nickname', 'last_changed_address_or_port',
|
|
|
- 'consensus_weight', 'or_addresses', 'dir_address']:
|
|
|
- if not f in details: raise Exception("Document has no %s field."%(f,))
|
|
|
-
|
|
|
- if not 'contact' in details:
|
|
|
- details['contact'] = None
|
|
|
- if not 'flags' in details or details['flags'] is None:
|
|
|
- details['flags'] = []
|
|
|
- if (not 'advertised_bandwidth' in details
|
|
|
- or details['advertised_bandwidth'] is None):
|
|
|
- # relays without advertised bandwidth have it calculated from their
|
|
|
- # consensus weight
|
|
|
- details['advertised_bandwidth'] = 0
|
|
|
- if (not 'effective_family' in details
|
|
|
- or details['effective_family'] is None):
|
|
|
- details['effective_family'] = []
|
|
|
- if not 'platform' in details:
|
|
|
- details['platform'] = None
|
|
|
- details['last_changed_address_or_port'] = parse_ts(
|
|
|
- details['last_changed_address_or_port'])
|
|
|
- self._data = details
|
|
|
- self._stable_sort_or_addresses()
|
|
|
-
|
|
|
- self._fpr = self._data['fingerprint']
|
|
|
- self._running = self._guard = self._v2dir = 0.
|
|
|
- self._split_dirport()
|
|
|
- self._compute_orport()
|
|
|
- if self.orport is None:
|
|
|
- raise Exception("Failed to get an orport for %s."%(self._fpr,))
|
|
|
- self._compute_ipv6addr()
|
|
|
- if not self.has_ipv6():
|
|
|
- logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
|
|
|
- self._compute_version()
|
|
|
- self._extra_info_cache = None
|
|
|
-
|
|
|
- def _stable_sort_or_addresses(self):
|
|
|
- # replace self._data['or_addresses'] with a stable ordering,
|
|
|
- # sorting the secondary addresses in string order
|
|
|
- # leave the received order in self._data['or_addresses_raw']
|
|
|
- self._data['or_addresses_raw'] = self._data['or_addresses']
|
|
|
- or_address_primary = self._data['or_addresses'][:1]
|
|
|
- # subsequent entries in the or_addresses array are in an arbitrary order
|
|
|
- # so we stabilise the addresses by sorting them in string order
|
|
|
- or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:])
|
|
|
- or_addresses_stable = or_address_primary + or_addresses_secondaries_stable
|
|
|
- self._data['or_addresses'] = or_addresses_stable
|
|
|
-
|
|
|
- def get_fingerprint(self):
|
|
|
- return self._fpr
|
|
|
-
|
|
|
- # is_valid_ipv[46]_address by gsathya, karsten, 2013
|
|
|
- @staticmethod
|
|
|
- def is_valid_ipv4_address(address):
|
|
|
- if not isinstance(address, (str, unicode)):
|
|
|
- return False
|
|
|
-
|
|
|
- # check if there are four period separated values
|
|
|
- if address.count(".") != 3:
|
|
|
- return False
|
|
|
-
|
|
|
- # checks that each value in the octet are decimal values between 0-255
|
|
|
- for entry in address.split("."):
|
|
|
- if not entry.isdigit() or int(entry) < 0 or int(entry) > 255:
|
|
|
- return False
|
|
|
- elif entry[0] == "0" and len(entry) > 1:
|
|
|
- return False # leading zeros, for instance in "1.2.3.001"
|
|
|
-
|
|
|
- return True
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def is_valid_ipv6_address(address):
|
|
|
- if not isinstance(address, (str, unicode)):
|
|
|
- return False
|
|
|
-
|
|
|
- # remove brackets
|
|
|
- address = address[1:-1]
|
|
|
-
|
|
|
- # addresses are made up of eight colon separated groups of four hex digits
|
|
|
- # with leading zeros being optional
|
|
|
- # https://en.wikipedia.org/wiki/IPv6#Address_format
|
|
|
-
|
|
|
- colon_count = address.count(":")
|
|
|
-
|
|
|
- if colon_count > 7:
|
|
|
- return False # too many groups
|
|
|
- elif colon_count != 7 and not "::" in address:
|
|
|
- return False # not enough groups and none are collapsed
|
|
|
- elif address.count("::") > 1 or ":::" in address:
|
|
|
- return False # multiple groupings of zeros can't be collapsed
|
|
|
-
|
|
|
- found_ipv4_on_previous_entry = False
|
|
|
- for entry in address.split(":"):
|
|
|
- # If an IPv6 address has an embedded IPv4 address,
|
|
|
- # it must be the last entry
|
|
|
- if found_ipv4_on_previous_entry:
|
|
|
- return False
|
|
|
- if not re.match("^[0-9a-fA-f]{0,4}$", entry):
|
|
|
- if not Candidate.is_valid_ipv4_address(entry):
|
|
|
- return False
|
|
|
- else:
|
|
|
- found_ipv4_on_previous_entry = True
|
|
|
-
|
|
|
- return True
|
|
|
-
|
|
|
- def _split_dirport(self):
|
|
|
- # Split the dir_address into dirip and dirport
|
|
|
- (self.dirip, _dirport) = self._data['dir_address'].split(':', 2)
|
|
|
- self.dirport = int(_dirport)
|
|
|
-
|
|
|
- def _compute_orport(self):
|
|
|
- # Choose the first ORPort that's on the same IPv4 address as the DirPort.
|
|
|
- # In rare circumstances, this might not be the primary ORPort address.
|
|
|
- # However, _stable_sort_or_addresses() ensures we choose the same one
|
|
|
- # every time, even if onionoo changes the order of the secondaries.
|
|
|
- self._split_dirport()
|
|
|
- self.orport = None
|
|
|
- for i in self._data['or_addresses']:
|
|
|
- if i != self._data['or_addresses'][0]:
|
|
|
- logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i))
|
|
|
- (ipaddr, port) = i.rsplit(':', 1)
|
|
|
- if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr):
|
|
|
- self.orport = int(port)
|
|
|
- return
|
|
|
-
|
|
|
- def _compute_ipv6addr(self):
|
|
|
- # Choose the first IPv6 address that uses the same port as the ORPort
|
|
|
- # Or, choose the first IPv6 address in the list
|
|
|
- # _stable_sort_or_addresses() ensures we choose the same IPv6 address
|
|
|
- # every time, even if onionoo changes the order of the secondaries.
|
|
|
- self.ipv6addr = None
|
|
|
- self.ipv6orport = None
|
|
|
- # Choose the first IPv6 address that uses the same port as the ORPort
|
|
|
- for i in self._data['or_addresses']:
|
|
|
- (ipaddr, port) = i.rsplit(':', 1)
|
|
|
- if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr):
|
|
|
- self.ipv6addr = ipaddr
|
|
|
- self.ipv6orport = int(port)
|
|
|
- return
|
|
|
- # Choose the first IPv6 address in the list
|
|
|
- for i in self._data['or_addresses']:
|
|
|
- (ipaddr, port) = i.rsplit(':', 1)
|
|
|
- if Candidate.is_valid_ipv6_address(ipaddr):
|
|
|
- self.ipv6addr = ipaddr
|
|
|
- self.ipv6orport = int(port)
|
|
|
- return
|
|
|
-
|
|
|
- def _compute_version(self):
|
|
|
- # parse the version out of the platform string
|
|
|
- # The platform looks like: "Tor 0.2.7.6 on Linux"
|
|
|
- self._data['version'] = None
|
|
|
- if self._data['platform'] is None:
|
|
|
- return
|
|
|
- # be tolerant of weird whitespacing, use a whitespace split
|
|
|
- tokens = self._data['platform'].split()
|
|
|
- for token in tokens:
|
|
|
- vnums = token.split('.')
|
|
|
- # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
|
|
|
- if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
|
|
|
- vnums[2].isdigit()):
|
|
|
- self._data['version'] = token
|
|
|
- return
|
|
|
-
|
|
|
- # From #20509
|
|
|
- # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
|
|
|
- # and version 0.3.0.0-alpha-dev
|
|
|
- # Exhaustive lists are hard to get wrong
|
|
|
- STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
|
|
|
- '0.2.9.2-alpha',
|
|
|
- '0.2.9.2-alpha-dev',
|
|
|
- '0.2.9.3-alpha',
|
|
|
- '0.2.9.3-alpha-dev',
|
|
|
- '0.2.9.4-alpha',
|
|
|
- '0.2.9.4-alpha-dev',
|
|
|
- '0.3.0.0-alpha-dev'
|
|
|
- ]
|
|
|
-
|
|
|
- def is_valid_version(self):
|
|
|
- # call _compute_version before calling this
|
|
|
- # is the version of the relay a version we want as a fallback?
|
|
|
- # checks both recommended versions and bug #20499 / #20509
|
|
|
- #
|
|
|
- # if the relay doesn't have a recommended version field, exclude the relay
|
|
|
- if not self._data.has_key('recommended_version'):
|
|
|
- log_excluded('%s not a candidate: no recommended_version field',
|
|
|
- self._fpr)
|
|
|
- return False
|
|
|
- if not self._data['recommended_version']:
|
|
|
- log_excluded('%s not a candidate: version not recommended', self._fpr)
|
|
|
- return False
|
|
|
- # if the relay doesn't have version field, exclude the relay
|
|
|
- if not self._data.has_key('version'):
|
|
|
- log_excluded('%s not a candidate: no version field', self._fpr)
|
|
|
- return False
|
|
|
- if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
|
|
|
- logging.warning('%s not a candidate: version delivers stale consensuses',
|
|
|
- self._fpr)
|
|
|
- return False
|
|
|
- return True
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _extract_generic_history(history, which='unknown'):
|
|
|
- # given a tree like this:
|
|
|
- # {
|
|
|
- # "1_month": {
|
|
|
- # "count": 187,
|
|
|
- # "factor": 0.001001001001001001,
|
|
|
- # "first": "2015-02-27 06:00:00",
|
|
|
- # "interval": 14400,
|
|
|
- # "last": "2015-03-30 06:00:00",
|
|
|
- # "values": [
|
|
|
- # 999,
|
|
|
- # 999
|
|
|
- # ]
|
|
|
- # },
|
|
|
- # "1_week": {
|
|
|
- # "count": 169,
|
|
|
- # "factor": 0.001001001001001001,
|
|
|
- # "first": "2015-03-23 07:30:00",
|
|
|
- # "interval": 3600,
|
|
|
- # "last": "2015-03-30 07:30:00",
|
|
|
- # "values": [ ...]
|
|
|
- # },
|
|
|
- # "1_year": {
|
|
|
- # "count": 177,
|
|
|
- # "factor": 0.001001001001001001,
|
|
|
- # "first": "2014-04-11 00:00:00",
|
|
|
- # "interval": 172800,
|
|
|
- # "last": "2015-03-29 00:00:00",
|
|
|
- # "values": [ ...]
|
|
|
- # },
|
|
|
- # "3_months": {
|
|
|
- # "count": 185,
|
|
|
- # "factor": 0.001001001001001001,
|
|
|
- # "first": "2014-12-28 06:00:00",
|
|
|
- # "interval": 43200,
|
|
|
- # "last": "2015-03-30 06:00:00",
|
|
|
- # "values": [ ...]
|
|
|
- # }
|
|
|
- # },
|
|
|
- # extract exactly one piece of data per time interval,
|
|
|
- # using smaller intervals where available.
|
|
|
- #
|
|
|
- # returns list of (age, length, value) dictionaries.
|
|
|
-
|
|
|
- generic_history = []
|
|
|
-
|
|
|
- periods = history.keys()
|
|
|
- periods.sort(key = lambda x: history[x]['interval'])
|
|
|
- now = datetime.datetime.utcnow()
|
|
|
- newest = now
|
|
|
- for p in periods:
|
|
|
- h = history[p]
|
|
|
- interval = datetime.timedelta(seconds = h['interval'])
|
|
|
- this_ts = parse_ts(h['last'])
|
|
|
-
|
|
|
- if (len(h['values']) != h['count']):
|
|
|
- logging.warning('Inconsistent value count in %s document for %s'
|
|
|
- %(p, which))
|
|
|
- for v in reversed(h['values']):
|
|
|
- if (this_ts <= newest):
|
|
|
- agt1 = now - this_ts
|
|
|
- agt2 = interval
|
|
|
- agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600)
|
|
|
- * 10**6) / 10**6
|
|
|
- agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600)
|
|
|
- * 10**6) / 10**6
|
|
|
- generic_history.append(
|
|
|
- { 'age': agetmp1,
|
|
|
- 'length': agetmp2,
|
|
|
- 'value': v
|
|
|
- })
|
|
|
- newest = this_ts
|
|
|
- this_ts -= interval
|
|
|
-
|
|
|
- if (this_ts + interval != parse_ts(h['first'])):
|
|
|
- logging.warning('Inconsistent time information in %s document for %s'
|
|
|
- %(p, which))
|
|
|
-
|
|
|
- #print json.dumps(generic_history, sort_keys=True,
|
|
|
- # indent=4, separators=(',', ': '))
|
|
|
- return generic_history
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _avg_generic_history(generic_history):
|
|
|
- a = []
|
|
|
- for i in generic_history:
|
|
|
- if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600):
|
|
|
- continue
|
|
|
- if (i['length'] is not None
|
|
|
- and i['age'] is not None
|
|
|
- and i['value'] is not None):
|
|
|
- w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24))
|
|
|
- a.append( (i['value'] * w, w) )
|
|
|
-
|
|
|
- sv = math.fsum(map(lambda x: x[0], a))
|
|
|
- sw = math.fsum(map(lambda x: x[1], a))
|
|
|
-
|
|
|
- if sw == 0.0:
|
|
|
- svw = 0.0
|
|
|
- else:
|
|
|
- svw = sv/sw
|
|
|
- return svw
|
|
|
-
|
|
|
- def _add_generic_history(self, history):
|
|
|
- periods = r['read_history'].keys()
|
|
|
- periods.sort(key = lambda x: r['read_history'][x]['interval'] )
|
|
|
-
|
|
|
- print periods
|
|
|
-
|
|
|
- def add_running_history(self, history):
|
|
|
- pass
|
|
|
-
|
|
|
- def add_uptime(self, uptime):
|
|
|
- logging.debug('Adding uptime %s.'%(self._fpr,))
|
|
|
-
|
|
|
- # flags we care about: Running, V2Dir, Guard
|
|
|
- if not 'flags' in uptime:
|
|
|
- logging.debug('No flags in document for %s.'%(self._fpr,))
|
|
|
- return
|
|
|
-
|
|
|
- for f in ['Running', 'Guard', 'V2Dir']:
|
|
|
- if not f in uptime['flags']:
|
|
|
- logging.debug('No %s in flags for %s.'%(f, self._fpr,))
|
|
|
- return
|
|
|
-
|
|
|
- running = self._extract_generic_history(uptime['flags']['Running'],
|
|
|
- '%s-Running'%(self._fpr))
|
|
|
- guard = self._extract_generic_history(uptime['flags']['Guard'],
|
|
|
- '%s-Guard'%(self._fpr))
|
|
|
- v2dir = self._extract_generic_history(uptime['flags']['V2Dir'],
|
|
|
- '%s-V2Dir'%(self._fpr))
|
|
|
- if 'BadExit' in uptime['flags']:
|
|
|
- badexit = self._extract_generic_history(uptime['flags']['BadExit'],
|
|
|
- '%s-BadExit'%(self._fpr))
|
|
|
-
|
|
|
- self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE
|
|
|
- self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE
|
|
|
- self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE
|
|
|
- self._badexit = None
|
|
|
- if 'BadExit' in uptime['flags']:
|
|
|
- self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
|
|
|
-
|
|
|
- def is_candidate(self):
|
|
|
- try:
|
|
|
- if (MUST_BE_RUNNING_NOW and not self.is_running()):
|
|
|
- log_excluded('%s not a candidate: not running now, unable to check ' +
|
|
|
- 'DirPort consensus download', self._fpr)
|
|
|
- return False
|
|
|
- if (self._data['last_changed_address_or_port'] >
|
|
|
- self.CUTOFF_ADDRESS_AND_PORT_STABLE):
|
|
|
- log_excluded('%s not a candidate: changed address/port recently (%s)',
|
|
|
- self._fpr, self._data['last_changed_address_or_port'])
|
|
|
- return False
|
|
|
- if self._running < CUTOFF_RUNNING:
|
|
|
- log_excluded('%s not a candidate: running avg too low (%lf)',
|
|
|
- self._fpr, self._running)
|
|
|
- return False
|
|
|
- if self._v2dir < CUTOFF_V2DIR:
|
|
|
- log_excluded('%s not a candidate: v2dir avg too low (%lf)',
|
|
|
- self._fpr, self._v2dir)
|
|
|
- return False
|
|
|
- if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
|
|
|
- log_excluded('%s not a candidate: badexit avg too high (%lf)',
|
|
|
- self._fpr, self._badexit)
|
|
|
- return False
|
|
|
- # this function logs a message depending on which check fails
|
|
|
- if not self.is_valid_version():
|
|
|
- return False
|
|
|
- if self._guard < CUTOFF_GUARD:
|
|
|
- log_excluded('%s not a candidate: guard avg too low (%lf)',
|
|
|
- self._fpr, self._guard)
|
|
|
- return False
|
|
|
- if (not self._data.has_key('consensus_weight')
|
|
|
- or self._data['consensus_weight'] < 1):
|
|
|
- log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
|
|
|
- return False
|
|
|
- except BaseException as e:
|
|
|
- logging.warning("Exception %s when checking if fallback is a candidate",
|
|
|
- str(e))
|
|
|
- return False
|
|
|
- return True
|
|
|
-
|
|
|
- def id_matches(self, id, exact=False):
|
|
|
- """ Does this fallback's id match id?
|
|
|
- exact is ignored. """
|
|
|
- return self._fpr == id
|
|
|
-
|
|
|
- def ipv4_addr_matches(self, ipv4_addr, exact=False):
|
|
|
- """ Does this fallback's IPv4 address match ipv4_addr?
|
|
|
- exact is ignored. """
|
|
|
- return self.dirip == ipv4_addr
|
|
|
-
|
|
|
- def ipv4_dirport_matches(self, ipv4_dirport, exact=False):
|
|
|
- """ Does this fallback's IPv4 dirport match ipv4_dirport?
|
|
|
- If exact is False, always return True. """
|
|
|
- if exact:
|
|
|
- return self.dirport == int(ipv4_dirport)
|
|
|
- else:
|
|
|
- return True
|
|
|
-
|
|
|
- def ipv4_and_dirport_matches(self, ipv4_addr, ipv4_dirport, exact=False):
|
|
|
- """ Does this fallback's IPv4 address match ipv4_addr?
|
|
|
- If exact is True, also check ipv4_dirport. """
|
|
|
- ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact)
|
|
|
- if exact:
|
|
|
- return ipv4_match and self.ipv4_dirport_matches(ipv4_dirport,
|
|
|
- exact=exact)
|
|
|
- else:
|
|
|
- return ipv4_match
|
|
|
-
|
|
|
- def ipv4_orport_matches(self, ipv4_orport, exact=False):
|
|
|
- """ Does this fallback's IPv4 orport match ipv4_orport?
|
|
|
- If exact is False, always return True. """
|
|
|
- if exact:
|
|
|
- return self.orport == int(ipv4_orport)
|
|
|
- else:
|
|
|
- return True
|
|
|
-
|
|
|
- def ipv4_and_orport_matches(self, ipv4_addr, ipv4_orport, exact=False):
|
|
|
- """ Does this fallback's IPv4 address match ipv4_addr?
|
|
|
- If exact is True, also check ipv4_orport. """
|
|
|
- ipv4_match = self.ipv4_addr_matches(ipv4_addr, exact=exact)
|
|
|
- if exact:
|
|
|
- return ipv4_match and self.ipv4_orport_matches(ipv4_orport,
|
|
|
- exact=exact)
|
|
|
- else:
|
|
|
- return ipv4_match
|
|
|
-
|
|
|
- def ipv6_addr_matches(self, ipv6_addr, exact=False):
|
|
|
- """ Does this fallback's IPv6 address match ipv6_addr?
|
|
|
- Both addresses must be present to match.
|
|
|
- exact is ignored. """
|
|
|
- if self.has_ipv6() and ipv6_addr is not None:
|
|
|
- # Check that we have a bracketed IPv6 address without a port
|
|
|
- assert(ipv6_addr.startswith('[') and ipv6_addr.endswith(']'))
|
|
|
- return self.ipv6addr == ipv6_addr
|
|
|
- else:
|
|
|
- return False
|
|
|
-
|
|
|
- def ipv6_orport_matches(self, ipv6_orport, exact=False):
|
|
|
- """ Does this fallback's IPv6 orport match ipv6_orport?
|
|
|
- Both ports must be present to match.
|
|
|
- If exact is False, always return True. """
|
|
|
- if exact:
|
|
|
- return (self.has_ipv6() and ipv6_orport is not None and
|
|
|
- self.ipv6orport == int(ipv6_orport))
|
|
|
- else:
|
|
|
- return True
|
|
|
-
|
|
|
- def ipv6_and_orport_matches(self, ipv6_addr, ipv6_orport, exact=False):
|
|
|
- """ Does this fallback's IPv6 address match ipv6_addr?
|
|
|
- If exact is True, also check ipv6_orport. """
|
|
|
- ipv6_match = self.ipv6_addr_matches(ipv6_addr, exact=exact)
|
|
|
- if exact:
|
|
|
- return ipv6_match and self.ipv6_orport_matches(ipv6_orport,
|
|
|
- exact=exact)
|
|
|
- else:
|
|
|
- return ipv6_match
|
|
|
-
|
|
|
- def entry_matches_exact(self, entry):
|
|
|
- """ Is entry an exact match for this fallback?
|
|
|
- A fallback is an exact match for entry if each key in entry matches:
|
|
|
- ipv4
|
|
|
- dirport
|
|
|
- orport
|
|
|
- id
|
|
|
- ipv6 address and port (if present in the fallback or the whitelist)
|
|
|
- If the fallback has an ipv6 key, the whitelist line must also have
|
|
|
- it, otherwise they don't match.
|
|
|
-
|
|
|
- Logs a warning-level message if the fallback would be an exact match,
|
|
|
- but one of the id, ipv4, ipv4 orport, ipv4 dirport, or ipv6 orport
|
|
|
- have changed. """
|
|
|
- if not self.id_matches(entry['id'], exact=True):
|
|
|
- # can't log here unless we match an IP and port, because every relay's
|
|
|
- # fingerprint is compared to every entry's fingerprint
|
|
|
- if self.ipv4_and_orport_matches(entry['ipv4'],
|
|
|
- entry['orport'],
|
|
|
- exact=True):
|
|
|
- logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
|
|
|
- '%s?', entry['id'], self.dirip, self.orport,
|
|
|
- self._fpr)
|
|
|
- if self.ipv6_and_orport_matches(entry.get('ipv6_addr'),
|
|
|
- entry.get('ipv6_orport'),
|
|
|
- exact=True):
|
|
|
- logging.warning('%s excluded: has OR %s changed fingerprint to ' +
|
|
|
- '%s?', entry['id'], entry['ipv6'], self._fpr)
|
|
|
- return False
|
|
|
- if not self.ipv4_addr_matches(entry['ipv4'], exact=True):
|
|
|
- logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
|
|
|
- self._fpr, entry['ipv4'], self.dirip)
|
|
|
- return False
|
|
|
- if not self.ipv4_dirport_matches(entry['dirport'], exact=True):
|
|
|
- logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
|
|
|
- '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
|
|
|
- self.dirip, self.dirport)
|
|
|
- return False
|
|
|
- if not self.ipv4_orport_matches(entry['orport'], exact=True):
|
|
|
- logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
|
|
|
- '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
|
|
|
- self.dirip, self.orport)
|
|
|
- return False
|
|
|
- if entry.has_key('ipv6') and self.has_ipv6():
|
|
|
- # if both entry and fallback have an ipv6 address, compare them
|
|
|
- if not self.ipv6_and_orport_matches(entry['ipv6_addr'],
|
|
|
- entry['ipv6_orport'],
|
|
|
- exact=True):
|
|
|
- logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
|
|
|
- 'to %s:%d?', self._fpr, entry['ipv6'],
|
|
|
- self.ipv6addr, self.ipv6orport)
|
|
|
- return False
|
|
|
- # if the fallback has an IPv6 address but the whitelist entry
|
|
|
- # doesn't, or vice versa, the whitelist entry doesn't match
|
|
|
- elif entry.has_key('ipv6') and not self.has_ipv6():
|
|
|
- logging.warning('%s excluded: has it lost its former IPv6 address %s?',
|
|
|
- self._fpr, entry['ipv6'])
|
|
|
- return False
|
|
|
- elif not entry.has_key('ipv6') and self.has_ipv6():
|
|
|
- logging.warning('%s excluded: has it gained an IPv6 address %s:%d?',
|
|
|
- self._fpr, self.ipv6addr, self.ipv6orport)
|
|
|
- return False
|
|
|
- return True
|
|
|
-
|
|
|
- def entry_matches_fuzzy(self, entry):
|
|
|
- """ Is entry a fuzzy match for this fallback?
|
|
|
- A fallback is a fuzzy match for entry if at least one of these keys
|
|
|
- in entry matches:
|
|
|
- id
|
|
|
- ipv4
|
|
|
- ipv6 (if present in both the fallback and whitelist)
|
|
|
- The ports and nickname are ignored. Missing or extra ipv6 addresses
|
|
|
- are ignored.
|
|
|
-
|
|
|
- Doesn't log any warning messages. """
|
|
|
- if self.id_matches(entry['id'], exact=False):
|
|
|
- return True
|
|
|
- if self.ipv4_addr_matches(entry['ipv4'], exact=False):
|
|
|
- return True
|
|
|
- if entry.has_key('ipv6') and self.has_ipv6():
|
|
|
- # if both entry and fallback have an ipv6 address, compare them
|
|
|
- if self.ipv6_addr_matches(entry['ipv6_addr'], exact=False):
|
|
|
- return True
|
|
|
- return False
|
|
|
-
|
|
|
- def is_in_whitelist(self, relaylist, exact=False):
|
|
|
- """ If exact is True (existing fallback list), check if this fallback is
|
|
|
- an exact match for any whitelist entry, using entry_matches_exact().
|
|
|
-
|
|
|
- If exact is False (new fallback whitelist), check if this fallback is
|
|
|
- a fuzzy match for any whitelist entry, using entry_matches_fuzzy(). """
|
|
|
- for entry in relaylist:
|
|
|
- if exact:
|
|
|
- if self.entry_matches_exact(entry):
|
|
|
- return True
|
|
|
- else:
|
|
|
- if self.entry_matches_fuzzy(entry):
|
|
|
- return True
|
|
|
- return False
|
|
|
-
|
|
|
- def cw_to_bw_factor(self):
|
|
|
- # any relays with a missing or zero consensus weight are not candidates
|
|
|
- # any relays with a missing advertised bandwidth have it set to zero
|
|
|
- return self._data['advertised_bandwidth'] / self._data['consensus_weight']
|
|
|
-
|
|
|
- # since advertised_bandwidth is reported by the relay, it can be gamed
|
|
|
- # to avoid this, use the median consensus weight to bandwidth factor to
|
|
|
- # estimate this relay's measured bandwidth, and make that the upper limit
|
|
|
- def measured_bandwidth(self, median_cw_to_bw_factor):
|
|
|
- cw_to_bw= median_cw_to_bw_factor
|
|
|
- # Reduce exit bandwidth to make sure we're not overloading them
|
|
|
- if self.is_exit():
|
|
|
- cw_to_bw *= EXIT_BANDWIDTH_FRACTION
|
|
|
- measured_bandwidth = self._data['consensus_weight'] * cw_to_bw
|
|
|
- if self._data['advertised_bandwidth'] != 0:
|
|
|
- # limit advertised bandwidth (if available) to measured bandwidth
|
|
|
- return min(measured_bandwidth, self._data['advertised_bandwidth'])
|
|
|
- else:
|
|
|
- return measured_bandwidth
|
|
|
-
|
|
|
- def set_measured_bandwidth(self, median_cw_to_bw_factor):
|
|
|
- self._data['measured_bandwidth'] = self.measured_bandwidth(
|
|
|
- median_cw_to_bw_factor)
|
|
|
-
|
|
|
- def is_exit(self):
|
|
|
- return 'Exit' in self._data['flags']
|
|
|
-
|
|
|
- def is_guard(self):
|
|
|
- return 'Guard' in self._data['flags']
|
|
|
-
|
|
|
- def is_running(self):
|
|
|
- return 'Running' in self._data['flags']
|
|
|
-
|
|
|
- # does this fallback have an IPv6 address and orport?
|
|
|
- def has_ipv6(self):
|
|
|
- return self.ipv6addr is not None and self.ipv6orport is not None
|
|
|
-
|
|
|
- # strip leading and trailing brackets from an IPv6 address
|
|
|
- # safe to use on non-bracketed IPv6 and on IPv4 addresses
|
|
|
- # also convert to unicode, and make None appear as ''
|
|
|
- @staticmethod
|
|
|
- def strip_ipv6_brackets(ip):
|
|
|
- if ip is None:
|
|
|
- return unicode('')
|
|
|
- if len(ip) < 2:
|
|
|
- return unicode(ip)
|
|
|
- if ip[0] == '[' and ip[-1] == ']':
|
|
|
- return unicode(ip[1:-1])
|
|
|
- return unicode(ip)
|
|
|
-
|
|
|
- # are ip_a and ip_b in the same netblock?
|
|
|
- # mask_bits is the size of the netblock
|
|
|
- # takes both IPv4 and IPv6 addresses
|
|
|
- # the versions of ip_a and ip_b must be the same
|
|
|
- # the mask must be valid for the IP version
|
|
|
- @staticmethod
|
|
|
- def netblocks_equal(ip_a, ip_b, mask_bits):
|
|
|
- if ip_a is None or ip_b is None:
|
|
|
- return False
|
|
|
- ip_a = Candidate.strip_ipv6_brackets(ip_a)
|
|
|
- ip_b = Candidate.strip_ipv6_brackets(ip_b)
|
|
|
- a = ipaddress.ip_address(ip_a)
|
|
|
- b = ipaddress.ip_address(ip_b)
|
|
|
- if a.version != b.version:
|
|
|
- raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b))
|
|
|
- if mask_bits > a.max_prefixlen:
|
|
|
- logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
|
|
|
- mask_bits = a.max_prefixlen
|
|
|
- if mask_bits < 0:
|
|
|
- logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
|
|
|
- mask_bits = 0
|
|
|
- a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False)
|
|
|
- return b in a_net
|
|
|
-
|
|
|
- # is this fallback's IPv4 address (dirip) in the same netblock as other's
|
|
|
- # IPv4 address?
|
|
|
- # mask_bits is the size of the netblock
|
|
|
- def ipv4_netblocks_equal(self, other, mask_bits):
|
|
|
- return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits)
|
|
|
-
|
|
|
- # is this fallback's IPv6 address (ipv6addr) in the same netblock as
|
|
|
- # other's IPv6 address?
|
|
|
- # Returns False if either fallback has no IPv6 address
|
|
|
- # mask_bits is the size of the netblock
|
|
|
- def ipv6_netblocks_equal(self, other, mask_bits):
|
|
|
- if not self.has_ipv6() or not other.has_ipv6():
|
|
|
- return False
|
|
|
- return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits)
|
|
|
-
|
|
|
- # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort?
|
|
|
- def dirport_equal(self, other):
|
|
|
- return self.dirport == other.dirport
|
|
|
-
|
|
|
- # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort?
|
|
|
- def ipv4_orport_equal(self, other):
|
|
|
- return self.orport == other.orport
|
|
|
-
|
|
|
- # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort?
|
|
|
- # Returns False if either fallback has no IPv6 address
|
|
|
- def ipv6_orport_equal(self, other):
|
|
|
- if not self.has_ipv6() or not other.has_ipv6():
|
|
|
- return False
|
|
|
- return self.ipv6orport == other.ipv6orport
|
|
|
-
|
|
|
- # does this fallback have the same DirPort, IPv4 ORPort, or
|
|
|
- # IPv6 ORPort as other?
|
|
|
- # Ignores IPv6 ORPort if either fallback has no IPv6 address
|
|
|
- def port_equal(self, other):
|
|
|
- return (self.dirport_equal(other) or self.ipv4_orport_equal(other)
|
|
|
- or self.ipv6_orport_equal(other))
|
|
|
-
|
|
|
- # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present)
|
|
|
- def port_list(self):
|
|
|
- ports = [self.dirport, self.orport]
|
|
|
- if self.has_ipv6() and not self.ipv6orport in ports:
|
|
|
- ports.append(self.ipv6orport)
|
|
|
- return ports
|
|
|
-
|
|
|
- # does this fallback share a port with other, regardless of whether the
|
|
|
- # port types match?
|
|
|
- # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80,
|
|
|
- # return True
|
|
|
- def port_shared(self, other):
|
|
|
- for p in self.port_list():
|
|
|
- if p in other.port_list():
|
|
|
- return True
|
|
|
- return False
|
|
|
-
|
|
|
- # log how long it takes to download a consensus from dirip:dirport
|
|
|
- # returns True if the download failed, False if it succeeded within max_time
|
|
|
- @staticmethod
|
|
|
- def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
|
|
|
- max_time):
|
|
|
- download_failed = False
|
|
|
- # some directory mirrors respond to requests in ways that hang python
|
|
|
- # sockets, which is why we log this line here
|
|
|
- logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
|
|
|
- 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
|
|
|
- nickname, dirip, dirport, fingerprint)
|
|
|
- # there appears to be about 1 second of overhead when comparing stem's
|
|
|
- # internal trace time and the elapsed time calculated here
|
|
|
- TIMEOUT_SLOP = 1.0
|
|
|
- start = datetime.datetime.utcnow()
|
|
|
- try:
|
|
|
- consensus = get_consensus(
|
|
|
- endpoints = [(dirip, dirport)],
|
|
|
- timeout = (max_time + TIMEOUT_SLOP),
|
|
|
- validate = True,
|
|
|
- retries = 0,
|
|
|
- fall_back_to_authority = False,
|
|
|
- document_handler = DocumentHandler.BARE_DOCUMENT,
|
|
|
- microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
|
|
|
- ).run()[0]
|
|
|
- end = datetime.datetime.utcnow()
|
|
|
- time_since_expiry = (end - consensus.valid_until).total_seconds()
|
|
|
- time_until_valid = (consensus.valid_after - end).total_seconds()
|
|
|
- except Exception, stem_error:
|
|
|
- end = datetime.datetime.utcnow()
|
|
|
- log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
|
|
|
- stem_error)
|
|
|
- status = 'error: "%s"' % (stem_error)
|
|
|
- level = logging.WARNING
|
|
|
- download_failed = True
|
|
|
- elapsed = (end - start).total_seconds()
|
|
|
- if download_failed:
|
|
|
- # keep the error failure status, and avoid using the variables
|
|
|
- pass
|
|
|
- elif elapsed > max_time:
|
|
|
- status = 'too slow'
|
|
|
- level = logging.WARNING
|
|
|
- download_failed = True
|
|
|
- elif (time_since_expiry > 0):
|
|
|
- status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
|
|
|
- if time_since_expiry <= REASONABLY_LIVE_TIME:
|
|
|
- status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME)
|
|
|
- level = logging.INFO
|
|
|
- else:
|
|
|
- status += ', invalid'
|
|
|
- level = logging.WARNING
|
|
|
- download_failed = True
|
|
|
- elif (time_until_valid > 0):
|
|
|
- status = 'future consensus, valid in %ds'%(int(time_until_valid))
|
|
|
- if time_until_valid <= REASONABLY_LIVE_TIME:
|
|
|
- status += ', tolerating up to %ds'%(REASONABLY_LIVE_TIME)
|
|
|
- level = logging.INFO
|
|
|
- else:
|
|
|
- status += ', invalid'
|
|
|
- level = logging.WARNING
|
|
|
- download_failed = True
|
|
|
- else:
|
|
|
- status = 'ok'
|
|
|
- level = logging.DEBUG
|
|
|
- logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
|
|
|
- 'max download time %0.1fs.', elapsed, status, nickname,
|
|
|
- dirip, dirport, fingerprint, max_time)
|
|
|
- return download_failed
|
|
|
-
|
|
|
- # does this fallback download the consensus fast enough?
|
|
|
- def check_fallback_download_consensus(self):
|
|
|
- # include the relay if we're not doing a check, or we can't check (IPv6)
|
|
|
- ipv4_failed = False
|
|
|
- ipv6_failed = False
|
|
|
- if PERFORM_IPV4_DIRPORT_CHECKS:
|
|
|
- ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
|
|
|
- self.dirport,
|
|
|
- self._data['nickname'],
|
|
|
- self._fpr,
|
|
|
- CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
- if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
- # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
|
|
|
- ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
|
|
|
- self.dirport,
|
|
|
- self._data['nickname'],
|
|
|
- self._fpr,
|
|
|
- CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
- return ((not ipv4_failed) and (not ipv6_failed))
|
|
|
-
|
|
|
- # if this fallback has not passed a download check, try it again,
|
|
|
- # and record the result, available in get_fallback_download_consensus
|
|
|
- def try_fallback_download_consensus(self):
|
|
|
- if not self.get_fallback_download_consensus():
|
|
|
- self._data['download_check'] = self.check_fallback_download_consensus()
|
|
|
-
|
|
|
- # did this fallback pass the download check?
|
|
|
- def get_fallback_download_consensus(self):
|
|
|
- # if we're not performing checks, return True
|
|
|
- if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
- return True
|
|
|
- # if we are performing checks, but haven't done one, return False
|
|
|
- if not self._data.has_key('download_check'):
|
|
|
- return False
|
|
|
- return self._data['download_check']
|
|
|
-
|
|
|
- # output an optional header comment and info for this fallback
|
|
|
- # try_fallback_download_consensus before calling this
|
|
|
- def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
|
|
|
- s = ''
|
|
|
- if OUTPUT_COMMENTS:
|
|
|
- s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
|
|
|
- # if the download speed is ok, output a C string
|
|
|
- # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
|
|
|
- if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
|
|
|
- s += self.fallbackdir_info(self.get_fallback_download_consensus())
|
|
|
- return s
|
|
|
-
|
|
|
- # output a header comment for this fallback
|
|
|
- def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
|
|
|
- # /*
|
|
|
- # nickname
|
|
|
- # flags
|
|
|
- # adjusted bandwidth, consensus weight
|
|
|
- # [contact]
|
|
|
- # [identical contact counts]
|
|
|
- # */
|
|
|
- # Multiline C comment
|
|
|
- s = '/*'
|
|
|
- s += '\n'
|
|
|
- s += cleanse_c_multiline_comment(self._data['nickname'])
|
|
|
- s += '\n'
|
|
|
- s += 'Flags: '
|
|
|
- s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
|
|
|
- s += '\n'
|
|
|
- # this is an adjusted bandwidth, see calculate_measured_bandwidth()
|
|
|
- bandwidth = self._data['measured_bandwidth']
|
|
|
- weight = self._data['consensus_weight']
|
|
|
- s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
|
|
|
- bandwidth/(1024.0*1024.0),
|
|
|
- weight)
|
|
|
- s += '\n'
|
|
|
- if self._data['contact'] is not None:
|
|
|
- s += cleanse_c_multiline_comment(self._data['contact'])
|
|
|
- if CONTACT_COUNT:
|
|
|
- fallback_count = len([f for f in fallbacks
|
|
|
- if f._data['contact'] == self._data['contact']])
|
|
|
- if fallback_count > 1:
|
|
|
- s += '\n'
|
|
|
- s += '%d identical contacts listed' % (fallback_count)
|
|
|
-
|
|
|
- # output the fallback info C string for this fallback
|
|
|
- # this is the text that would go after FallbackDir in a torrc
|
|
|
- # if this relay failed the download test and we OUTPUT_COMMENTS,
|
|
|
- # comment-out the returned string
|
|
|
- def fallbackdir_info(self, dl_speed_ok):
|
|
|
- # "address:dirport orport=port id=fingerprint"
|
|
|
- # (insert additional madatory fields here)
|
|
|
- # "[ipv6=addr:orport]"
|
|
|
- # (insert additional optional fields here)
|
|
|
- # /* nickname=name */
|
|
|
- # /* extrainfo={0,1} */
|
|
|
- # (insert additional comment fields here)
|
|
|
- # /* ===== */
|
|
|
- # ,
|
|
|
- #
|
|
|
- # Do we want a C string, or a commented-out string?
|
|
|
- c_string = dl_speed_ok
|
|
|
- comment_string = not dl_speed_ok and OUTPUT_COMMENTS
|
|
|
- # If we don't want either kind of string, bail
|
|
|
- if not c_string and not comment_string:
|
|
|
- return ''
|
|
|
- s = ''
|
|
|
- # Comment out the fallback directory entry if it's too slow
|
|
|
- # See the debug output for which address and port is failing
|
|
|
- if comment_string:
|
|
|
- s += '/* Consensus download failed or was too slow:\n'
|
|
|
- # Multi-Line C string with trailing comma (part of a string list)
|
|
|
- # This makes it easier to diff the file, and remove IPv6 lines using grep
|
|
|
- # Integers don't need escaping
|
|
|
- s += '"%s orport=%d id=%s"'%(
|
|
|
- cleanse_c_string(self._data['dir_address']),
|
|
|
- self.orport,
|
|
|
- cleanse_c_string(self._fpr))
|
|
|
- s += '\n'
|
|
|
- # (insert additional madatory fields here)
|
|
|
- if self.has_ipv6():
|
|
|
- s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
|
|
|
- s += '\n'
|
|
|
- # (insert additional optional fields here)
|
|
|
- if not comment_string:
|
|
|
- s += '/* '
|
|
|
- s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
|
|
|
- if not comment_string:
|
|
|
- s += ' */'
|
|
|
- s += '\n'
|
|
|
- # if we know that the fallback is an extrainfo cache, flag it
|
|
|
- # and if we don't know, assume it is not
|
|
|
- if not comment_string:
|
|
|
- s += '/* '
|
|
|
- s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
|
|
|
- if not comment_string:
|
|
|
- s += ' */'
|
|
|
- s += '\n'
|
|
|
- # (insert additional comment fields here)
|
|
|
- # The terminator and comma must be the last line in each fallback entry
|
|
|
- if not comment_string:
|
|
|
- s += '/* '
|
|
|
- s += SECTION_SEPARATOR_BASE
|
|
|
- if not comment_string:
|
|
|
- s += ' */'
|
|
|
- s += '\n'
|
|
|
- s += ','
|
|
|
- if comment_string:
|
|
|
- s += '\n'
|
|
|
- s += '*/'
|
|
|
- return s
|
|
|
-
|
|
|
-## Fallback Candidate List Class
|
|
|
-
|
|
|
-class CandidateList(dict):
|
|
|
- def __init__(self):
|
|
|
- pass
|
|
|
-
|
|
|
- def _add_relay(self, details):
|
|
|
- if not 'dir_address' in details: return
|
|
|
- c = Candidate(details)
|
|
|
- self[ c.get_fingerprint() ] = c
|
|
|
-
|
|
|
- def _add_uptime(self, uptime):
|
|
|
- try:
|
|
|
- fpr = uptime['fingerprint']
|
|
|
- except KeyError:
|
|
|
- raise Exception("Document has no fingerprint field.")
|
|
|
-
|
|
|
- try:
|
|
|
- c = self[fpr]
|
|
|
- except KeyError:
|
|
|
- logging.debug('Got unknown relay %s in uptime document.'%(fpr,))
|
|
|
- return
|
|
|
-
|
|
|
- c.add_uptime(uptime)
|
|
|
-
|
|
|
- def _add_details(self):
|
|
|
- logging.debug('Loading details document.')
|
|
|
- d = fetch('details',
|
|
|
- fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
|
|
|
- 'consensus_weight,advertised_bandwidth,or_addresses,' +
|
|
|
- 'dir_address,recommended_version,flags,effective_family,' +
|
|
|
- 'platform'))
|
|
|
- logging.debug('Loading details document done.')
|
|
|
-
|
|
|
- if not 'relays' in d: raise Exception("No relays found in document.")
|
|
|
-
|
|
|
- for r in d['relays']: self._add_relay(r)
|
|
|
-
|
|
|
- def _add_uptimes(self):
|
|
|
- logging.debug('Loading uptime document.')
|
|
|
- d = fetch('uptime')
|
|
|
- logging.debug('Loading uptime document done.')
|
|
|
-
|
|
|
- if not 'relays' in d: raise Exception("No relays found in document.")
|
|
|
- for r in d['relays']: self._add_uptime(r)
|
|
|
-
|
|
|
- def add_relays(self):
|
|
|
- self._add_details()
|
|
|
- self._add_uptimes()
|
|
|
-
|
|
|
- def count_guards(self):
|
|
|
- guard_count = 0
|
|
|
- for fpr in self.keys():
|
|
|
- if self[fpr].is_guard():
|
|
|
- guard_count += 1
|
|
|
- return guard_count
|
|
|
-
|
|
|
- # Find fallbacks that fit the uptime, stability, and flags criteria,
|
|
|
- # and make an array of them in self.fallbacks
|
|
|
- def compute_fallbacks(self):
|
|
|
- self.fallbacks = map(lambda x: self[x],
|
|
|
- filter(lambda x: self[x].is_candidate(),
|
|
|
- self.keys()))
|
|
|
-
|
|
|
- # sort fallbacks by their consensus weight to advertised bandwidth factor,
|
|
|
- # lowest to highest
|
|
|
- # used to find the median cw_to_bw_factor()
|
|
|
- def sort_fallbacks_by_cw_to_bw_factor(self):
|
|
|
- self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor())
|
|
|
-
|
|
|
- # sort fallbacks by their measured bandwidth, highest to lowest
|
|
|
- # calculate_measured_bandwidth before calling this
|
|
|
- # this is useful for reviewing candidates in priority order
|
|
|
- def sort_fallbacks_by_measured_bandwidth(self):
|
|
|
- self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
|
|
|
- reverse=True)
|
|
|
-
|
|
|
- # sort fallbacks by the data field data_field, lowest to highest
|
|
|
- def sort_fallbacks_by(self, data_field):
|
|
|
- self.fallbacks.sort(key=lambda f: f._data[data_field])
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def load_relaylist(file_obj):
|
|
|
- """ Read each line in the file, and parse it like a FallbackDir line:
|
|
|
- an IPv4 address and optional port:
|
|
|
- <IPv4 address>:<port>
|
|
|
- which are parsed into dictionary entries:
|
|
|
- ipv4=<IPv4 address>
|
|
|
- dirport=<port>
|
|
|
- followed by a series of key=value entries:
|
|
|
- orport=<port>
|
|
|
- id=<fingerprint>
|
|
|
- ipv6=<IPv6 address>:<IPv6 orport>
|
|
|
- each line's key/value pairs are placed in a dictonary,
|
|
|
- (of string -> string key/value pairs),
|
|
|
- and these dictionaries are placed in an array.
|
|
|
- comments start with # and are ignored. """
|
|
|
- file_data = file_obj['data']
|
|
|
- file_name = file_obj['name']
|
|
|
- relaylist = []
|
|
|
- if file_data is None:
|
|
|
- return relaylist
|
|
|
- for line in file_data.split('\n'):
|
|
|
- relay_entry = {}
|
|
|
- # ignore comments
|
|
|
- line_comment_split = line.split('#')
|
|
|
- line = line_comment_split[0]
|
|
|
- # cleanup whitespace
|
|
|
- line = cleanse_whitespace(line)
|
|
|
- line = line.strip()
|
|
|
- if len(line) == 0:
|
|
|
- continue
|
|
|
- for item in line.split(' '):
|
|
|
- item = item.strip()
|
|
|
- if len(item) == 0:
|
|
|
- continue
|
|
|
- key_value_split = item.split('=')
|
|
|
- kvl = len(key_value_split)
|
|
|
- if kvl < 1 or kvl > 2:
|
|
|
- print '#error Bad %s item: %s, format is key=value.'%(
|
|
|
- file_name, item)
|
|
|
- if kvl == 1:
|
|
|
- # assume that entries without a key are the ipv4 address,
|
|
|
- # perhaps with a dirport
|
|
|
- ipv4_maybe_dirport = key_value_split[0]
|
|
|
- ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':')
|
|
|
- dirl = len(ipv4_maybe_dirport_split)
|
|
|
- if dirl < 1 or dirl > 2:
|
|
|
- print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%(
|
|
|
- file_name, item)
|
|
|
- if dirl >= 1:
|
|
|
- relay_entry['ipv4'] = ipv4_maybe_dirport_split[0]
|
|
|
- if dirl == 2:
|
|
|
- relay_entry['dirport'] = ipv4_maybe_dirport_split[1]
|
|
|
- elif kvl == 2:
|
|
|
- relay_entry[key_value_split[0]] = key_value_split[1]
|
|
|
- # split ipv6 addresses and orports
|
|
|
- if key_value_split[0] == 'ipv6':
|
|
|
- ipv6_orport_split = key_value_split[1].rsplit(':', 1)
|
|
|
- ipv6l = len(ipv6_orport_split)
|
|
|
- if ipv6l != 2:
|
|
|
- print '#error Bad %s IPv6 item: %s, format is [ipv6]:orport.'%(
|
|
|
- file_name, item)
|
|
|
- relay_entry['ipv6_addr'] = ipv6_orport_split[0]
|
|
|
- relay_entry['ipv6_orport'] = ipv6_orport_split[1]
|
|
|
- relaylist.append(relay_entry)
|
|
|
- return relaylist
|
|
|
-
|
|
|
- def apply_filter_lists(self, whitelist_obj, exact=False):
|
|
|
- """ Apply the fallback whitelist_obj to this fallback list,
|
|
|
- passing exact to is_in_whitelist(). """
|
|
|
- excluded_count = 0
|
|
|
- list_type = 'whitelist'
|
|
|
- if whitelist_obj['check_existing']:
|
|
|
- list_type = 'fallback list'
|
|
|
-
|
|
|
- logging.debug('Applying {}'.format(list_type))
|
|
|
- # parse the whitelist
|
|
|
- whitelist = self.load_relaylist(whitelist_obj)
|
|
|
- filtered_fallbacks = []
|
|
|
- for f in self.fallbacks:
|
|
|
- in_whitelist = f.is_in_whitelist(whitelist, exact=exact)
|
|
|
- if in_whitelist:
|
|
|
- # include
|
|
|
- filtered_fallbacks.append(f)
|
|
|
- elif INCLUDE_UNLISTED_ENTRIES:
|
|
|
- # include
|
|
|
- filtered_fallbacks.append(f)
|
|
|
- else:
|
|
|
- # exclude
|
|
|
- excluded_count += 1
|
|
|
- log_excluded('Excluding %s: not in %s.',
|
|
|
- f._fpr, list_type)
|
|
|
- self.fallbacks = filtered_fallbacks
|
|
|
- return excluded_count
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def summarise_filters(initial_count, excluded_count, check_existing):
|
|
|
- list_type = 'Whitelist'
|
|
|
- if check_existing:
|
|
|
- list_type = 'Fallback list'
|
|
|
-
|
|
|
- return '/* %s excluded %d of %d candidates. */'%(list_type,
|
|
|
- excluded_count, initial_count)
|
|
|
-
|
|
|
- # calculate each fallback's measured bandwidth based on the median
|
|
|
- # consensus weight to advertised bandwidth ratio
|
|
|
- def calculate_measured_bandwidth(self):
|
|
|
- self.sort_fallbacks_by_cw_to_bw_factor()
|
|
|
- median_fallback = self.fallback_median(True)
|
|
|
- if median_fallback is not None:
|
|
|
- median_cw_to_bw_factor = median_fallback.cw_to_bw_factor()
|
|
|
- else:
|
|
|
- # this will never be used, because there are no fallbacks
|
|
|
- median_cw_to_bw_factor = None
|
|
|
- for f in self.fallbacks:
|
|
|
- f.set_measured_bandwidth(median_cw_to_bw_factor)
|
|
|
-
|
|
|
- # remove relays with low measured bandwidth from the fallback list
|
|
|
- # calculate_measured_bandwidth for each relay before calling this
|
|
|
- def remove_low_bandwidth_relays(self):
|
|
|
- if MIN_BANDWIDTH is None:
|
|
|
- return
|
|
|
- above_min_bw_fallbacks = []
|
|
|
- for f in self.fallbacks:
|
|
|
- if f._data['measured_bandwidth'] >= MIN_BANDWIDTH:
|
|
|
- above_min_bw_fallbacks.append(f)
|
|
|
- else:
|
|
|
- # the bandwidth we log here is limited by the relay's consensus weight
|
|
|
- # as well as its adverttised bandwidth. See set_measured_bandwidth
|
|
|
- # for details
|
|
|
- log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
|
|
|
- 'must be at least %.1fMByte/s', f._fpr,
|
|
|
- f._data['measured_bandwidth']/(1024.0*1024.0),
|
|
|
- MIN_BANDWIDTH/(1024.0*1024.0))
|
|
|
- self.fallbacks = above_min_bw_fallbacks
|
|
|
-
|
|
|
- # the minimum fallback in the list
|
|
|
- # call one of the sort_fallbacks_* functions before calling this
|
|
|
- def fallback_min(self):
|
|
|
- if len(self.fallbacks) > 0:
|
|
|
- return self.fallbacks[-1]
|
|
|
- else:
|
|
|
- return None
|
|
|
-
|
|
|
- # the median fallback in the list
|
|
|
- # call one of the sort_fallbacks_* functions before calling this
|
|
|
- def fallback_median(self, require_advertised_bandwidth):
|
|
|
- # use the low-median when there are an evan number of fallbacks,
|
|
|
- # for consistency with the bandwidth authorities
|
|
|
- if len(self.fallbacks) > 0:
|
|
|
- median_position = (len(self.fallbacks) - 1) / 2
|
|
|
- if not require_advertised_bandwidth:
|
|
|
- return self.fallbacks[median_position]
|
|
|
- # if we need advertised_bandwidth but this relay doesn't have it,
|
|
|
- # move to a fallback with greater consensus weight until we find one
|
|
|
- while not self.fallbacks[median_position]._data['advertised_bandwidth']:
|
|
|
- median_position += 1
|
|
|
- if median_position >= len(self.fallbacks):
|
|
|
- return None
|
|
|
- return self.fallbacks[median_position]
|
|
|
- else:
|
|
|
- return None
|
|
|
-
|
|
|
- # the maximum fallback in the list
|
|
|
- # call one of the sort_fallbacks_* functions before calling this
|
|
|
- def fallback_max(self):
|
|
|
- if len(self.fallbacks) > 0:
|
|
|
- return self.fallbacks[0]
|
|
|
- else:
|
|
|
- return None
|
|
|
-
|
|
|
- # return a new bag suitable for storing attributes
|
|
|
- @staticmethod
|
|
|
- def attribute_new():
|
|
|
- return dict()
|
|
|
-
|
|
|
- # get the count of attribute in attribute_bag
|
|
|
- # if attribute is None or the empty string, return 0
|
|
|
- @staticmethod
|
|
|
- def attribute_count(attribute, attribute_bag):
|
|
|
- if attribute is None or attribute == '':
|
|
|
- return 0
|
|
|
- if attribute not in attribute_bag:
|
|
|
- return 0
|
|
|
- return attribute_bag[attribute]
|
|
|
-
|
|
|
- # does attribute_bag contain more than max_count instances of attribute?
|
|
|
- # if so, return False
|
|
|
- # if not, return True
|
|
|
- # if attribute is None or the empty string, or max_count is invalid,
|
|
|
- # always return True
|
|
|
- @staticmethod
|
|
|
- def attribute_allow(attribute, attribute_bag, max_count=1):
|
|
|
- if attribute is None or attribute == '' or max_count <= 0:
|
|
|
- return True
|
|
|
- elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
|
|
|
- return False
|
|
|
- else:
|
|
|
- return True
|
|
|
-
|
|
|
- # add attribute to attribute_bag, incrementing the count if it is already
|
|
|
- # present
|
|
|
- # if attribute is None or the empty string, or count is invalid,
|
|
|
- # do nothing
|
|
|
- @staticmethod
|
|
|
- def attribute_add(attribute, attribute_bag, count=1):
|
|
|
- if attribute is None or attribute == '' or count <= 0:
|
|
|
- pass
|
|
|
- attribute_bag.setdefault(attribute, 0)
|
|
|
- attribute_bag[attribute] += count
|
|
|
-
|
|
|
- # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
|
|
|
- # and per IPv6 address
|
|
|
- # there is only one IPv4 address on each fallback: the IPv4 DirPort address
|
|
|
- # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
|
|
|
- # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
|
|
|
- # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
|
|
|
- # (clients only use the IPv6 ORPort)
|
|
|
- # if there is no IPv6 address, only the IPv4 address is checked
|
|
|
- # return the number of candidates we excluded
|
|
|
- def limit_fallbacks_same_ip(self):
|
|
|
- ip_limit_fallbacks = []
|
|
|
- ip_list = CandidateList.attribute_new()
|
|
|
- for f in self.fallbacks:
|
|
|
- if (CandidateList.attribute_allow(f.dirip, ip_list,
|
|
|
- MAX_FALLBACKS_PER_IPV4)
|
|
|
- and CandidateList.attribute_allow(f.ipv6addr, ip_list,
|
|
|
- MAX_FALLBACKS_PER_IPV6)):
|
|
|
- ip_limit_fallbacks.append(f)
|
|
|
- CandidateList.attribute_add(f.dirip, ip_list)
|
|
|
- if f.has_ipv6():
|
|
|
- CandidateList.attribute_add(f.ipv6addr, ip_list)
|
|
|
- elif not CandidateList.attribute_allow(f.dirip, ip_list,
|
|
|
- MAX_FALLBACKS_PER_IPV4):
|
|
|
- log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
|
|
|
- %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
|
|
|
- f.dirip))
|
|
|
- elif (f.has_ipv6() and
|
|
|
- not CandidateList.attribute_allow(f.ipv6addr, ip_list,
|
|
|
- MAX_FALLBACKS_PER_IPV6)):
|
|
|
- log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
|
|
|
- %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
|
|
|
- ip_list),
|
|
|
- f.ipv6addr))
|
|
|
- original_count = len(self.fallbacks)
|
|
|
- self.fallbacks = ip_limit_fallbacks
|
|
|
- return original_count - len(self.fallbacks)
|
|
|
-
|
|
|
- # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
|
|
|
- # ContactInfo
|
|
|
- # if there is no ContactInfo, allow the fallback
|
|
|
- # this check can be gamed by providing no ContactInfo, or by setting the
|
|
|
- # ContactInfo to match another fallback
|
|
|
- # However, given the likelihood that relays with the same ContactInfo will
|
|
|
- # go down at similar times, its usefulness outweighs the risk
|
|
|
- def limit_fallbacks_same_contact(self):
|
|
|
- contact_limit_fallbacks = []
|
|
|
- contact_list = CandidateList.attribute_new()
|
|
|
- for f in self.fallbacks:
|
|
|
- if CandidateList.attribute_allow(f._data['contact'], contact_list,
|
|
|
- MAX_FALLBACKS_PER_CONTACT):
|
|
|
- contact_limit_fallbacks.append(f)
|
|
|
- CandidateList.attribute_add(f._data['contact'], contact_list)
|
|
|
- else:
|
|
|
- log_excluded(
|
|
|
- 'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
|
|
|
- %(f._fpr, CandidateList.attribute_count(f._data['contact'],
|
|
|
- contact_list),
|
|
|
- f._data['contact']))
|
|
|
- original_count = len(self.fallbacks)
|
|
|
- self.fallbacks = contact_limit_fallbacks
|
|
|
- return original_count - len(self.fallbacks)
|
|
|
-
|
|
|
- # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
|
|
|
- # family
|
|
|
- # if there is no family, allow the fallback
|
|
|
- # we use effective family, which ensures mutual family declarations
|
|
|
- # but the check can be gamed by not declaring a family at all
|
|
|
- # if any indirect families exist, the result depends on the order in which
|
|
|
- # fallbacks are sorted in the list
|
|
|
- def limit_fallbacks_same_family(self):
|
|
|
- family_limit_fallbacks = []
|
|
|
- fingerprint_list = CandidateList.attribute_new()
|
|
|
- for f in self.fallbacks:
|
|
|
- if CandidateList.attribute_allow(f._fpr, fingerprint_list,
|
|
|
- MAX_FALLBACKS_PER_FAMILY):
|
|
|
- family_limit_fallbacks.append(f)
|
|
|
- CandidateList.attribute_add(f._fpr, fingerprint_list)
|
|
|
- for family_fingerprint in f._data['effective_family']:
|
|
|
- CandidateList.attribute_add(family_fingerprint, fingerprint_list)
|
|
|
- else:
|
|
|
- # we already have a fallback with this fallback in its effective
|
|
|
- # family
|
|
|
- log_excluded(
|
|
|
- 'Eliminated %s: already have %d fallback(s) in effective family'
|
|
|
- %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
|
|
|
- original_count = len(self.fallbacks)
|
|
|
- self.fallbacks = family_limit_fallbacks
|
|
|
- return original_count - len(self.fallbacks)
|
|
|
-
|
|
|
- # try once to get the descriptors for fingerprint_list using stem
|
|
|
- # returns an empty list on exception
|
|
|
- @staticmethod
|
|
|
- def get_fallback_descriptors_once(fingerprint_list):
|
|
|
- desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
|
|
|
- return desc_list
|
|
|
-
|
|
|
- # try up to max_retries times to get the descriptors for fingerprint_list
|
|
|
- # using stem. Stops retrying when all descriptors have been retrieved.
|
|
|
- # returns a list containing the descriptors that were retrieved
|
|
|
- @staticmethod
|
|
|
- def get_fallback_descriptors(fingerprint_list, max_retries=5):
|
|
|
- # we can't use stem's retries=, because we want to support more than 96
|
|
|
- # descriptors
|
|
|
- #
|
|
|
- # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
|
|
|
- max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
|
|
|
- remaining_list = fingerprint_list
|
|
|
- desc_list = []
|
|
|
- for _ in xrange(max_retries):
|
|
|
- if len(remaining_list) == 0:
|
|
|
- break
|
|
|
- new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
|
|
|
- for d in new_desc_list:
|
|
|
- try:
|
|
|
- remaining_list.remove(d.fingerprint)
|
|
|
- except ValueError:
|
|
|
- # warn and ignore if a directory mirror returned a bad descriptor
|
|
|
- logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
|
|
|
- d.fingerprint)
|
|
|
- continue
|
|
|
- desc_list.append(d)
|
|
|
- return desc_list
|
|
|
-
|
|
|
- # find the fallbacks that cache extra-info documents
|
|
|
- # Onionoo doesn't know this, so we have to use stem
|
|
|
- def mark_extra_info_caches(self):
|
|
|
- fingerprint_list = [ f._fpr for f in self.fallbacks ]
|
|
|
- logging.info("Downloading fallback descriptors to find extra-info caches")
|
|
|
- desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
|
|
|
- for d in desc_list:
|
|
|
- self[d.fingerprint]._extra_info_cache = d.extra_info_cache
|
|
|
- missing_descriptor_list = [ f._fpr for f in self.fallbacks
|
|
|
- if f._extra_info_cache is None ]
|
|
|
- for f in missing_descriptor_list:
|
|
|
- logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
|
|
|
-
|
|
|
- # try a download check on each fallback candidate in order
|
|
|
- # stop after max_count successful downloads
|
|
|
- # but don't remove any candidates from the array
|
|
|
- def try_download_consensus_checks(self, max_count):
|
|
|
- dl_ok_count = 0
|
|
|
- for f in self.fallbacks:
|
|
|
- f.try_fallback_download_consensus()
|
|
|
- if f.get_fallback_download_consensus():
|
|
|
- # this fallback downloaded a consensus ok
|
|
|
- dl_ok_count += 1
|
|
|
- if dl_ok_count >= max_count:
|
|
|
- # we have enough fallbacks
|
|
|
- return
|
|
|
-
|
|
|
- # put max_count successful candidates in the fallbacks array:
|
|
|
- # - perform download checks on each fallback candidate
|
|
|
- # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
|
|
|
- # - eliminate failed candidates
|
|
|
- # - if there are more than max_count candidates, eliminate lowest bandwidth
|
|
|
- # - if there are fewer than max_count candidates, leave only successful
|
|
|
- # Return the number of fallbacks that failed the consensus check
|
|
|
- def perform_download_consensus_checks(self, max_count):
|
|
|
- self.sort_fallbacks_by_measured_bandwidth()
|
|
|
- self.try_download_consensus_checks(max_count)
|
|
|
- if CONSENSUS_DOWNLOAD_RETRY:
|
|
|
- # try unsuccessful candidates again
|
|
|
- # we could end up with more than max_count successful candidates here
|
|
|
- self.try_download_consensus_checks(max_count)
|
|
|
- # now we have at least max_count successful candidates,
|
|
|
- # or we've tried them all
|
|
|
- original_count = len(self.fallbacks)
|
|
|
- self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
|
|
|
- self.fallbacks)
|
|
|
- # some of these failed the check, others skipped the check,
|
|
|
- # if we already had enough successful downloads
|
|
|
- failed_count = original_count - len(self.fallbacks)
|
|
|
- self.fallbacks = self.fallbacks[:max_count]
|
|
|
- return failed_count
|
|
|
-
|
|
|
- # return a string that describes a/b as a percentage
|
|
|
- @staticmethod
|
|
|
- def describe_percentage(a, b):
|
|
|
- if b != 0:
|
|
|
- return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b)
|
|
|
- else:
|
|
|
- # technically, 0/0 is undefined, but 0.0% is a sensible result
|
|
|
- return '%d/%d = %.0f%%'%(a, b, 0.0)
|
|
|
-
|
|
|
- # return a dictionary of lists of fallbacks by IPv4 netblock
|
|
|
- # the dictionary is keyed by the fingerprint of an arbitrary fallback
|
|
|
- # in each netblock
|
|
|
- # mask_bits is the size of the netblock
|
|
|
- def fallbacks_by_ipv4_netblock(self, mask_bits):
|
|
|
- netblocks = {}
|
|
|
- for f in self.fallbacks:
|
|
|
- found_netblock = False
|
|
|
- for b in netblocks.keys():
|
|
|
- # we found an existing netblock containing this fallback
|
|
|
- if f.ipv4_netblocks_equal(self[b], mask_bits):
|
|
|
- # add it to the list
|
|
|
- netblocks[b].append(f)
|
|
|
- found_netblock = True
|
|
|
- break
|
|
|
- # make a new netblock based on this fallback's fingerprint
|
|
|
- if not found_netblock:
|
|
|
- netblocks[f._fpr] = [f]
|
|
|
- return netblocks
|
|
|
-
|
|
|
- # return a dictionary of lists of fallbacks by IPv6 netblock
|
|
|
- # where mask_bits is the size of the netblock
|
|
|
- def fallbacks_by_ipv6_netblock(self, mask_bits):
|
|
|
- netblocks = {}
|
|
|
- for f in self.fallbacks:
|
|
|
- # skip fallbacks without IPv6 addresses
|
|
|
- if not f.has_ipv6():
|
|
|
- continue
|
|
|
- found_netblock = False
|
|
|
- for b in netblocks.keys():
|
|
|
- # we found an existing netblock containing this fallback
|
|
|
- if f.ipv6_netblocks_equal(self[b], mask_bits):
|
|
|
- # add it to the list
|
|
|
- netblocks[b].append(f)
|
|
|
- found_netblock = True
|
|
|
- break
|
|
|
- # make a new netblock based on this fallback's fingerprint
|
|
|
- if not found_netblock:
|
|
|
- netblocks[f._fpr] = [f]
|
|
|
- return netblocks
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks in each IPv4 netblock,
|
|
|
- # where mask_bits is the size of the netblock
|
|
|
- def describe_fallback_ipv4_netblock_mask(self, mask_bits):
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- shared_netblock_fallback_count = 0
|
|
|
- most_frequent_netblock = None
|
|
|
- netblocks = self.fallbacks_by_ipv4_netblock(mask_bits)
|
|
|
- for b in netblocks.keys():
|
|
|
- if len(netblocks[b]) > 1:
|
|
|
- # how many fallbacks are in a netblock with other fallbacks?
|
|
|
- shared_netblock_fallback_count += len(netblocks[b])
|
|
|
- # what's the netblock with the most fallbacks?
|
|
|
- if (most_frequent_netblock is None
|
|
|
- or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
|
|
|
- most_frequent_netblock = b
|
|
|
- logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits))
|
|
|
- for f in netblocks[b]:
|
|
|
- logging.debug('%s - %s', f.dirip, f._fpr)
|
|
|
- if most_frequent_netblock is not None:
|
|
|
- logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%(
|
|
|
- CandidateList.describe_percentage(
|
|
|
- len(netblocks[most_frequent_netblock]),
|
|
|
- fallback_count),
|
|
|
- mask_bits,
|
|
|
- self[most_frequent_netblock].dirip))
|
|
|
- if shared_netblock_fallback_count > 0:
|
|
|
- logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' +
|
|
|
- 'fallbacks')%(CandidateList.describe_percentage(
|
|
|
- shared_netblock_fallback_count,
|
|
|
- fallback_count),
|
|
|
- mask_bits))
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks in each IPv6 netblock,
|
|
|
- # where mask_bits is the size of the netblock
|
|
|
- def describe_fallback_ipv6_netblock_mask(self, mask_bits):
|
|
|
- fallback_count = len(self.fallbacks_with_ipv6())
|
|
|
- shared_netblock_fallback_count = 0
|
|
|
- most_frequent_netblock = None
|
|
|
- netblocks = self.fallbacks_by_ipv6_netblock(mask_bits)
|
|
|
- for b in netblocks.keys():
|
|
|
- if len(netblocks[b]) > 1:
|
|
|
- # how many fallbacks are in a netblock with other fallbacks?
|
|
|
- shared_netblock_fallback_count += len(netblocks[b])
|
|
|
- # what's the netblock with the most fallbacks?
|
|
|
- if (most_frequent_netblock is None
|
|
|
- or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
|
|
|
- most_frequent_netblock = b
|
|
|
- logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits))
|
|
|
- for f in netblocks[b]:
|
|
|
- logging.debug('%s - %s', f.ipv6addr, f._fpr)
|
|
|
- if most_frequent_netblock is not None:
|
|
|
- logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%(
|
|
|
- CandidateList.describe_percentage(
|
|
|
- len(netblocks[most_frequent_netblock]),
|
|
|
- fallback_count),
|
|
|
- mask_bits,
|
|
|
- self[most_frequent_netblock].ipv6addr))
|
|
|
- if shared_netblock_fallback_count > 0:
|
|
|
- logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' +
|
|
|
- 'fallbacks')%(CandidateList.describe_percentage(
|
|
|
- shared_netblock_fallback_count,
|
|
|
- fallback_count),
|
|
|
- mask_bits))
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks in each IPv4 /8, /16,
|
|
|
- # and /24
|
|
|
- def describe_fallback_ipv4_netblocks(self):
|
|
|
- # this doesn't actually tell us anything useful
|
|
|
- #self.describe_fallback_ipv4_netblock_mask(8)
|
|
|
- self.describe_fallback_ipv4_netblock_mask(16)
|
|
|
- #self.describe_fallback_ipv4_netblock_mask(24)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
|
|
|
- # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
|
|
|
- # https://www.iana.org/assignments/ipv6-unicast-address-assignments/
|
|
|
- def describe_fallback_ipv6_netblocks(self):
|
|
|
- # these don't actually tell us anything useful
|
|
|
- #self.describe_fallback_ipv6_netblock_mask(12)
|
|
|
- #self.describe_fallback_ipv6_netblock_mask(23)
|
|
|
- self.describe_fallback_ipv6_netblock_mask(32)
|
|
|
- #self.describe_fallback_ipv6_netblock_mask(48)
|
|
|
- self.describe_fallback_ipv6_netblock_mask(64)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks in each IPv4 and IPv6
|
|
|
- # netblock
|
|
|
- def describe_fallback_netblocks(self):
|
|
|
- self.describe_fallback_ipv4_netblocks()
|
|
|
- self.describe_fallback_ipv6_netblocks()
|
|
|
-
|
|
|
- # return a list of fallbacks which are on the IPv4 ORPort port
|
|
|
- def fallbacks_on_ipv4_orport(self, port):
|
|
|
- return filter(lambda x: x.orport == port, self.fallbacks)
|
|
|
-
|
|
|
- # return a list of fallbacks which are on the IPv6 ORPort port
|
|
|
- def fallbacks_on_ipv6_orport(self, port):
|
|
|
- return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6())
|
|
|
-
|
|
|
- # return a list of fallbacks which are on the DirPort port
|
|
|
- def fallbacks_on_dirport(self, port):
|
|
|
- return filter(lambda x: x.dirport == port, self.fallbacks)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks on IPv4 ORPort port
|
|
|
- # and return that count
|
|
|
- def describe_fallback_ipv4_orport(self, port):
|
|
|
- port_count = len(self.fallbacks_on_ipv4_orport(port))
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- logging.warning('%s of fallbacks are on IPv4 ORPort %d'%(
|
|
|
- CandidateList.describe_percentage(port_count,
|
|
|
- fallback_count),
|
|
|
- port))
|
|
|
- return port_count
|
|
|
-
|
|
|
- # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port
|
|
|
- # and return that count
|
|
|
- def describe_fallback_ipv6_orport(self, port):
|
|
|
- port_count = len(self.fallbacks_on_ipv6_orport(port))
|
|
|
- fallback_count = len(self.fallbacks_with_ipv6())
|
|
|
- logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%(
|
|
|
- CandidateList.describe_percentage(port_count,
|
|
|
- fallback_count),
|
|
|
- port))
|
|
|
- return port_count
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks on DirPort port
|
|
|
- # and return that count
|
|
|
- def describe_fallback_dirport(self, port):
|
|
|
- port_count = len(self.fallbacks_on_dirport(port))
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- logging.warning('%s of fallbacks are on DirPort %d'%(
|
|
|
- CandidateList.describe_percentage(port_count,
|
|
|
- fallback_count),
|
|
|
- port))
|
|
|
- return port_count
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks on each dirport,
|
|
|
- # each IPv4 orport, and each IPv6 orport
|
|
|
- def describe_fallback_ports(self):
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- ipv4_or_count = fallback_count
|
|
|
- ipv4_or_count -= self.describe_fallback_ipv4_orport(443)
|
|
|
- ipv4_or_count -= self.describe_fallback_ipv4_orport(9001)
|
|
|
- logging.warning('%s of fallbacks are on other IPv4 ORPorts'%(
|
|
|
- CandidateList.describe_percentage(ipv4_or_count,
|
|
|
- fallback_count)))
|
|
|
- ipv6_fallback_count = len(self.fallbacks_with_ipv6())
|
|
|
- ipv6_or_count = ipv6_fallback_count
|
|
|
- ipv6_or_count -= self.describe_fallback_ipv6_orport(443)
|
|
|
- ipv6_or_count -= self.describe_fallback_ipv6_orport(9001)
|
|
|
- logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%(
|
|
|
- CandidateList.describe_percentage(ipv6_or_count,
|
|
|
- ipv6_fallback_count)))
|
|
|
- dir_count = fallback_count
|
|
|
- dir_count -= self.describe_fallback_dirport(80)
|
|
|
- dir_count -= self.describe_fallback_dirport(9030)
|
|
|
- logging.warning('%s of fallbacks are on other DirPorts'%(
|
|
|
- CandidateList.describe_percentage(dir_count,
|
|
|
- fallback_count)))
|
|
|
-
|
|
|
- # return a list of fallbacks which cache extra-info documents
|
|
|
- def fallbacks_with_extra_info_cache(self):
|
|
|
- return filter(lambda x: x._extra_info_cache, self.fallbacks)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks that cache extra-info docs
|
|
|
- def describe_fallback_extra_info_caches(self):
|
|
|
- extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- logging.warning('%s of fallbacks cache extra-info documents'%(
|
|
|
- CandidateList.describe_percentage(extra_info_falback_count,
|
|
|
- fallback_count)))
|
|
|
-
|
|
|
- # return a list of fallbacks which have the Exit flag
|
|
|
- def fallbacks_with_exit(self):
|
|
|
- return filter(lambda x: x.is_exit(), self.fallbacks)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks with an Exit flag
|
|
|
- def describe_fallback_exit_flag(self):
|
|
|
- exit_falback_count = len(self.fallbacks_with_exit())
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- logging.warning('%s of fallbacks have the Exit flag'%(
|
|
|
- CandidateList.describe_percentage(exit_falback_count,
|
|
|
- fallback_count)))
|
|
|
-
|
|
|
- # return a list of fallbacks which have an IPv6 address
|
|
|
- def fallbacks_with_ipv6(self):
|
|
|
- return filter(lambda x: x.has_ipv6(), self.fallbacks)
|
|
|
-
|
|
|
- # log a message about the proportion of fallbacks on IPv6
|
|
|
- def describe_fallback_ip_family(self):
|
|
|
- ipv6_falback_count = len(self.fallbacks_with_ipv6())
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- logging.warning('%s of fallbacks are on IPv6'%(
|
|
|
- CandidateList.describe_percentage(ipv6_falback_count,
|
|
|
- fallback_count)))
|
|
|
-
|
|
|
- def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
|
|
|
- guard_count, target_count, check_existing):
|
|
|
- s = ''
|
|
|
- # Report:
|
|
|
- # whether we checked consensus download times
|
|
|
- # the number of fallback directories (and limits/exclusions, if relevant)
|
|
|
- # min & max fallback bandwidths
|
|
|
- # #error if below minimum count
|
|
|
- if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
- s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%(
|
|
|
- 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '',
|
|
|
- ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS
|
|
|
- and PERFORM_IPV6_DIRPORT_CHECKS) else '',
|
|
|
- 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '',
|
|
|
- CONSENSUS_DOWNLOAD_SPEED_MAX)
|
|
|
- else:
|
|
|
- s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */'
|
|
|
- s += '\n'
|
|
|
- # Multiline C comment with #error if things go bad
|
|
|
- s += '/*'
|
|
|
- s += '\n'
|
|
|
- # Integers don't need escaping in C comments
|
|
|
- fallback_count = len(self.fallbacks)
|
|
|
- if FALLBACK_PROPORTION_OF_GUARDS is None:
|
|
|
- fallback_proportion = ''
|
|
|
- else:
|
|
|
- fallback_proportion = ', Target %d (%d * %.2f)'%(target_count,
|
|
|
- guard_count,
|
|
|
- FALLBACK_PROPORTION_OF_GUARDS)
|
|
|
- s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count,
|
|
|
- fallback_proportion)
|
|
|
- if MAX_FALLBACK_COUNT is not None:
|
|
|
- s += ', Max %d'%(MAX_FALLBACK_COUNT)
|
|
|
- s += ')\n'
|
|
|
- if eligible_count != fallback_count:
|
|
|
- removed_count = eligible_count - fallback_count
|
|
|
- excess_to_target_or_max = (eligible_count - operator_count - failed_count
|
|
|
- - fallback_count)
|
|
|
- # some 'Failed' failed the check, others 'Skipped' the check,
|
|
|
- # if we already had enough successful downloads
|
|
|
- s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' +
|
|
|
- 'Excess %d)')%(removed_count, operator_count, failed_count,
|
|
|
- excess_to_target_or_max)
|
|
|
- s += '\n'
|
|
|
- min_fb = self.fallback_min()
|
|
|
- min_bw = min_fb._data['measured_bandwidth']
|
|
|
- max_fb = self.fallback_max()
|
|
|
- max_bw = max_fb._data['measured_bandwidth']
|
|
|
- s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
|
|
|
- max_bw/(1024.0*1024.0))
|
|
|
- s += '\n'
|
|
|
- s += '*/'
|
|
|
- if fallback_count < MIN_FALLBACK_COUNT:
|
|
|
- list_type = 'whitelist'
|
|
|
- if check_existing:
|
|
|
- list_type = 'fallback list'
|
|
|
- # We must have a minimum number of fallbacks so they are always
|
|
|
- # reachable, and are in diverse locations
|
|
|
- s += '\n'
|
|
|
- s += '#error Fallback Count %d is too low. '%(fallback_count)
|
|
|
- s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT)
|
|
|
- s += 'Try adding entries to %s, '%(list_type)
|
|
|
- s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
|
|
|
- return s
|
|
|
-
|
|
|
-def process_existing():
|
|
|
- logging.basicConfig(level=logging.INFO)
|
|
|
- logging.getLogger('stem').setLevel(logging.INFO)
|
|
|
- whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
|
|
|
- 'name': FALLBACK_FILE_NAME,
|
|
|
- 'check_existing' : True}
|
|
|
- list_fallbacks(whitelist, exact=True)
|
|
|
-
|
|
|
-def process_default():
|
|
|
- logging.basicConfig(level=logging.WARNING)
|
|
|
- logging.getLogger('stem').setLevel(logging.WARNING)
|
|
|
- whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
|
|
|
- 'name': WHITELIST_FILE_NAME,
|
|
|
- 'check_existing': False}
|
|
|
- list_fallbacks(whitelist, exact=False)
|
|
|
-
|
|
|
-## Main Function
|
|
|
-def main():
|
|
|
- if get_command() == 'check_existing':
|
|
|
- process_existing()
|
|
|
- else:
|
|
|
- process_default()
|
|
|
-
|
|
|
-def get_command():
|
|
|
- if len(sys.argv) == 2:
|
|
|
- return sys.argv[1]
|
|
|
- else:
|
|
|
- return None
|
|
|
-
|
|
|
-def log_excluded(msg, *args):
|
|
|
- if get_command() == 'check_existing':
|
|
|
- logging.warning(msg, *args)
|
|
|
- else:
|
|
|
- logging.info(msg, *args)
|
|
|
-
|
|
|
-def list_fallbacks(whitelist, exact=False):
|
|
|
- """ Fetches required onionoo documents and evaluates the
|
|
|
- fallback directory criteria for each of the relays,
|
|
|
- passing exact to apply_filter_lists(). """
|
|
|
- if whitelist['check_existing']:
|
|
|
- print "/* type=fallback */"
|
|
|
- else:
|
|
|
- print "/* type=whitelist */"
|
|
|
-
|
|
|
- print ("/* version={} */"
|
|
|
- .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
|
|
|
- now = datetime.datetime.utcnow()
|
|
|
- timestamp = now.strftime('%Y%m%d%H%M%S')
|
|
|
- print ("/* timestamp={} */"
|
|
|
- .format(cleanse_c_multiline_comment(timestamp)))
|
|
|
- # end the header with a separator, to make it easier for parsers
|
|
|
- print SECTION_SEPARATOR_COMMENT
|
|
|
-
|
|
|
- logging.warning('Downloading and parsing Onionoo data. ' +
|
|
|
- 'This may take some time.')
|
|
|
- # find relays that could be fallbacks
|
|
|
- candidates = CandidateList()
|
|
|
- candidates.add_relays()
|
|
|
-
|
|
|
- # work out how many fallbacks we want
|
|
|
- guard_count = candidates.count_guards()
|
|
|
- if FALLBACK_PROPORTION_OF_GUARDS is None:
|
|
|
- target_count = guard_count
|
|
|
- else:
|
|
|
- target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS)
|
|
|
- # the maximum number of fallbacks is the least of:
|
|
|
- # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count)
|
|
|
- # - the maximum fallback count (MAX_FALLBACK_COUNT)
|
|
|
- if MAX_FALLBACK_COUNT is None:
|
|
|
- max_count = target_count
|
|
|
- else:
|
|
|
- max_count = min(target_count, MAX_FALLBACK_COUNT)
|
|
|
-
|
|
|
- candidates.compute_fallbacks()
|
|
|
- prefilter_fallbacks = copy.copy(candidates.fallbacks)
|
|
|
-
|
|
|
- # filter with the whitelist
|
|
|
- # if a relay has changed IPv4 address or ports recently, it will be excluded
|
|
|
- # as ineligible before we call apply_filter_lists, and so there will be no
|
|
|
- # warning that the details have changed from those in the whitelist.
|
|
|
- # instead, there will be an info-level log during the eligibility check.
|
|
|
- initial_count = len(candidates.fallbacks)
|
|
|
- excluded_count = candidates.apply_filter_lists(whitelist, exact=exact)
|
|
|
- print candidates.summarise_filters(initial_count, excluded_count,
|
|
|
- whitelist['check_existing'])
|
|
|
- eligible_count = len(candidates.fallbacks)
|
|
|
-
|
|
|
- # calculate the measured bandwidth of each relay,
|
|
|
- # then remove low-bandwidth relays
|
|
|
- candidates.calculate_measured_bandwidth()
|
|
|
- candidates.remove_low_bandwidth_relays()
|
|
|
-
|
|
|
- # print the raw fallback list
|
|
|
- #for x in candidates.fallbacks:
|
|
|
- # print x.fallbackdir_line(True)
|
|
|
- # print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
|
|
|
- # separators=(',', ': '), default=json_util.default)
|
|
|
-
|
|
|
- # impose mandatory conditions here, like one per contact, family, IP
|
|
|
- # in measured bandwidth order
|
|
|
- candidates.sort_fallbacks_by_measured_bandwidth()
|
|
|
- operator_count = 0
|
|
|
- # only impose these limits on the final list - operators can nominate
|
|
|
- # multiple candidate fallbacks, and then we choose the best set
|
|
|
- if not OUTPUT_CANDIDATES:
|
|
|
- operator_count += candidates.limit_fallbacks_same_ip()
|
|
|
- operator_count += candidates.limit_fallbacks_same_contact()
|
|
|
- operator_count += candidates.limit_fallbacks_same_family()
|
|
|
-
|
|
|
- # check if each candidate can serve a consensus
|
|
|
- # there's a small risk we've eliminated relays from the same operator that
|
|
|
- # can serve a consensus, in favour of one that can't
|
|
|
- # but given it takes up to 15 seconds to check each consensus download,
|
|
|
- # the risk is worth it
|
|
|
- if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
|
|
|
- logging.warning('Checking consensus download speeds. ' +
|
|
|
- 'This may take some time.')
|
|
|
- failed_count = candidates.perform_download_consensus_checks(max_count)
|
|
|
-
|
|
|
- # work out which fallbacks cache extra-infos
|
|
|
- candidates.mark_extra_info_caches()
|
|
|
-
|
|
|
- # analyse and log interesting diversity metrics
|
|
|
- # like netblock, ports, exit, IPv4-only
|
|
|
- # (we can't easily analyse AS, and it's hard to accurately analyse country)
|
|
|
- candidates.describe_fallback_ip_family()
|
|
|
- # if we can't import the ipaddress module, we can't do netblock analysis
|
|
|
- if HAVE_IPADDRESS:
|
|
|
- candidates.describe_fallback_netblocks()
|
|
|
- candidates.describe_fallback_ports()
|
|
|
- candidates.describe_fallback_extra_info_caches()
|
|
|
- candidates.describe_fallback_exit_flag()
|
|
|
-
|
|
|
- # output C comments summarising the fallback selection process
|
|
|
- if len(candidates.fallbacks) > 0:
|
|
|
- print candidates.summarise_fallbacks(eligible_count, operator_count,
|
|
|
- failed_count, guard_count,
|
|
|
- target_count,
|
|
|
- whitelist['check_existing'])
|
|
|
- else:
|
|
|
- print '/* No Fallbacks met criteria */'
|
|
|
-
|
|
|
- # output C comments specifying the OnionOO data used to create the list
|
|
|
- for s in fetch_source_list():
|
|
|
- print describe_fetch_source(s)
|
|
|
-
|
|
|
- # start the list with a separator, to make it easy for parsers
|
|
|
- print SECTION_SEPARATOR_COMMENT
|
|
|
-
|
|
|
- # sort the list differently depending on why we've created it:
|
|
|
- # if we're outputting the final fallback list, sort by fingerprint
|
|
|
- # this makes diffs much more stable
|
|
|
- # otherwise, if we're trying to find a bandwidth cutoff, or we want to
|
|
|
- # contact operators in priority order, sort by bandwidth (not yet
|
|
|
- # implemented)
|
|
|
- # otherwise, if we're contacting operators, sort by contact
|
|
|
- candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
|
|
|
-
|
|
|
- for x in candidates.fallbacks:
|
|
|
- print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
|
|
|
-
|
|
|
-if __name__ == "__main__":
|
|
|
- main()
|
Nick Mathewson