updateFallbackDirs.py 93 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343
  1. #!/usr/bin/env python
  2. # Usage:
  3. #
  4. # Regenerate the list:
  5. # scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc 2> fallback_dirs.log
  6. #
  7. # Check the existing list:
  8. # scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
  9. # mv fallback_dirs.inc.ok src/or/fallback_dirs.inc
  10. #
  11. # This script should be run from a stable, reliable network connection,
  12. # with no other network activity (and not over tor).
  13. # If this is not possible, please disable:
  14. # PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
  15. #
  16. # Needs dateutil, stem, and potentially other python packages.
  17. # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
  18. # for netblock analysis.
  19. #
  20. # Then read the logs to make sure the fallbacks aren't dominated by a single
  21. # netblock or port.
  22. # Script by weasel, April 2015
  23. # Portions by gsathya & karsten, 2013
  24. # https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py
  25. # Modifications by teor, 2015
  26. import StringIO
  27. import string
  28. import re
  29. import datetime
  30. import gzip
  31. import os.path
  32. import json
  33. import math
  34. import sys
  35. import urllib
  36. import urllib2
  37. import hashlib
  38. import dateutil.parser
  39. # bson_lazy provides bson
  40. #from bson import json_util
  41. import copy
  42. import re
  43. from stem.descriptor import DocumentHandler
  44. from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
  45. import logging
  46. logging.root.name = ''
  47. HAVE_IPADDRESS = False
  48. try:
  49. # python 3 builtin, or install package py2-ipaddress
  50. # there are several ipaddress implementations for python 2
  51. # with slightly different semantics with str typed text
  52. # fortunately, all our IP addresses are in unicode
  53. import ipaddress
  54. HAVE_IPADDRESS = True
  55. except ImportError:
  56. # if this happens, we avoid doing netblock analysis
  57. logging.warning('Unable to import ipaddress, please install py2-ipaddress.' +
  58. ' A fallback list will be created, but optional netblock' +
  59. ' analysis will not be performed.')
  60. ## Top-Level Configuration
  61. # We use semantic versioning: https://semver.org
  62. # In particular:
  63. # * major changes include removing a mandatory field, or anything else that
  64. # would break an appropriately tolerant parser,
  65. # * minor changes include adding a field,
  66. # * patch changes include changing header comments or other unstructured
  67. # content
  68. FALLBACK_FORMAT_VERSION = '2.0.0'
  69. SECTION_SEPARATOR_BASE = '====='
  70. SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
  71. # Output all candidate fallbacks, or only output selected fallbacks?
  72. OUTPUT_CANDIDATES = False
  73. # Perform DirPort checks over IPv4?
  74. # Change this to False if IPv4 doesn't work for you, or if you don't want to
  75. # download a consensus for each fallback
  76. # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
  77. PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
  78. # Perform DirPort checks over IPv6?
  79. # If you know IPv6 works for you, set this to True
  80. # This will exclude IPv6 relays without an IPv6 DirPort configured
  81. # So it's best left at False until #18394 is implemented
  82. # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
  83. PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
  84. # Must relays be running now?
  85. MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
  86. or PERFORM_IPV6_DIRPORT_CHECKS)
  87. # Clients have been using microdesc consensuses by default for a while now
  88. DOWNLOAD_MICRODESC_CONSENSUS = True
  89. # If a relay delivers an expired consensus, if it expired less than this many
  90. # seconds ago, we still allow the relay. This should never be less than -90,
  91. # as all directory mirrors should have downloaded a consensus 90 minutes
  92. # before it expires. It should never be more than 24 hours, because clients
  93. # reject consensuses that are older than REASONABLY_LIVE_TIME.
  94. # For the consensus expiry check to be accurate, the machine running this
  95. # script needs an accurate clock.
  96. #
  97. # Relays on 0.3.0 and later return a 404 when they are about to serve an
  98. # expired consensus. This makes them fail the download check.
  99. # We use a tolerance of 0, so that 0.2.x series relays also fail the download
  100. # check if they serve an expired consensus.
  101. CONSENSUS_EXPIRY_TOLERANCE = 0
  102. # Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
  103. OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
  104. # Output matching ContactInfo in fallbacks list or the blacklist?
  105. # Useful if you're trying to contact operators
  106. CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
  107. CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
  108. # How the list should be sorted:
  109. # fingerprint: is useful for stable diffs of fallback lists
  110. # measured_bandwidth: is useful when pruning the list based on bandwidth
  111. # contact: is useful for contacting operators once the list has been pruned
  112. OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
  113. ## OnionOO Settings
  114. ONIONOO = 'https://onionoo.torproject.org/'
  115. #ONIONOO = 'https://onionoo.thecthulhu.com/'
  116. # Don't bother going out to the Internet, just use the files available locally,
  117. # even if they're very old
  118. LOCAL_FILES_ONLY = False
  119. ## Whitelist / Blacklist Filter Settings
  120. # The whitelist contains entries that are included if all attributes match
  121. # (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport)
  122. # The blacklist contains (partial) entries that are excluded if any
  123. # sufficiently specific group of attributes matches:
  124. # IPv4 & DirPort
  125. # IPv4 & ORPort
  126. # ID
  127. # IPv6 & DirPort
  128. # IPv6 & IPv6 ORPort
  129. # If neither port is included in the blacklist, the entire IP address is
  130. # blacklisted.
  131. # What happens to entries in neither list?
  132. # When True, they are included, when False, they are excluded
  133. INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
  134. # If an entry is in both lists, what happens?
  135. # When True, it is excluded, when False, it is included
  136. BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True
  137. WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
  138. BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist'
  139. FALLBACK_FILE_NAME = 'src/or/fallback_dirs.inc'
  140. # The number of bytes we'll read from a filter file before giving up
  141. MAX_LIST_FILE_SIZE = 1024 * 1024
  142. ## Eligibility Settings
  143. # Require fallbacks to have the same address and port for a set amount of time
  144. # We used to have this at 1 week, but that caused many fallback failures, which
  145. # meant that we had to rebuild the list more often. We want fallbacks to be
  146. # stable for 2 years, so we set it to a few months.
  147. #
  148. # There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily
  149. # submits a 0 DirPort when restarted.
  150. # This causes OnionOO to (correctly) reset its stability timer.
  151. # Affected relays should upgrade to Tor 0.2.9 or later, which has a fix
  152. # for this issue.
  153. #
  154. # If a relay changes address or port, that's it, it's not useful any more,
  155. # because clients can't find it
  156. ADDRESS_AND_PORT_STABLE_DAYS = 90
  157. # We ignore relays that have been down for more than this period
  158. MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
  159. # FallbackDirs must have a time-weighted-fraction that is greater than or
  160. # equal to:
  161. # Mirrors that are down half the time are still useful half the time
  162. CUTOFF_RUNNING = .50
  163. CUTOFF_V2DIR = .50
  164. # Guard flags are removed for some time after a relay restarts, so we ignore
  165. # the guard flag.
  166. CUTOFF_GUARD = .00
  167. # FallbackDirs must have a time-weighted-fraction that is less than or equal
  168. # to:
  169. # .00 means no bad exits
  170. PERMITTED_BADEXIT = .00
  171. # older entries' weights are adjusted with ALPHA^(age in days)
  172. AGE_ALPHA = 0.99
  173. # this factor is used to scale OnionOO entries to [0,1]
  174. ONIONOO_SCALE_ONE = 999.
  175. ## Fallback Count Limits
  176. # The target for these parameters is 20% of the guards in the network
  177. # This is around 200 as of October 2015
  178. _FB_POG = 0.2
  179. FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
  180. # Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
  181. MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
  182. # Emit a C #error if the number of fallbacks is less than expected
  183. MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
  184. # The maximum number of fallbacks on the same address, contact, or family
  185. #
  186. # With 150 fallbacks, this means each operator sees 5% of client bootstraps.
  187. # For comparison:
  188. # - We try to limit guard and exit operators to 5% of the network
  189. # - The directory authorities used to see 11% of client bootstraps each
  190. #
  191. # We also don't want too much of the list to go down if a single operator
  192. # has to move all their relays.
  193. MAX_FALLBACKS_PER_IP = 1
  194. MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
  195. MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
  196. MAX_FALLBACKS_PER_CONTACT = 7
  197. MAX_FALLBACKS_PER_FAMILY = 7
  198. ## Fallback Bandwidth Requirements
  199. # Any fallback with the Exit flag has its bandwidth multiplied by this fraction
  200. # to make sure we aren't further overloading exits
  201. # (Set to 1.0, because we asked that only lightly loaded exits opt-in,
  202. # and the extra load really isn't that much for large relays.)
  203. EXIT_BANDWIDTH_FRACTION = 1.0
  204. # If a single fallback's bandwidth is too low, it's pointless adding it
  205. # We expect fallbacks to handle an extra 10 kilobytes per second of traffic
  206. # Make sure they can support fifty times the expected extra load
  207. #
  208. # We convert this to a consensus weight before applying the filter,
  209. # because all the bandwidth amounts are specified by the relay
  210. MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
  211. # Clients will time out after 30 seconds trying to download a consensus
  212. # So allow fallback directories half that to deliver a consensus
  213. # The exact download times might change based on the network connection
  214. # running this script, but only by a few seconds
  215. # There is also about a second of python overhead
  216. CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
  217. # If the relay fails a consensus check, retry the download
  218. # This avoids delisting a relay due to transient network conditions
  219. CONSENSUS_DOWNLOAD_RETRY = True
  220. ## Parsing Functions
  221. def parse_ts(t):
  222. return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
  223. def remove_bad_chars(raw_string, bad_char_list):
  224. # Remove each character in the bad_char_list
  225. cleansed_string = raw_string
  226. for c in bad_char_list:
  227. cleansed_string = cleansed_string.replace(c, '')
  228. return cleansed_string
  229. def cleanse_unprintable(raw_string):
  230. # Remove all unprintable characters
  231. cleansed_string = ''
  232. for c in raw_string:
  233. if c in string.printable:
  234. cleansed_string += c
  235. return cleansed_string
  236. def cleanse_whitespace(raw_string):
  237. # Replace all whitespace characters with a space
  238. cleansed_string = raw_string
  239. for c in string.whitespace:
  240. cleansed_string = cleansed_string.replace(c, ' ')
  241. return cleansed_string
  242. def cleanse_c_multiline_comment(raw_string):
  243. cleansed_string = raw_string
  244. # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
  245. cleansed_string = cleanse_whitespace(cleansed_string)
  246. # ContactInfo and Version can be arbitrary binary data
  247. cleansed_string = cleanse_unprintable(cleansed_string)
  248. # Prevent a malicious / unanticipated string from breaking out
  249. # of a C-style multiline comment
  250. # This removes '/*' and '*/' and '//'
  251. bad_char_list = '*/'
  252. # Prevent a malicious string from using C nulls
  253. bad_char_list += '\0'
  254. # Avoid confusing parsers by making sure there is only one comma per fallback
  255. bad_char_list += ','
  256. # Avoid confusing parsers by making sure there is only one equals per field
  257. bad_char_list += '='
  258. # Be safer by removing bad characters entirely
  259. cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
  260. # Some compilers may further process the content of comments
  261. # There isn't much we can do to cover every possible case
  262. # But comment-based directives are typically only advisory
  263. return cleansed_string
  264. def cleanse_c_string(raw_string):
  265. cleansed_string = raw_string
  266. # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
  267. cleansed_string = cleanse_whitespace(cleansed_string)
  268. # ContactInfo and Version can be arbitrary binary data
  269. cleansed_string = cleanse_unprintable(cleansed_string)
  270. # Prevent a malicious address/fingerprint string from breaking out
  271. # of a C-style string
  272. bad_char_list = '"'
  273. # Prevent a malicious string from using escapes
  274. bad_char_list += '\\'
  275. # Prevent a malicious string from using C nulls
  276. bad_char_list += '\0'
  277. # Avoid confusing parsers by making sure there is only one comma per fallback
  278. bad_char_list += ','
  279. # Avoid confusing parsers by making sure there is only one equals per field
  280. bad_char_list += '='
  281. # Be safer by removing bad characters entirely
  282. cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
  283. # Some compilers may further process the content of strings
  284. # There isn't much we can do to cover every possible case
  285. # But this typically only results in changes to the string data
  286. return cleansed_string
  287. ## OnionOO Source Functions
  288. # a dictionary of source metadata for each onionoo query we've made
  289. fetch_source = {}
  290. # register source metadata for 'what'
  291. # assumes we only retrieve one document for each 'what'
  292. def register_fetch_source(what, url, relays_published, version):
  293. fetch_source[what] = {}
  294. fetch_source[what]['url'] = url
  295. fetch_source[what]['relays_published'] = relays_published
  296. fetch_source[what]['version'] = version
  297. # list each registered source's 'what'
  298. def fetch_source_list():
  299. return sorted(fetch_source.keys())
  300. # given 'what', provide a multiline C comment describing the source
  301. def describe_fetch_source(what):
  302. desc = '/*'
  303. desc += '\n'
  304. desc += 'Onionoo Source: '
  305. desc += cleanse_c_multiline_comment(what)
  306. desc += ' Date: '
  307. desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published'])
  308. desc += ' Version: '
  309. desc += cleanse_c_multiline_comment(fetch_source[what]['version'])
  310. desc += '\n'
  311. desc += 'URL: '
  312. desc += cleanse_c_multiline_comment(fetch_source[what]['url'])
  313. desc += '\n'
  314. desc += '*/'
  315. return desc
  316. ## File Processing Functions
  317. def write_to_file(str, file_name, max_len):
  318. try:
  319. with open(file_name, 'w') as f:
  320. f.write(str[0:max_len])
  321. except EnvironmentError, error:
  322. logging.error('Writing file %s failed: %d: %s'%
  323. (file_name,
  324. error.errno,
  325. error.strerror)
  326. )
  327. def read_from_file(file_name, max_len):
  328. try:
  329. if os.path.isfile(file_name):
  330. with open(file_name, 'r') as f:
  331. return f.read(max_len)
  332. except EnvironmentError, error:
  333. logging.info('Loading file %s failed: %d: %s'%
  334. (file_name,
  335. error.errno,
  336. error.strerror)
  337. )
  338. return None
  339. def parse_fallback_file(file_name):
  340. file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
  341. file_data = cleanse_unprintable(file_data)
  342. file_data = remove_bad_chars(file_data, '\n"\0')
  343. file_data = re.sub('/\*.*?\*/', '', file_data)
  344. file_data = file_data.replace(',', '\n')
  345. file_data = file_data.replace(' weight=10', '')
  346. return file_data
  347. def load_possibly_compressed_response_json(response):
  348. if response.info().get('Content-Encoding') == 'gzip':
  349. buf = StringIO.StringIO( response.read() )
  350. f = gzip.GzipFile(fileobj=buf)
  351. return json.load(f)
  352. else:
  353. return json.load(response)
  354. def load_json_from_file(json_file_name):
  355. # An exception here may be resolved by deleting the .last_modified
  356. # and .json files, and re-running the script
  357. try:
  358. with open(json_file_name, 'r') as f:
  359. return json.load(f)
  360. except EnvironmentError, error:
  361. raise Exception('Reading not-modified json file %s failed: %d: %s'%
  362. (json_file_name,
  363. error.errno,
  364. error.strerror)
  365. )
  366. ## OnionOO Functions
  367. def datestr_to_datetime(datestr):
  368. # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT
  369. if datestr is not None:
  370. dt = dateutil.parser.parse(datestr)
  371. else:
  372. # Never modified - use start of epoch
  373. dt = datetime.datetime.utcfromtimestamp(0)
  374. # strip any timezone out (in case they're supported in future)
  375. dt = dt.replace(tzinfo=None)
  376. return dt
  377. def onionoo_fetch(what, **kwargs):
  378. params = kwargs
  379. params['type'] = 'relay'
  380. #params['limit'] = 10
  381. params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
  382. params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
  383. params['flag'] = 'V2Dir'
  384. url = ONIONOO + what + '?' + urllib.urlencode(params)
  385. # Unfortunately, the URL is too long for some OS filenames,
  386. # but we still don't want to get files from different URLs mixed up
  387. base_file_name = what + '-' + hashlib.sha1(url).hexdigest()
  388. full_url_file_name = base_file_name + '.full_url'
  389. MAX_FULL_URL_LENGTH = 1024
  390. last_modified_file_name = base_file_name + '.last_modified'
  391. MAX_LAST_MODIFIED_LENGTH = 64
  392. json_file_name = base_file_name + '.json'
  393. if LOCAL_FILES_ONLY:
  394. # Read from the local file, don't write to anything
  395. response_json = load_json_from_file(json_file_name)
  396. else:
  397. # store the full URL to a file for debugging
  398. # no need to compare as long as you trust SHA-1
  399. write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH)
  400. request = urllib2.Request(url)
  401. request.add_header('Accept-encoding', 'gzip')
  402. # load the last modified date from the file, if it exists
  403. last_mod_date = read_from_file(last_modified_file_name,
  404. MAX_LAST_MODIFIED_LENGTH)
  405. if last_mod_date is not None:
  406. request.add_header('If-modified-since', last_mod_date)
  407. # Parse last modified date
  408. last_mod = datestr_to_datetime(last_mod_date)
  409. # Not Modified and still recent enough to be useful
  410. # Onionoo / Globe used to use 6 hours, but we can afford a day
  411. required_freshness = datetime.datetime.utcnow()
  412. # strip any timezone out (to match dateutil.parser)
  413. required_freshness = required_freshness.replace(tzinfo=None)
  414. required_freshness -= datetime.timedelta(hours=24)
  415. # Make the OnionOO request
  416. response_code = 0
  417. try:
  418. response = urllib2.urlopen(request)
  419. response_code = response.getcode()
  420. except urllib2.HTTPError, error:
  421. response_code = error.code
  422. if response_code == 304: # not modified
  423. pass
  424. else:
  425. raise Exception("Could not get " + url + ": "
  426. + str(error.code) + ": " + error.reason)
  427. if response_code == 200: # OK
  428. last_mod = datestr_to_datetime(response.info().get('Last-Modified'))
  429. # Check for freshness
  430. if last_mod < required_freshness:
  431. if last_mod_date is not None:
  432. # This check sometimes fails transiently, retry the script if it does
  433. date_message = "Outdated data: last updated " + last_mod_date
  434. else:
  435. date_message = "No data: never downloaded "
  436. raise Exception(date_message + " from " + url)
  437. # Process the data
  438. if response_code == 200: # OK
  439. response_json = load_possibly_compressed_response_json(response)
  440. with open(json_file_name, 'w') as f:
  441. # use the most compact json representation to save space
  442. json.dump(response_json, f, separators=(',',':'))
  443. # store the last modified date in its own file
  444. if response.info().get('Last-modified') is not None:
  445. write_to_file(response.info().get('Last-Modified'),
  446. last_modified_file_name,
  447. MAX_LAST_MODIFIED_LENGTH)
  448. elif response_code == 304: # Not Modified
  449. response_json = load_json_from_file(json_file_name)
  450. else: # Unexpected HTTP response code not covered in the HTTPError above
  451. raise Exception("Unexpected HTTP response code to " + url + ": "
  452. + str(response_code))
  453. register_fetch_source(what,
  454. url,
  455. response_json['relays_published'],
  456. response_json['version'])
  457. return response_json
  458. def fetch(what, **kwargs):
  459. #x = onionoo_fetch(what, **kwargs)
  460. # don't use sort_keys, as the order of or_addresses is significant
  461. #print json.dumps(x, indent=4, separators=(',', ': '))
  462. #sys.exit(0)
  463. return onionoo_fetch(what, **kwargs)
  464. ## Fallback Candidate Class
  465. class Candidate(object):
  466. CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow()
  467. - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS))
  468. def __init__(self, details):
  469. for f in ['fingerprint', 'nickname', 'last_changed_address_or_port',
  470. 'consensus_weight', 'or_addresses', 'dir_address']:
  471. if not f in details: raise Exception("Document has no %s field."%(f,))
  472. if not 'contact' in details:
  473. details['contact'] = None
  474. if not 'flags' in details or details['flags'] is None:
  475. details['flags'] = []
  476. if (not 'advertised_bandwidth' in details
  477. or details['advertised_bandwidth'] is None):
  478. # relays without advertised bandwidth have it calculated from their
  479. # consensus weight
  480. details['advertised_bandwidth'] = 0
  481. if (not 'effective_family' in details
  482. or details['effective_family'] is None):
  483. details['effective_family'] = []
  484. if not 'platform' in details:
  485. details['platform'] = None
  486. details['last_changed_address_or_port'] = parse_ts(
  487. details['last_changed_address_or_port'])
  488. self._data = details
  489. self._stable_sort_or_addresses()
  490. self._fpr = self._data['fingerprint']
  491. self._running = self._guard = self._v2dir = 0.
  492. self._split_dirport()
  493. self._compute_orport()
  494. if self.orport is None:
  495. raise Exception("Failed to get an orport for %s."%(self._fpr,))
  496. self._compute_ipv6addr()
  497. if not self.has_ipv6():
  498. logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
  499. self._compute_version()
  500. self._extra_info_cache = None
  501. def _stable_sort_or_addresses(self):
  502. # replace self._data['or_addresses'] with a stable ordering,
  503. # sorting the secondary addresses in string order
  504. # leave the received order in self._data['or_addresses_raw']
  505. self._data['or_addresses_raw'] = self._data['or_addresses']
  506. or_address_primary = self._data['or_addresses'][:1]
  507. # subsequent entries in the or_addresses array are in an arbitrary order
  508. # so we stabilise the addresses by sorting them in string order
  509. or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:])
  510. or_addresses_stable = or_address_primary + or_addresses_secondaries_stable
  511. self._data['or_addresses'] = or_addresses_stable
  512. def get_fingerprint(self):
  513. return self._fpr
  514. # is_valid_ipv[46]_address by gsathya, karsten, 2013
  515. @staticmethod
  516. def is_valid_ipv4_address(address):
  517. if not isinstance(address, (str, unicode)):
  518. return False
  519. # check if there are four period separated values
  520. if address.count(".") != 3:
  521. return False
  522. # checks that each value in the octet are decimal values between 0-255
  523. for entry in address.split("."):
  524. if not entry.isdigit() or int(entry) < 0 or int(entry) > 255:
  525. return False
  526. elif entry[0] == "0" and len(entry) > 1:
  527. return False # leading zeros, for instance in "1.2.3.001"
  528. return True
  529. @staticmethod
  530. def is_valid_ipv6_address(address):
  531. if not isinstance(address, (str, unicode)):
  532. return False
  533. # remove brackets
  534. address = address[1:-1]
  535. # addresses are made up of eight colon separated groups of four hex digits
  536. # with leading zeros being optional
  537. # https://en.wikipedia.org/wiki/IPv6#Address_format
  538. colon_count = address.count(":")
  539. if colon_count > 7:
  540. return False # too many groups
  541. elif colon_count != 7 and not "::" in address:
  542. return False # not enough groups and none are collapsed
  543. elif address.count("::") > 1 or ":::" in address:
  544. return False # multiple groupings of zeros can't be collapsed
  545. found_ipv4_on_previous_entry = False
  546. for entry in address.split(":"):
  547. # If an IPv6 address has an embedded IPv4 address,
  548. # it must be the last entry
  549. if found_ipv4_on_previous_entry:
  550. return False
  551. if not re.match("^[0-9a-fA-f]{0,4}$", entry):
  552. if not Candidate.is_valid_ipv4_address(entry):
  553. return False
  554. else:
  555. found_ipv4_on_previous_entry = True
  556. return True
  557. def _split_dirport(self):
  558. # Split the dir_address into dirip and dirport
  559. (self.dirip, _dirport) = self._data['dir_address'].split(':', 2)
  560. self.dirport = int(_dirport)
  561. def _compute_orport(self):
  562. # Choose the first ORPort that's on the same IPv4 address as the DirPort.
  563. # In rare circumstances, this might not be the primary ORPort address.
  564. # However, _stable_sort_or_addresses() ensures we choose the same one
  565. # every time, even if onionoo changes the order of the secondaries.
  566. self._split_dirport()
  567. self.orport = None
  568. for i in self._data['or_addresses']:
  569. if i != self._data['or_addresses'][0]:
  570. logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i))
  571. (ipaddr, port) = i.rsplit(':', 1)
  572. if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr):
  573. self.orport = int(port)
  574. return
  575. def _compute_ipv6addr(self):
  576. # Choose the first IPv6 address that uses the same port as the ORPort
  577. # Or, choose the first IPv6 address in the list
  578. # _stable_sort_or_addresses() ensures we choose the same IPv6 address
  579. # every time, even if onionoo changes the order of the secondaries.
  580. self.ipv6addr = None
  581. self.ipv6orport = None
  582. # Choose the first IPv6 address that uses the same port as the ORPort
  583. for i in self._data['or_addresses']:
  584. (ipaddr, port) = i.rsplit(':', 1)
  585. if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr):
  586. self.ipv6addr = ipaddr
  587. self.ipv6orport = int(port)
  588. return
  589. # Choose the first IPv6 address in the list
  590. for i in self._data['or_addresses']:
  591. (ipaddr, port) = i.rsplit(':', 1)
  592. if Candidate.is_valid_ipv6_address(ipaddr):
  593. self.ipv6addr = ipaddr
  594. self.ipv6orport = int(port)
  595. return
  596. def _compute_version(self):
  597. # parse the version out of the platform string
  598. # The platform looks like: "Tor 0.2.7.6 on Linux"
  599. self._data['version'] = None
  600. if self._data['platform'] is None:
  601. return
  602. # be tolerant of weird whitespacing, use a whitespace split
  603. tokens = self._data['platform'].split()
  604. for token in tokens:
  605. vnums = token.split('.')
  606. # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
  607. if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
  608. vnums[2].isdigit()):
  609. self._data['version'] = token
  610. return
  611. # From #20509
  612. # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
  613. # and version 0.3.0.0-alpha-dev
  614. # Exhaustive lists are hard to get wrong
  615. STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
  616. '0.2.9.2-alpha',
  617. '0.2.9.2-alpha-dev',
  618. '0.2.9.3-alpha',
  619. '0.2.9.3-alpha-dev',
  620. '0.2.9.4-alpha',
  621. '0.2.9.4-alpha-dev',
  622. '0.3.0.0-alpha-dev'
  623. ]
  624. def is_valid_version(self):
  625. # call _compute_version before calling this
  626. # is the version of the relay a version we want as a fallback?
  627. # checks both recommended versions and bug #20499 / #20509
  628. #
  629. # if the relay doesn't have a recommended version field, exclude the relay
  630. if not self._data.has_key('recommended_version'):
  631. log_excluded('%s not a candidate: no recommended_version field',
  632. self._fpr)
  633. return False
  634. if not self._data['recommended_version']:
  635. log_excluded('%s not a candidate: version not recommended', self._fpr)
  636. return False
  637. # if the relay doesn't have version field, exclude the relay
  638. if not self._data.has_key('version'):
  639. log_excluded('%s not a candidate: no version field', self._fpr)
  640. return False
  641. if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
  642. logging.warning('%s not a candidate: version delivers stale consensuses',
  643. self._fpr)
  644. return False
  645. return True
  646. @staticmethod
  647. def _extract_generic_history(history, which='unknown'):
  648. # given a tree like this:
  649. # {
  650. # "1_month": {
  651. # "count": 187,
  652. # "factor": 0.001001001001001001,
  653. # "first": "2015-02-27 06:00:00",
  654. # "interval": 14400,
  655. # "last": "2015-03-30 06:00:00",
  656. # "values": [
  657. # 999,
  658. # 999
  659. # ]
  660. # },
  661. # "1_week": {
  662. # "count": 169,
  663. # "factor": 0.001001001001001001,
  664. # "first": "2015-03-23 07:30:00",
  665. # "interval": 3600,
  666. # "last": "2015-03-30 07:30:00",
  667. # "values": [ ...]
  668. # },
  669. # "1_year": {
  670. # "count": 177,
  671. # "factor": 0.001001001001001001,
  672. # "first": "2014-04-11 00:00:00",
  673. # "interval": 172800,
  674. # "last": "2015-03-29 00:00:00",
  675. # "values": [ ...]
  676. # },
  677. # "3_months": {
  678. # "count": 185,
  679. # "factor": 0.001001001001001001,
  680. # "first": "2014-12-28 06:00:00",
  681. # "interval": 43200,
  682. # "last": "2015-03-30 06:00:00",
  683. # "values": [ ...]
  684. # }
  685. # },
  686. # extract exactly one piece of data per time interval,
  687. # using smaller intervals where available.
  688. #
  689. # returns list of (age, length, value) dictionaries.
  690. generic_history = []
  691. periods = history.keys()
  692. periods.sort(key = lambda x: history[x]['interval'])
  693. now = datetime.datetime.utcnow()
  694. newest = now
  695. for p in periods:
  696. h = history[p]
  697. interval = datetime.timedelta(seconds = h['interval'])
  698. this_ts = parse_ts(h['last'])
  699. if (len(h['values']) != h['count']):
  700. logging.warning('Inconsistent value count in %s document for %s'
  701. %(p, which))
  702. for v in reversed(h['values']):
  703. if (this_ts <= newest):
  704. agt1 = now - this_ts
  705. agt2 = interval
  706. agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600)
  707. * 10**6) / 10**6
  708. agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600)
  709. * 10**6) / 10**6
  710. generic_history.append(
  711. { 'age': agetmp1,
  712. 'length': agetmp2,
  713. 'value': v
  714. })
  715. newest = this_ts
  716. this_ts -= interval
  717. if (this_ts + interval != parse_ts(h['first'])):
  718. logging.warning('Inconsistent time information in %s document for %s'
  719. %(p, which))
  720. #print json.dumps(generic_history, sort_keys=True,
  721. # indent=4, separators=(',', ': '))
  722. return generic_history
  723. @staticmethod
  724. def _avg_generic_history(generic_history):
  725. a = []
  726. for i in generic_history:
  727. if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600):
  728. continue
  729. if (i['length'] is not None
  730. and i['age'] is not None
  731. and i['value'] is not None):
  732. w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24))
  733. a.append( (i['value'] * w, w) )
  734. sv = math.fsum(map(lambda x: x[0], a))
  735. sw = math.fsum(map(lambda x: x[1], a))
  736. if sw == 0.0:
  737. svw = 0.0
  738. else:
  739. svw = sv/sw
  740. return svw
  741. def _add_generic_history(self, history):
  742. periods = r['read_history'].keys()
  743. periods.sort(key = lambda x: r['read_history'][x]['interval'] )
  744. print periods
  745. def add_running_history(self, history):
  746. pass
  747. def add_uptime(self, uptime):
  748. logging.debug('Adding uptime %s.'%(self._fpr,))
  749. # flags we care about: Running, V2Dir, Guard
  750. if not 'flags' in uptime:
  751. logging.debug('No flags in document for %s.'%(self._fpr,))
  752. return
  753. for f in ['Running', 'Guard', 'V2Dir']:
  754. if not f in uptime['flags']:
  755. logging.debug('No %s in flags for %s.'%(f, self._fpr,))
  756. return
  757. running = self._extract_generic_history(uptime['flags']['Running'],
  758. '%s-Running'%(self._fpr))
  759. guard = self._extract_generic_history(uptime['flags']['Guard'],
  760. '%s-Guard'%(self._fpr))
  761. v2dir = self._extract_generic_history(uptime['flags']['V2Dir'],
  762. '%s-V2Dir'%(self._fpr))
  763. if 'BadExit' in uptime['flags']:
  764. badexit = self._extract_generic_history(uptime['flags']['BadExit'],
  765. '%s-BadExit'%(self._fpr))
  766. self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE
  767. self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE
  768. self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE
  769. self._badexit = None
  770. if 'BadExit' in uptime['flags']:
  771. self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
  772. def is_candidate(self):
  773. try:
  774. if (MUST_BE_RUNNING_NOW and not self.is_running()):
  775. log_excluded('%s not a candidate: not running now, unable to check ' +
  776. 'DirPort consensus download', self._fpr)
  777. return False
  778. if (self._data['last_changed_address_or_port'] >
  779. self.CUTOFF_ADDRESS_AND_PORT_STABLE):
  780. log_excluded('%s not a candidate: changed address/port recently (%s)',
  781. self._fpr, self._data['last_changed_address_or_port'])
  782. return False
  783. if self._running < CUTOFF_RUNNING:
  784. log_excluded('%s not a candidate: running avg too low (%lf)',
  785. self._fpr, self._running)
  786. return False
  787. if self._v2dir < CUTOFF_V2DIR:
  788. log_excluded('%s not a candidate: v2dir avg too low (%lf)',
  789. self._fpr, self._v2dir)
  790. return False
  791. if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
  792. log_excluded('%s not a candidate: badexit avg too high (%lf)',
  793. self._fpr, self._badexit)
  794. return False
  795. # this function logs a message depending on which check fails
  796. if not self.is_valid_version():
  797. return False
  798. if self._guard < CUTOFF_GUARD:
  799. log_excluded('%s not a candidate: guard avg too low (%lf)',
  800. self._fpr, self._guard)
  801. return False
  802. if (not self._data.has_key('consensus_weight')
  803. or self._data['consensus_weight'] < 1):
  804. log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
  805. return False
  806. except BaseException as e:
  807. logging.warning("Exception %s when checking if fallback is a candidate",
  808. str(e))
  809. return False
  810. return True
  811. def is_in_whitelist(self, relaylist):
  812. """ A fallback matches if each key in the whitelist line matches:
  813. ipv4
  814. dirport
  815. orport
  816. id
  817. ipv6 address and port (if present)
  818. If the fallback has an ipv6 key, the whitelist line must also have
  819. it, and vice versa, otherwise they don't match. """
  820. ipv6 = None
  821. if self.has_ipv6():
  822. ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
  823. for entry in relaylist:
  824. if entry['id'] != self._fpr:
  825. # can't log here unless we match an IP and port, because every relay's
  826. # fingerprint is compared to every entry's fingerprint
  827. if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport:
  828. logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
  829. '%s?', entry['id'], self.dirip, self.orport,
  830. self._fpr)
  831. if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6:
  832. logging.warning('%s excluded: has OR %s changed fingerprint to ' +
  833. '%s?', entry['id'], ipv6, self._fpr)
  834. continue
  835. if entry['ipv4'] != self.dirip:
  836. logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
  837. self._fpr, entry['ipv4'], self.dirip)
  838. continue
  839. if int(entry['dirport']) != self.dirport:
  840. logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
  841. '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
  842. self.dirip, self.dirport)
  843. continue
  844. if int(entry['orport']) != self.orport:
  845. logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
  846. '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
  847. self.dirip, self.orport)
  848. continue
  849. if entry.has_key('ipv6') and self.has_ipv6():
  850. # if both entry and fallback have an ipv6 address, compare them
  851. if entry['ipv6'] != ipv6:
  852. logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
  853. 'to %s?', self._fpr, entry['ipv6'], ipv6)
  854. continue
  855. # if the fallback has an IPv6 address but the whitelist entry
  856. # doesn't, or vice versa, the whitelist entry doesn't match
  857. elif entry.has_key('ipv6') and not self.has_ipv6():
  858. logging.warning('%s excluded: has it lost its former IPv6 address %s?',
  859. self._fpr, entry['ipv6'])
  860. continue
  861. elif not entry.has_key('ipv6') and self.has_ipv6():
  862. logging.warning('%s excluded: has it gained an IPv6 address %s?',
  863. self._fpr, ipv6)
  864. continue
  865. return True
  866. return False
  867. def is_in_blacklist(self, relaylist):
  868. """ A fallback matches a blacklist line if a sufficiently specific group
  869. of attributes matches:
  870. ipv4 & dirport
  871. ipv4 & orport
  872. id
  873. ipv6 & dirport
  874. ipv6 & ipv6 orport
  875. If the fallback and the blacklist line both have an ipv6 key,
  876. their values will be compared, otherwise, they will be ignored.
  877. If there is no dirport and no orport, the entry matches all relays on
  878. that ip. """
  879. for entry in relaylist:
  880. for key in entry:
  881. value = entry[key]
  882. if key == 'id' and value == self._fpr:
  883. log_excluded('%s is in the blacklist: fingerprint matches',
  884. self._fpr)
  885. return True
  886. if key == 'ipv4' and value == self.dirip:
  887. # if the dirport is present, check it too
  888. if entry.has_key('dirport'):
  889. if int(entry['dirport']) == self.dirport:
  890. log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
  891. 'DirPort (%d) match', self._fpr, self.dirip,
  892. self.dirport)
  893. return True
  894. # if the orport is present, check it too
  895. elif entry.has_key('orport'):
  896. if int(entry['orport']) == self.orport:
  897. log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
  898. 'ORPort (%d) match', self._fpr, self.dirip,
  899. self.orport)
  900. return True
  901. else:
  902. log_excluded('%s is in the blacklist: IPv4 (%s) matches, and ' +
  903. 'entry has no DirPort or ORPort', self._fpr,
  904. self.dirip)
  905. return True
  906. ipv6 = None
  907. if self.has_ipv6():
  908. ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
  909. if (key == 'ipv6' and self.has_ipv6()):
  910. # if both entry and fallback have an ipv6 address, compare them,
  911. # otherwise, disregard ipv6 addresses
  912. if value == ipv6:
  913. # if the dirport is present, check it too
  914. if entry.has_key('dirport'):
  915. if int(entry['dirport']) == self.dirport:
  916. log_excluded('%s is in the blacklist: IPv6 (%s) and ' +
  917. 'DirPort (%d) match', self._fpr, ipv6,
  918. self.dirport)
  919. return True
  920. # we've already checked the ORPort, it's part of entry['ipv6']
  921. else:
  922. log_excluded('%s is in the blacklist: IPv6 (%s) matches, and' +
  923. 'entry has no DirPort', self._fpr, ipv6)
  924. return True
  925. elif (key == 'ipv6' or self.has_ipv6()):
  926. # only log if the fingerprint matches but the IPv6 doesn't
  927. if entry.has_key('id') and entry['id'] == self._fpr:
  928. log_excluded('%s skipping IPv6 blacklist comparison: relay ' +
  929. 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr,
  930. '' if self.has_ipv6() else ' no',
  931. (' (' + ipv6 + ')') if self.has_ipv6() else '',
  932. '' if key == 'ipv6' else ' no',
  933. (' (' + value + ')') if key == 'ipv6' else '')
  934. logging.warning('Has %s %s IPv6 address %s?', self._fpr,
  935. 'gained an' if self.has_ipv6() else 'lost its former',
  936. ipv6 if self.has_ipv6() else value)
  937. return False
  938. def cw_to_bw_factor(self):
  939. # any relays with a missing or zero consensus weight are not candidates
  940. # any relays with a missing advertised bandwidth have it set to zero
  941. return self._data['advertised_bandwidth'] / self._data['consensus_weight']
  942. # since advertised_bandwidth is reported by the relay, it can be gamed
  943. # to avoid this, use the median consensus weight to bandwidth factor to
  944. # estimate this relay's measured bandwidth, and make that the upper limit
  945. def measured_bandwidth(self, median_cw_to_bw_factor):
  946. cw_to_bw= median_cw_to_bw_factor
  947. # Reduce exit bandwidth to make sure we're not overloading them
  948. if self.is_exit():
  949. cw_to_bw *= EXIT_BANDWIDTH_FRACTION
  950. measured_bandwidth = self._data['consensus_weight'] * cw_to_bw
  951. if self._data['advertised_bandwidth'] != 0:
  952. # limit advertised bandwidth (if available) to measured bandwidth
  953. return min(measured_bandwidth, self._data['advertised_bandwidth'])
  954. else:
  955. return measured_bandwidth
  956. def set_measured_bandwidth(self, median_cw_to_bw_factor):
  957. self._data['measured_bandwidth'] = self.measured_bandwidth(
  958. median_cw_to_bw_factor)
  959. def is_exit(self):
  960. return 'Exit' in self._data['flags']
  961. def is_guard(self):
  962. return 'Guard' in self._data['flags']
  963. def is_running(self):
  964. return 'Running' in self._data['flags']
  965. # does this fallback have an IPv6 address and orport?
  966. def has_ipv6(self):
  967. return self.ipv6addr is not None and self.ipv6orport is not None
  968. # strip leading and trailing brackets from an IPv6 address
  969. # safe to use on non-bracketed IPv6 and on IPv4 addresses
  970. # also convert to unicode, and make None appear as ''
  971. @staticmethod
  972. def strip_ipv6_brackets(ip):
  973. if ip is None:
  974. return unicode('')
  975. if len(ip) < 2:
  976. return unicode(ip)
  977. if ip[0] == '[' and ip[-1] == ']':
  978. return unicode(ip[1:-1])
  979. return unicode(ip)
  980. # are ip_a and ip_b in the same netblock?
  981. # mask_bits is the size of the netblock
  982. # takes both IPv4 and IPv6 addresses
  983. # the versions of ip_a and ip_b must be the same
  984. # the mask must be valid for the IP version
  985. @staticmethod
  986. def netblocks_equal(ip_a, ip_b, mask_bits):
  987. if ip_a is None or ip_b is None:
  988. return False
  989. ip_a = Candidate.strip_ipv6_brackets(ip_a)
  990. ip_b = Candidate.strip_ipv6_brackets(ip_b)
  991. a = ipaddress.ip_address(ip_a)
  992. b = ipaddress.ip_address(ip_b)
  993. if a.version != b.version:
  994. raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b))
  995. if mask_bits > a.max_prefixlen:
  996. logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
  997. mask_bits = a.max_prefixlen
  998. if mask_bits < 0:
  999. logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
  1000. mask_bits = 0
  1001. a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False)
  1002. return b in a_net
  1003. # is this fallback's IPv4 address (dirip) in the same netblock as other's
  1004. # IPv4 address?
  1005. # mask_bits is the size of the netblock
  1006. def ipv4_netblocks_equal(self, other, mask_bits):
  1007. return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits)
  1008. # is this fallback's IPv6 address (ipv6addr) in the same netblock as
  1009. # other's IPv6 address?
  1010. # Returns False if either fallback has no IPv6 address
  1011. # mask_bits is the size of the netblock
  1012. def ipv6_netblocks_equal(self, other, mask_bits):
  1013. if not self.has_ipv6() or not other.has_ipv6():
  1014. return False
  1015. return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits)
  1016. # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort?
  1017. def dirport_equal(self, other):
  1018. return self.dirport == other.dirport
  1019. # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort?
  1020. def ipv4_orport_equal(self, other):
  1021. return self.orport == other.orport
  1022. # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort?
  1023. # Returns False if either fallback has no IPv6 address
  1024. def ipv6_orport_equal(self, other):
  1025. if not self.has_ipv6() or not other.has_ipv6():
  1026. return False
  1027. return self.ipv6orport == other.ipv6orport
  1028. # does this fallback have the same DirPort, IPv4 ORPort, or
  1029. # IPv6 ORPort as other?
  1030. # Ignores IPv6 ORPort if either fallback has no IPv6 address
  1031. def port_equal(self, other):
  1032. return (self.dirport_equal(other) or self.ipv4_orport_equal(other)
  1033. or self.ipv6_orport_equal(other))
  1034. # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present)
  1035. def port_list(self):
  1036. ports = [self.dirport, self.orport]
  1037. if self.has_ipv6() and not self.ipv6orport in ports:
  1038. ports.append(self.ipv6orport)
  1039. return ports
  1040. # does this fallback share a port with other, regardless of whether the
  1041. # port types match?
  1042. # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80,
  1043. # return True
  1044. def port_shared(self, other):
  1045. for p in self.port_list():
  1046. if p in other.port_list():
  1047. return True
  1048. return False
  1049. # log how long it takes to download a consensus from dirip:dirport
  1050. # returns True if the download failed, False if it succeeded within max_time
  1051. @staticmethod
  1052. def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
  1053. max_time):
  1054. download_failed = False
  1055. # some directory mirrors respond to requests in ways that hang python
  1056. # sockets, which is why we log this line here
  1057. logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
  1058. 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
  1059. nickname, dirip, dirport, fingerprint)
  1060. # there appears to be about 1 second of overhead when comparing stem's
  1061. # internal trace time and the elapsed time calculated here
  1062. TIMEOUT_SLOP = 1.0
  1063. start = datetime.datetime.utcnow()
  1064. try:
  1065. consensus = get_consensus(
  1066. endpoints = [(dirip, dirport)],
  1067. timeout = (max_time + TIMEOUT_SLOP),
  1068. validate = True,
  1069. retries = 0,
  1070. fall_back_to_authority = False,
  1071. document_handler = DocumentHandler.BARE_DOCUMENT,
  1072. microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
  1073. ).run()[0]
  1074. end = datetime.datetime.utcnow()
  1075. time_since_expiry = (end - consensus.valid_until).total_seconds()
  1076. except Exception, stem_error:
  1077. end = datetime.datetime.utcnow()
  1078. log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
  1079. stem_error)
  1080. status = 'error: "%s"' % (stem_error)
  1081. level = logging.WARNING
  1082. download_failed = True
  1083. elapsed = (end - start).total_seconds()
  1084. if download_failed:
  1085. # keep the error failure status, and avoid using the variables
  1086. pass
  1087. elif elapsed > max_time:
  1088. status = 'too slow'
  1089. level = logging.WARNING
  1090. download_failed = True
  1091. elif (time_since_expiry > 0):
  1092. status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
  1093. if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
  1094. status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
  1095. level = logging.INFO
  1096. else:
  1097. status += ', invalid'
  1098. level = logging.WARNING
  1099. download_failed = True
  1100. else:
  1101. status = 'ok'
  1102. level = logging.DEBUG
  1103. logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
  1104. 'max download time %0.1fs.', elapsed, status, nickname,
  1105. dirip, dirport, fingerprint, max_time)
  1106. return download_failed
  1107. # does this fallback download the consensus fast enough?
  1108. def check_fallback_download_consensus(self):
  1109. # include the relay if we're not doing a check, or we can't check (IPv6)
  1110. ipv4_failed = False
  1111. ipv6_failed = False
  1112. if PERFORM_IPV4_DIRPORT_CHECKS:
  1113. ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
  1114. self.dirport,
  1115. self._data['nickname'],
  1116. self._fpr,
  1117. CONSENSUS_DOWNLOAD_SPEED_MAX)
  1118. if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
  1119. # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
  1120. ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
  1121. self.dirport,
  1122. self._data['nickname'],
  1123. self._fpr,
  1124. CONSENSUS_DOWNLOAD_SPEED_MAX)
  1125. return ((not ipv4_failed) and (not ipv6_failed))
  1126. # if this fallback has not passed a download check, try it again,
  1127. # and record the result, available in get_fallback_download_consensus
  1128. def try_fallback_download_consensus(self):
  1129. if not self.get_fallback_download_consensus():
  1130. self._data['download_check'] = self.check_fallback_download_consensus()
  1131. # did this fallback pass the download check?
  1132. def get_fallback_download_consensus(self):
  1133. # if we're not performing checks, return True
  1134. if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
  1135. return True
  1136. # if we are performing checks, but haven't done one, return False
  1137. if not self._data.has_key('download_check'):
  1138. return False
  1139. return self._data['download_check']
  1140. # output an optional header comment and info for this fallback
  1141. # try_fallback_download_consensus before calling this
  1142. def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
  1143. s = ''
  1144. if OUTPUT_COMMENTS:
  1145. s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
  1146. # if the download speed is ok, output a C string
  1147. # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
  1148. if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
  1149. s += self.fallbackdir_info(self.get_fallback_download_consensus())
  1150. return s
  1151. # output a header comment for this fallback
  1152. def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
  1153. # /*
  1154. # nickname
  1155. # flags
  1156. # adjusted bandwidth, consensus weight
  1157. # [contact]
  1158. # [identical contact counts]
  1159. # */
  1160. # Multiline C comment
  1161. s = '/*'
  1162. s += '\n'
  1163. s += cleanse_c_multiline_comment(self._data['nickname'])
  1164. s += '\n'
  1165. s += 'Flags: '
  1166. s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
  1167. s += '\n'
  1168. # this is an adjusted bandwidth, see calculate_measured_bandwidth()
  1169. bandwidth = self._data['measured_bandwidth']
  1170. weight = self._data['consensus_weight']
  1171. s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
  1172. bandwidth/(1024.0*1024.0),
  1173. weight)
  1174. s += '\n'
  1175. if self._data['contact'] is not None:
  1176. s += cleanse_c_multiline_comment(self._data['contact'])
  1177. if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT:
  1178. fallback_count = len([f for f in fallbacks
  1179. if f._data['contact'] == self._data['contact']])
  1180. if fallback_count > 1:
  1181. s += '\n'
  1182. s += '%d identical contacts listed' % (fallback_count)
  1183. if CONTACT_BLACKLIST_COUNT:
  1184. prefilter_count = len([f for f in prefilter_fallbacks
  1185. if f._data['contact'] == self._data['contact']])
  1186. filter_count = prefilter_count - fallback_count
  1187. if filter_count > 0:
  1188. if fallback_count > 1:
  1189. s += ' '
  1190. else:
  1191. s += '\n'
  1192. s += '%d blacklisted' % (filter_count)
  1193. s += '\n'
  1194. s += '*/'
  1195. s += '\n'
  1196. return s
  1197. # output the fallback info C string for this fallback
  1198. # this is the text that would go after FallbackDir in a torrc
  1199. # if this relay failed the download test and we OUTPUT_COMMENTS,
  1200. # comment-out the returned string
  1201. def fallbackdir_info(self, dl_speed_ok):
  1202. # "address:dirport orport=port id=fingerprint"
  1203. # (insert additional madatory fields here)
  1204. # "[ipv6=addr:orport]"
  1205. # (insert additional optional fields here)
  1206. # /* nickname=name */
  1207. # /* extrainfo={0,1} */
  1208. # (insert additional comment fields here)
  1209. # /* ===== */
  1210. # ,
  1211. #
  1212. # Do we want a C string, or a commented-out string?
  1213. c_string = dl_speed_ok
  1214. comment_string = not dl_speed_ok and OUTPUT_COMMENTS
  1215. # If we don't want either kind of string, bail
  1216. if not c_string and not comment_string:
  1217. return ''
  1218. s = ''
  1219. # Comment out the fallback directory entry if it's too slow
  1220. # See the debug output for which address and port is failing
  1221. if comment_string:
  1222. s += '/* Consensus download failed or was too slow:\n'
  1223. # Multi-Line C string with trailing comma (part of a string list)
  1224. # This makes it easier to diff the file, and remove IPv6 lines using grep
  1225. # Integers don't need escaping
  1226. s += '"%s orport=%d id=%s"'%(
  1227. cleanse_c_string(self._data['dir_address']),
  1228. self.orport,
  1229. cleanse_c_string(self._fpr))
  1230. s += '\n'
  1231. # (insert additional madatory fields here)
  1232. if self.has_ipv6():
  1233. s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
  1234. s += '\n'
  1235. # (insert additional optional fields here)
  1236. if not comment_string:
  1237. s += '/* '
  1238. s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
  1239. if not comment_string:
  1240. s += ' */'
  1241. s += '\n'
  1242. # if we know that the fallback is an extrainfo cache, flag it
  1243. # and if we don't know, assume it is not
  1244. if not comment_string:
  1245. s += '/* '
  1246. s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
  1247. if not comment_string:
  1248. s += ' */'
  1249. s += '\n'
  1250. # (insert additional comment fields here)
  1251. # The terminator and comma must be the last line in each fallback entry
  1252. if not comment_string:
  1253. s += '/* '
  1254. s += SECTION_SEPARATOR_BASE
  1255. if not comment_string:
  1256. s += ' */'
  1257. s += '\n'
  1258. s += ','
  1259. if comment_string:
  1260. s += '\n'
  1261. s += '*/'
  1262. return s
  1263. ## Fallback Candidate List Class
  1264. class CandidateList(dict):
  1265. def __init__(self):
  1266. pass
  1267. def _add_relay(self, details):
  1268. if not 'dir_address' in details: return
  1269. c = Candidate(details)
  1270. self[ c.get_fingerprint() ] = c
  1271. def _add_uptime(self, uptime):
  1272. try:
  1273. fpr = uptime['fingerprint']
  1274. except KeyError:
  1275. raise Exception("Document has no fingerprint field.")
  1276. try:
  1277. c = self[fpr]
  1278. except KeyError:
  1279. logging.debug('Got unknown relay %s in uptime document.'%(fpr,))
  1280. return
  1281. c.add_uptime(uptime)
  1282. def _add_details(self):
  1283. logging.debug('Loading details document.')
  1284. d = fetch('details',
  1285. fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
  1286. 'consensus_weight,advertised_bandwidth,or_addresses,' +
  1287. 'dir_address,recommended_version,flags,effective_family,' +
  1288. 'platform'))
  1289. logging.debug('Loading details document done.')
  1290. if not 'relays' in d: raise Exception("No relays found in document.")
  1291. for r in d['relays']: self._add_relay(r)
  1292. def _add_uptimes(self):
  1293. logging.debug('Loading uptime document.')
  1294. d = fetch('uptime')
  1295. logging.debug('Loading uptime document done.')
  1296. if not 'relays' in d: raise Exception("No relays found in document.")
  1297. for r in d['relays']: self._add_uptime(r)
  1298. def add_relays(self):
  1299. self._add_details()
  1300. self._add_uptimes()
  1301. def count_guards(self):
  1302. guard_count = 0
  1303. for fpr in self.keys():
  1304. if self[fpr].is_guard():
  1305. guard_count += 1
  1306. return guard_count
  1307. # Find fallbacks that fit the uptime, stability, and flags criteria,
  1308. # and make an array of them in self.fallbacks
  1309. def compute_fallbacks(self):
  1310. self.fallbacks = map(lambda x: self[x],
  1311. filter(lambda x: self[x].is_candidate(),
  1312. self.keys()))
  1313. # sort fallbacks by their consensus weight to advertised bandwidth factor,
  1314. # lowest to highest
  1315. # used to find the median cw_to_bw_factor()
  1316. def sort_fallbacks_by_cw_to_bw_factor(self):
  1317. self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor())
  1318. # sort fallbacks by their measured bandwidth, highest to lowest
  1319. # calculate_measured_bandwidth before calling this
  1320. # this is useful for reviewing candidates in priority order
  1321. def sort_fallbacks_by_measured_bandwidth(self):
  1322. self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
  1323. reverse=True)
  1324. # sort fallbacks by the data field data_field, lowest to highest
  1325. def sort_fallbacks_by(self, data_field):
  1326. self.fallbacks.sort(key=lambda f: f._data[data_field])
  1327. @staticmethod
  1328. def load_relaylist(file_obj):
  1329. """ Read each line in the file, and parse it like a FallbackDir line:
  1330. an IPv4 address and optional port:
  1331. <IPv4 address>:<port>
  1332. which are parsed into dictionary entries:
  1333. ipv4=<IPv4 address>
  1334. dirport=<port>
  1335. followed by a series of key=value entries:
  1336. orport=<port>
  1337. id=<fingerprint>
  1338. ipv6=<IPv6 address>:<IPv6 orport>
  1339. each line's key/value pairs are placed in a dictonary,
  1340. (of string -> string key/value pairs),
  1341. and these dictionaries are placed in an array.
  1342. comments start with # and are ignored """
  1343. file_data = file_obj['data']
  1344. file_name = file_obj['name']
  1345. relaylist = []
  1346. if file_data is None:
  1347. return relaylist
  1348. for line in file_data.split('\n'):
  1349. relay_entry = {}
  1350. # ignore comments
  1351. line_comment_split = line.split('#')
  1352. line = line_comment_split[0]
  1353. # cleanup whitespace
  1354. line = cleanse_whitespace(line)
  1355. line = line.strip()
  1356. if len(line) == 0:
  1357. continue
  1358. for item in line.split(' '):
  1359. item = item.strip()
  1360. if len(item) == 0:
  1361. continue
  1362. key_value_split = item.split('=')
  1363. kvl = len(key_value_split)
  1364. if kvl < 1 or kvl > 2:
  1365. print '#error Bad %s item: %s, format is key=value.'%(
  1366. file_name, item)
  1367. if kvl == 1:
  1368. # assume that entries without a key are the ipv4 address,
  1369. # perhaps with a dirport
  1370. ipv4_maybe_dirport = key_value_split[0]
  1371. ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':')
  1372. dirl = len(ipv4_maybe_dirport_split)
  1373. if dirl < 1 or dirl > 2:
  1374. print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%(
  1375. file_name, item)
  1376. if dirl >= 1:
  1377. relay_entry['ipv4'] = ipv4_maybe_dirport_split[0]
  1378. if dirl == 2:
  1379. relay_entry['dirport'] = ipv4_maybe_dirport_split[1]
  1380. elif kvl == 2:
  1381. relay_entry[key_value_split[0]] = key_value_split[1]
  1382. relaylist.append(relay_entry)
  1383. return relaylist
  1384. # apply the fallback whitelist and blacklist
  1385. def apply_filter_lists(self, whitelist_obj, blacklist_obj):
  1386. excluded_count = 0
  1387. logging.debug('Applying whitelist and blacklist.')
  1388. # parse the whitelist and blacklist
  1389. whitelist = self.load_relaylist(whitelist_obj)
  1390. blacklist = self.load_relaylist(blacklist_obj)
  1391. filtered_fallbacks = []
  1392. for f in self.fallbacks:
  1393. in_whitelist = f.is_in_whitelist(whitelist)
  1394. in_blacklist = f.is_in_blacklist(blacklist)
  1395. if in_whitelist and in_blacklist:
  1396. if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES:
  1397. # exclude
  1398. excluded_count += 1
  1399. logging.warning('Excluding %s: in both blacklist and whitelist.',
  1400. f._fpr)
  1401. else:
  1402. # include
  1403. filtered_fallbacks.append(f)
  1404. elif in_whitelist:
  1405. # include
  1406. filtered_fallbacks.append(f)
  1407. elif in_blacklist:
  1408. # exclude
  1409. excluded_count += 1
  1410. log_excluded('Excluding %s: in blacklist.', f._fpr)
  1411. else:
  1412. if INCLUDE_UNLISTED_ENTRIES:
  1413. # include
  1414. filtered_fallbacks.append(f)
  1415. else:
  1416. # exclude
  1417. excluded_count += 1
  1418. log_excluded('Excluding %s: in neither blacklist nor whitelist.',
  1419. f._fpr)
  1420. self.fallbacks = filtered_fallbacks
  1421. return excluded_count
  1422. @staticmethod
  1423. def summarise_filters(initial_count, excluded_count):
  1424. return '/* Whitelist & blacklist excluded %d of %d candidates. */'%(
  1425. excluded_count, initial_count)
  1426. # calculate each fallback's measured bandwidth based on the median
  1427. # consensus weight to advertised bandwidth ratio
  1428. def calculate_measured_bandwidth(self):
  1429. self.sort_fallbacks_by_cw_to_bw_factor()
  1430. median_fallback = self.fallback_median(True)
  1431. if median_fallback is not None:
  1432. median_cw_to_bw_factor = median_fallback.cw_to_bw_factor()
  1433. else:
  1434. # this will never be used, because there are no fallbacks
  1435. median_cw_to_bw_factor = None
  1436. for f in self.fallbacks:
  1437. f.set_measured_bandwidth(median_cw_to_bw_factor)
  1438. # remove relays with low measured bandwidth from the fallback list
  1439. # calculate_measured_bandwidth for each relay before calling this
  1440. def remove_low_bandwidth_relays(self):
  1441. if MIN_BANDWIDTH is None:
  1442. return
  1443. above_min_bw_fallbacks = []
  1444. for f in self.fallbacks:
  1445. if f._data['measured_bandwidth'] >= MIN_BANDWIDTH:
  1446. above_min_bw_fallbacks.append(f)
  1447. else:
  1448. # the bandwidth we log here is limited by the relay's consensus weight
  1449. # as well as its adverttised bandwidth. See set_measured_bandwidth
  1450. # for details
  1451. log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
  1452. 'must be at least %.1fMByte/s', f._fpr,
  1453. f._data['measured_bandwidth']/(1024.0*1024.0),
  1454. MIN_BANDWIDTH/(1024.0*1024.0))
  1455. self.fallbacks = above_min_bw_fallbacks
  1456. # the minimum fallback in the list
  1457. # call one of the sort_fallbacks_* functions before calling this
  1458. def fallback_min(self):
  1459. if len(self.fallbacks) > 0:
  1460. return self.fallbacks[-1]
  1461. else:
  1462. return None
  1463. # the median fallback in the list
  1464. # call one of the sort_fallbacks_* functions before calling this
  1465. def fallback_median(self, require_advertised_bandwidth):
  1466. # use the low-median when there are an evan number of fallbacks,
  1467. # for consistency with the bandwidth authorities
  1468. if len(self.fallbacks) > 0:
  1469. median_position = (len(self.fallbacks) - 1) / 2
  1470. if not require_advertised_bandwidth:
  1471. return self.fallbacks[median_position]
  1472. # if we need advertised_bandwidth but this relay doesn't have it,
  1473. # move to a fallback with greater consensus weight until we find one
  1474. while not self.fallbacks[median_position]._data['advertised_bandwidth']:
  1475. median_position += 1
  1476. if median_position >= len(self.fallbacks):
  1477. return None
  1478. return self.fallbacks[median_position]
  1479. else:
  1480. return None
  1481. # the maximum fallback in the list
  1482. # call one of the sort_fallbacks_* functions before calling this
  1483. def fallback_max(self):
  1484. if len(self.fallbacks) > 0:
  1485. return self.fallbacks[0]
  1486. else:
  1487. return None
  1488. # return a new bag suitable for storing attributes
  1489. @staticmethod
  1490. def attribute_new():
  1491. return dict()
  1492. # get the count of attribute in attribute_bag
  1493. # if attribute is None or the empty string, return 0
  1494. @staticmethod
  1495. def attribute_count(attribute, attribute_bag):
  1496. if attribute is None or attribute == '':
  1497. return 0
  1498. if attribute not in attribute_bag:
  1499. return 0
  1500. return attribute_bag[attribute]
  1501. # does attribute_bag contain more than max_count instances of attribute?
  1502. # if so, return False
  1503. # if not, return True
  1504. # if attribute is None or the empty string, or max_count is invalid,
  1505. # always return True
  1506. @staticmethod
  1507. def attribute_allow(attribute, attribute_bag, max_count=1):
  1508. if attribute is None or attribute == '' or max_count <= 0:
  1509. return True
  1510. elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
  1511. return False
  1512. else:
  1513. return True
  1514. # add attribute to attribute_bag, incrementing the count if it is already
  1515. # present
  1516. # if attribute is None or the empty string, or count is invalid,
  1517. # do nothing
  1518. @staticmethod
  1519. def attribute_add(attribute, attribute_bag, count=1):
  1520. if attribute is None or attribute == '' or count <= 0:
  1521. pass
  1522. attribute_bag.setdefault(attribute, 0)
  1523. attribute_bag[attribute] += count
  1524. # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
  1525. # and per IPv6 address
  1526. # there is only one IPv4 address on each fallback: the IPv4 DirPort address
  1527. # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
  1528. # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
  1529. # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
  1530. # (clients only use the IPv6 ORPort)
  1531. # if there is no IPv6 address, only the IPv4 address is checked
  1532. # return the number of candidates we excluded
  1533. def limit_fallbacks_same_ip(self):
  1534. ip_limit_fallbacks = []
  1535. ip_list = CandidateList.attribute_new()
  1536. for f in self.fallbacks:
  1537. if (CandidateList.attribute_allow(f.dirip, ip_list,
  1538. MAX_FALLBACKS_PER_IPV4)
  1539. and CandidateList.attribute_allow(f.ipv6addr, ip_list,
  1540. MAX_FALLBACKS_PER_IPV6)):
  1541. ip_limit_fallbacks.append(f)
  1542. CandidateList.attribute_add(f.dirip, ip_list)
  1543. if f.has_ipv6():
  1544. CandidateList.attribute_add(f.ipv6addr, ip_list)
  1545. elif not CandidateList.attribute_allow(f.dirip, ip_list,
  1546. MAX_FALLBACKS_PER_IPV4):
  1547. log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
  1548. %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
  1549. f.dirip))
  1550. elif (f.has_ipv6() and
  1551. not CandidateList.attribute_allow(f.ipv6addr, ip_list,
  1552. MAX_FALLBACKS_PER_IPV6)):
  1553. log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
  1554. %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
  1555. ip_list),
  1556. f.ipv6addr))
  1557. original_count = len(self.fallbacks)
  1558. self.fallbacks = ip_limit_fallbacks
  1559. return original_count - len(self.fallbacks)
  1560. # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
  1561. # ContactInfo
  1562. # if there is no ContactInfo, allow the fallback
  1563. # this check can be gamed by providing no ContactInfo, or by setting the
  1564. # ContactInfo to match another fallback
  1565. # However, given the likelihood that relays with the same ContactInfo will
  1566. # go down at similar times, its usefulness outweighs the risk
  1567. def limit_fallbacks_same_contact(self):
  1568. contact_limit_fallbacks = []
  1569. contact_list = CandidateList.attribute_new()
  1570. for f in self.fallbacks:
  1571. if CandidateList.attribute_allow(f._data['contact'], contact_list,
  1572. MAX_FALLBACKS_PER_CONTACT):
  1573. contact_limit_fallbacks.append(f)
  1574. CandidateList.attribute_add(f._data['contact'], contact_list)
  1575. else:
  1576. log_excluded(
  1577. 'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
  1578. %(f._fpr, CandidateList.attribute_count(f._data['contact'],
  1579. contact_list),
  1580. f._data['contact']))
  1581. original_count = len(self.fallbacks)
  1582. self.fallbacks = contact_limit_fallbacks
  1583. return original_count - len(self.fallbacks)
  1584. # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
  1585. # family
  1586. # if there is no family, allow the fallback
  1587. # we use effective family, which ensures mutual family declarations
  1588. # but the check can be gamed by not declaring a family at all
  1589. # if any indirect families exist, the result depends on the order in which
  1590. # fallbacks are sorted in the list
  1591. def limit_fallbacks_same_family(self):
  1592. family_limit_fallbacks = []
  1593. fingerprint_list = CandidateList.attribute_new()
  1594. for f in self.fallbacks:
  1595. if CandidateList.attribute_allow(f._fpr, fingerprint_list,
  1596. MAX_FALLBACKS_PER_FAMILY):
  1597. family_limit_fallbacks.append(f)
  1598. CandidateList.attribute_add(f._fpr, fingerprint_list)
  1599. for family_fingerprint in f._data['effective_family']:
  1600. CandidateList.attribute_add(family_fingerprint, fingerprint_list)
  1601. else:
  1602. # we already have a fallback with this fallback in its effective
  1603. # family
  1604. log_excluded(
  1605. 'Eliminated %s: already have %d fallback(s) in effective family'
  1606. %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
  1607. original_count = len(self.fallbacks)
  1608. self.fallbacks = family_limit_fallbacks
  1609. return original_count - len(self.fallbacks)
  1610. # try once to get the descriptors for fingerprint_list using stem
  1611. # returns an empty list on exception
  1612. @staticmethod
  1613. def get_fallback_descriptors_once(fingerprint_list):
  1614. desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
  1615. return desc_list
  1616. # try up to max_retries times to get the descriptors for fingerprint_list
  1617. # using stem. Stops retrying when all descriptors have been retrieved.
  1618. # returns a list containing the descriptors that were retrieved
  1619. @staticmethod
  1620. def get_fallback_descriptors(fingerprint_list, max_retries=5):
  1621. # we can't use stem's retries=, because we want to support more than 96
  1622. # descriptors
  1623. #
  1624. # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
  1625. max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
  1626. remaining_list = fingerprint_list
  1627. desc_list = []
  1628. for _ in xrange(max_retries):
  1629. if len(remaining_list) == 0:
  1630. break
  1631. new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
  1632. for d in new_desc_list:
  1633. try:
  1634. remaining_list.remove(d.fingerprint)
  1635. except ValueError:
  1636. # warn and ignore if a directory mirror returned a bad descriptor
  1637. logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
  1638. d.fingerprint)
  1639. continue
  1640. desc_list.append(d)
  1641. return desc_list
  1642. # find the fallbacks that cache extra-info documents
  1643. # Onionoo doesn't know this, so we have to use stem
  1644. def mark_extra_info_caches(self):
  1645. fingerprint_list = [ f._fpr for f in self.fallbacks ]
  1646. logging.info("Downloading fallback descriptors to find extra-info caches")
  1647. desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
  1648. for d in desc_list:
  1649. self[d.fingerprint]._extra_info_cache = d.extra_info_cache
  1650. missing_descriptor_list = [ f._fpr for f in self.fallbacks
  1651. if f._extra_info_cache is None ]
  1652. for f in missing_descriptor_list:
  1653. logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
  1654. # try a download check on each fallback candidate in order
  1655. # stop after max_count successful downloads
  1656. # but don't remove any candidates from the array
  1657. def try_download_consensus_checks(self, max_count):
  1658. dl_ok_count = 0
  1659. for f in self.fallbacks:
  1660. f.try_fallback_download_consensus()
  1661. if f.get_fallback_download_consensus():
  1662. # this fallback downloaded a consensus ok
  1663. dl_ok_count += 1
  1664. if dl_ok_count >= max_count:
  1665. # we have enough fallbacks
  1666. return
  1667. # put max_count successful candidates in the fallbacks array:
  1668. # - perform download checks on each fallback candidate
  1669. # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
  1670. # - eliminate failed candidates
  1671. # - if there are more than max_count candidates, eliminate lowest bandwidth
  1672. # - if there are fewer than max_count candidates, leave only successful
  1673. # Return the number of fallbacks that failed the consensus check
  1674. def perform_download_consensus_checks(self, max_count):
  1675. self.sort_fallbacks_by_measured_bandwidth()
  1676. self.try_download_consensus_checks(max_count)
  1677. if CONSENSUS_DOWNLOAD_RETRY:
  1678. # try unsuccessful candidates again
  1679. # we could end up with more than max_count successful candidates here
  1680. self.try_download_consensus_checks(max_count)
  1681. # now we have at least max_count successful candidates,
  1682. # or we've tried them all
  1683. original_count = len(self.fallbacks)
  1684. self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
  1685. self.fallbacks)
  1686. # some of these failed the check, others skipped the check,
  1687. # if we already had enough successful downloads
  1688. failed_count = original_count - len(self.fallbacks)
  1689. self.fallbacks = self.fallbacks[:max_count]
  1690. return failed_count
  1691. # return a string that describes a/b as a percentage
  1692. @staticmethod
  1693. def describe_percentage(a, b):
  1694. if b != 0:
  1695. return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b)
  1696. else:
  1697. # technically, 0/0 is undefined, but 0.0% is a sensible result
  1698. return '%d/%d = %.0f%%'%(a, b, 0.0)
  1699. # return a dictionary of lists of fallbacks by IPv4 netblock
  1700. # the dictionary is keyed by the fingerprint of an arbitrary fallback
  1701. # in each netblock
  1702. # mask_bits is the size of the netblock
  1703. def fallbacks_by_ipv4_netblock(self, mask_bits):
  1704. netblocks = {}
  1705. for f in self.fallbacks:
  1706. found_netblock = False
  1707. for b in netblocks.keys():
  1708. # we found an existing netblock containing this fallback
  1709. if f.ipv4_netblocks_equal(self[b], mask_bits):
  1710. # add it to the list
  1711. netblocks[b].append(f)
  1712. found_netblock = True
  1713. break
  1714. # make a new netblock based on this fallback's fingerprint
  1715. if not found_netblock:
  1716. netblocks[f._fpr] = [f]
  1717. return netblocks
  1718. # return a dictionary of lists of fallbacks by IPv6 netblock
  1719. # where mask_bits is the size of the netblock
  1720. def fallbacks_by_ipv6_netblock(self, mask_bits):
  1721. netblocks = {}
  1722. for f in self.fallbacks:
  1723. # skip fallbacks without IPv6 addresses
  1724. if not f.has_ipv6():
  1725. continue
  1726. found_netblock = False
  1727. for b in netblocks.keys():
  1728. # we found an existing netblock containing this fallback
  1729. if f.ipv6_netblocks_equal(self[b], mask_bits):
  1730. # add it to the list
  1731. netblocks[b].append(f)
  1732. found_netblock = True
  1733. break
  1734. # make a new netblock based on this fallback's fingerprint
  1735. if not found_netblock:
  1736. netblocks[f._fpr] = [f]
  1737. return netblocks
  1738. # log a message about the proportion of fallbacks in each IPv4 netblock,
  1739. # where mask_bits is the size of the netblock
  1740. def describe_fallback_ipv4_netblock_mask(self, mask_bits):
  1741. fallback_count = len(self.fallbacks)
  1742. shared_netblock_fallback_count = 0
  1743. most_frequent_netblock = None
  1744. netblocks = self.fallbacks_by_ipv4_netblock(mask_bits)
  1745. for b in netblocks.keys():
  1746. if len(netblocks[b]) > 1:
  1747. # how many fallbacks are in a netblock with other fallbacks?
  1748. shared_netblock_fallback_count += len(netblocks[b])
  1749. # what's the netblock with the most fallbacks?
  1750. if (most_frequent_netblock is None
  1751. or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
  1752. most_frequent_netblock = b
  1753. logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits))
  1754. for f in netblocks[b]:
  1755. logging.debug('%s - %s', f.dirip, f._fpr)
  1756. if most_frequent_netblock is not None:
  1757. logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%(
  1758. CandidateList.describe_percentage(
  1759. len(netblocks[most_frequent_netblock]),
  1760. fallback_count),
  1761. mask_bits,
  1762. self[most_frequent_netblock].dirip))
  1763. if shared_netblock_fallback_count > 0:
  1764. logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' +
  1765. 'fallbacks')%(CandidateList.describe_percentage(
  1766. shared_netblock_fallback_count,
  1767. fallback_count),
  1768. mask_bits))
  1769. # log a message about the proportion of fallbacks in each IPv6 netblock,
  1770. # where mask_bits is the size of the netblock
  1771. def describe_fallback_ipv6_netblock_mask(self, mask_bits):
  1772. fallback_count = len(self.fallbacks_with_ipv6())
  1773. shared_netblock_fallback_count = 0
  1774. most_frequent_netblock = None
  1775. netblocks = self.fallbacks_by_ipv6_netblock(mask_bits)
  1776. for b in netblocks.keys():
  1777. if len(netblocks[b]) > 1:
  1778. # how many fallbacks are in a netblock with other fallbacks?
  1779. shared_netblock_fallback_count += len(netblocks[b])
  1780. # what's the netblock with the most fallbacks?
  1781. if (most_frequent_netblock is None
  1782. or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
  1783. most_frequent_netblock = b
  1784. logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits))
  1785. for f in netblocks[b]:
  1786. logging.debug('%s - %s', f.ipv6addr, f._fpr)
  1787. if most_frequent_netblock is not None:
  1788. logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%(
  1789. CandidateList.describe_percentage(
  1790. len(netblocks[most_frequent_netblock]),
  1791. fallback_count),
  1792. mask_bits,
  1793. self[most_frequent_netblock].ipv6addr))
  1794. if shared_netblock_fallback_count > 0:
  1795. logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' +
  1796. 'fallbacks')%(CandidateList.describe_percentage(
  1797. shared_netblock_fallback_count,
  1798. fallback_count),
  1799. mask_bits))
  1800. # log a message about the proportion of fallbacks in each IPv4 /8, /16,
  1801. # and /24
  1802. def describe_fallback_ipv4_netblocks(self):
  1803. # this doesn't actually tell us anything useful
  1804. #self.describe_fallback_ipv4_netblock_mask(8)
  1805. self.describe_fallback_ipv4_netblock_mask(16)
  1806. #self.describe_fallback_ipv4_netblock_mask(24)
  1807. # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
  1808. # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
  1809. # https://www.iana.org/assignments/ipv6-unicast-address-assignments/
  1810. def describe_fallback_ipv6_netblocks(self):
  1811. # these don't actually tell us anything useful
  1812. #self.describe_fallback_ipv6_netblock_mask(12)
  1813. #self.describe_fallback_ipv6_netblock_mask(23)
  1814. self.describe_fallback_ipv6_netblock_mask(32)
  1815. #self.describe_fallback_ipv6_netblock_mask(48)
  1816. self.describe_fallback_ipv6_netblock_mask(64)
  1817. # log a message about the proportion of fallbacks in each IPv4 and IPv6
  1818. # netblock
  1819. def describe_fallback_netblocks(self):
  1820. self.describe_fallback_ipv4_netblocks()
  1821. self.describe_fallback_ipv6_netblocks()
  1822. # return a list of fallbacks which are on the IPv4 ORPort port
  1823. def fallbacks_on_ipv4_orport(self, port):
  1824. return filter(lambda x: x.orport == port, self.fallbacks)
  1825. # return a list of fallbacks which are on the IPv6 ORPort port
  1826. def fallbacks_on_ipv6_orport(self, port):
  1827. return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6())
  1828. # return a list of fallbacks which are on the DirPort port
  1829. def fallbacks_on_dirport(self, port):
  1830. return filter(lambda x: x.dirport == port, self.fallbacks)
  1831. # log a message about the proportion of fallbacks on IPv4 ORPort port
  1832. # and return that count
  1833. def describe_fallback_ipv4_orport(self, port):
  1834. port_count = len(self.fallbacks_on_ipv4_orport(port))
  1835. fallback_count = len(self.fallbacks)
  1836. logging.warning('%s of fallbacks are on IPv4 ORPort %d'%(
  1837. CandidateList.describe_percentage(port_count,
  1838. fallback_count),
  1839. port))
  1840. return port_count
  1841. # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port
  1842. # and return that count
  1843. def describe_fallback_ipv6_orport(self, port):
  1844. port_count = len(self.fallbacks_on_ipv6_orport(port))
  1845. fallback_count = len(self.fallbacks_with_ipv6())
  1846. logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%(
  1847. CandidateList.describe_percentage(port_count,
  1848. fallback_count),
  1849. port))
  1850. return port_count
  1851. # log a message about the proportion of fallbacks on DirPort port
  1852. # and return that count
  1853. def describe_fallback_dirport(self, port):
  1854. port_count = len(self.fallbacks_on_dirport(port))
  1855. fallback_count = len(self.fallbacks)
  1856. logging.warning('%s of fallbacks are on DirPort %d'%(
  1857. CandidateList.describe_percentage(port_count,
  1858. fallback_count),
  1859. port))
  1860. return port_count
  1861. # log a message about the proportion of fallbacks on each dirport,
  1862. # each IPv4 orport, and each IPv6 orport
  1863. def describe_fallback_ports(self):
  1864. fallback_count = len(self.fallbacks)
  1865. ipv4_or_count = fallback_count
  1866. ipv4_or_count -= self.describe_fallback_ipv4_orport(443)
  1867. ipv4_or_count -= self.describe_fallback_ipv4_orport(9001)
  1868. logging.warning('%s of fallbacks are on other IPv4 ORPorts'%(
  1869. CandidateList.describe_percentage(ipv4_or_count,
  1870. fallback_count)))
  1871. ipv6_fallback_count = len(self.fallbacks_with_ipv6())
  1872. ipv6_or_count = ipv6_fallback_count
  1873. ipv6_or_count -= self.describe_fallback_ipv6_orport(443)
  1874. ipv6_or_count -= self.describe_fallback_ipv6_orport(9001)
  1875. logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%(
  1876. CandidateList.describe_percentage(ipv6_or_count,
  1877. ipv6_fallback_count)))
  1878. dir_count = fallback_count
  1879. dir_count -= self.describe_fallback_dirport(80)
  1880. dir_count -= self.describe_fallback_dirport(9030)
  1881. logging.warning('%s of fallbacks are on other DirPorts'%(
  1882. CandidateList.describe_percentage(dir_count,
  1883. fallback_count)))
  1884. # return a list of fallbacks which cache extra-info documents
  1885. def fallbacks_with_extra_info_cache(self):
  1886. return filter(lambda x: x._extra_info_cache, self.fallbacks)
  1887. # log a message about the proportion of fallbacks that cache extra-info docs
  1888. def describe_fallback_extra_info_caches(self):
  1889. extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
  1890. fallback_count = len(self.fallbacks)
  1891. logging.warning('%s of fallbacks cache extra-info documents'%(
  1892. CandidateList.describe_percentage(extra_info_falback_count,
  1893. fallback_count)))
  1894. # return a list of fallbacks which have the Exit flag
  1895. def fallbacks_with_exit(self):
  1896. return filter(lambda x: x.is_exit(), self.fallbacks)
  1897. # log a message about the proportion of fallbacks with an Exit flag
  1898. def describe_fallback_exit_flag(self):
  1899. exit_falback_count = len(self.fallbacks_with_exit())
  1900. fallback_count = len(self.fallbacks)
  1901. logging.warning('%s of fallbacks have the Exit flag'%(
  1902. CandidateList.describe_percentage(exit_falback_count,
  1903. fallback_count)))
  1904. # return a list of fallbacks which have an IPv6 address
  1905. def fallbacks_with_ipv6(self):
  1906. return filter(lambda x: x.has_ipv6(), self.fallbacks)
  1907. # log a message about the proportion of fallbacks on IPv6
  1908. def describe_fallback_ip_family(self):
  1909. ipv6_falback_count = len(self.fallbacks_with_ipv6())
  1910. fallback_count = len(self.fallbacks)
  1911. logging.warning('%s of fallbacks are on IPv6'%(
  1912. CandidateList.describe_percentage(ipv6_falback_count,
  1913. fallback_count)))
  1914. def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
  1915. guard_count, target_count):
  1916. s = ''
  1917. # Report:
  1918. # whether we checked consensus download times
  1919. # the number of fallback directories (and limits/exclusions, if relevant)
  1920. # min & max fallback bandwidths
  1921. # #error if below minimum count
  1922. if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
  1923. s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%(
  1924. 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '',
  1925. ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS
  1926. and PERFORM_IPV6_DIRPORT_CHECKS) else '',
  1927. 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '',
  1928. CONSENSUS_DOWNLOAD_SPEED_MAX)
  1929. else:
  1930. s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */'
  1931. s += '\n'
  1932. # Multiline C comment with #error if things go bad
  1933. s += '/*'
  1934. s += '\n'
  1935. # Integers don't need escaping in C comments
  1936. fallback_count = len(self.fallbacks)
  1937. if FALLBACK_PROPORTION_OF_GUARDS is None:
  1938. fallback_proportion = ''
  1939. else:
  1940. fallback_proportion = ', Target %d (%d * %.2f)'%(target_count,
  1941. guard_count,
  1942. FALLBACK_PROPORTION_OF_GUARDS)
  1943. s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count,
  1944. fallback_proportion)
  1945. if MAX_FALLBACK_COUNT is not None:
  1946. s += ', Max %d'%(MAX_FALLBACK_COUNT)
  1947. s += ')\n'
  1948. if eligible_count != fallback_count:
  1949. removed_count = eligible_count - fallback_count
  1950. excess_to_target_or_max = (eligible_count - operator_count - failed_count
  1951. - fallback_count)
  1952. # some 'Failed' failed the check, others 'Skipped' the check,
  1953. # if we already had enough successful downloads
  1954. s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' +
  1955. 'Excess %d)')%(removed_count, operator_count, failed_count,
  1956. excess_to_target_or_max)
  1957. s += '\n'
  1958. min_fb = self.fallback_min()
  1959. min_bw = min_fb._data['measured_bandwidth']
  1960. max_fb = self.fallback_max()
  1961. max_bw = max_fb._data['measured_bandwidth']
  1962. s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
  1963. max_bw/(1024.0*1024.0))
  1964. s += '\n'
  1965. s += '*/'
  1966. if fallback_count < MIN_FALLBACK_COUNT:
  1967. # We must have a minimum number of fallbacks so they are always
  1968. # reachable, and are in diverse locations
  1969. s += '\n'
  1970. s += '#error Fallback Count %d is too low. '%(fallback_count)
  1971. s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT)
  1972. s += 'Try adding entries to the whitelist, '
  1973. s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
  1974. return s
  1975. def process_existing():
  1976. logging.basicConfig(level=logging.INFO)
  1977. logging.getLogger('stem').setLevel(logging.INFO)
  1978. whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
  1979. 'name': FALLBACK_FILE_NAME}
  1980. blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
  1981. 'name': BLACKLIST_FILE_NAME}
  1982. list_fallbacks(whitelist, blacklist)
  1983. def process_default():
  1984. logging.basicConfig(level=logging.WARNING)
  1985. logging.getLogger('stem').setLevel(logging.WARNING)
  1986. whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
  1987. 'name': WHITELIST_FILE_NAME}
  1988. blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
  1989. 'name': BLACKLIST_FILE_NAME}
  1990. list_fallbacks(whitelist, blacklist)
  1991. ## Main Function
  1992. def main():
  1993. if get_command() == 'check_existing':
  1994. process_existing()
  1995. else:
  1996. process_default()
  1997. def get_command():
  1998. if len(sys.argv) == 2:
  1999. return sys.argv[1]
  2000. else:
  2001. return None
  2002. def log_excluded(msg, *args):
  2003. if get_command() == 'check_existing':
  2004. logging.warning(msg, *args)
  2005. else:
  2006. logging.info(msg, *args)
  2007. def list_fallbacks(whitelist, blacklist):
  2008. """ Fetches required onionoo documents and evaluates the
  2009. fallback directory criteria for each of the relays """
  2010. print "/* type=fallback */"
  2011. print ("/* version={} */"
  2012. .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
  2013. now = datetime.datetime.utcnow()
  2014. timestamp = now.strftime('%Y%m%d%H%M%S')
  2015. print ("/* timestamp={} */"
  2016. .format(cleanse_c_multiline_comment(timestamp)))
  2017. # end the header with a separator, to make it easier for parsers
  2018. print SECTION_SEPARATOR_COMMENT
  2019. logging.warning('Downloading and parsing Onionoo data. ' +
  2020. 'This may take some time.')
  2021. # find relays that could be fallbacks
  2022. candidates = CandidateList()
  2023. candidates.add_relays()
  2024. # work out how many fallbacks we want
  2025. guard_count = candidates.count_guards()
  2026. if FALLBACK_PROPORTION_OF_GUARDS is None:
  2027. target_count = guard_count
  2028. else:
  2029. target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS)
  2030. # the maximum number of fallbacks is the least of:
  2031. # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count)
  2032. # - the maximum fallback count (MAX_FALLBACK_COUNT)
  2033. if MAX_FALLBACK_COUNT is None:
  2034. max_count = target_count
  2035. else:
  2036. max_count = min(target_count, MAX_FALLBACK_COUNT)
  2037. candidates.compute_fallbacks()
  2038. prefilter_fallbacks = copy.copy(candidates.fallbacks)
  2039. # filter with the whitelist and blacklist
  2040. # if a relay has changed IPv4 address or ports recently, it will be excluded
  2041. # as ineligible before we call apply_filter_lists, and so there will be no
  2042. # warning that the details have changed from those in the whitelist.
  2043. # instead, there will be an info-level log during the eligibility check.
  2044. initial_count = len(candidates.fallbacks)
  2045. excluded_count = candidates.apply_filter_lists(whitelist, blacklist)
  2046. print candidates.summarise_filters(initial_count, excluded_count)
  2047. eligible_count = len(candidates.fallbacks)
  2048. # calculate the measured bandwidth of each relay,
  2049. # then remove low-bandwidth relays
  2050. candidates.calculate_measured_bandwidth()
  2051. candidates.remove_low_bandwidth_relays()
  2052. # print the raw fallback list
  2053. #for x in candidates.fallbacks:
  2054. # print x.fallbackdir_line(True)
  2055. # print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
  2056. # separators=(',', ': '), default=json_util.default)
  2057. # impose mandatory conditions here, like one per contact, family, IP
  2058. # in measured bandwidth order
  2059. candidates.sort_fallbacks_by_measured_bandwidth()
  2060. operator_count = 0
  2061. # only impose these limits on the final list - operators can nominate
  2062. # multiple candidate fallbacks, and then we choose the best set
  2063. if not OUTPUT_CANDIDATES:
  2064. operator_count += candidates.limit_fallbacks_same_ip()
  2065. operator_count += candidates.limit_fallbacks_same_contact()
  2066. operator_count += candidates.limit_fallbacks_same_family()
  2067. # check if each candidate can serve a consensus
  2068. # there's a small risk we've eliminated relays from the same operator that
  2069. # can serve a consensus, in favour of one that can't
  2070. # but given it takes up to 15 seconds to check each consensus download,
  2071. # the risk is worth it
  2072. if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
  2073. logging.warning('Checking consensus download speeds. ' +
  2074. 'This may take some time.')
  2075. failed_count = candidates.perform_download_consensus_checks(max_count)
  2076. # work out which fallbacks cache extra-infos
  2077. candidates.mark_extra_info_caches()
  2078. # analyse and log interesting diversity metrics
  2079. # like netblock, ports, exit, IPv4-only
  2080. # (we can't easily analyse AS, and it's hard to accurately analyse country)
  2081. candidates.describe_fallback_ip_family()
  2082. # if we can't import the ipaddress module, we can't do netblock analysis
  2083. if HAVE_IPADDRESS:
  2084. candidates.describe_fallback_netblocks()
  2085. candidates.describe_fallback_ports()
  2086. candidates.describe_fallback_extra_info_caches()
  2087. candidates.describe_fallback_exit_flag()
  2088. # output C comments summarising the fallback selection process
  2089. if len(candidates.fallbacks) > 0:
  2090. print candidates.summarise_fallbacks(eligible_count, operator_count,
  2091. failed_count, guard_count,
  2092. target_count)
  2093. else:
  2094. print '/* No Fallbacks met criteria */'
  2095. # output C comments specifying the OnionOO data used to create the list
  2096. for s in fetch_source_list():
  2097. print describe_fetch_source(s)
  2098. # start the list with a separator, to make it easy for parsers
  2099. print SECTION_SEPARATOR_COMMENT
  2100. # sort the list differently depending on why we've created it:
  2101. # if we're outputting the final fallback list, sort by fingerprint
  2102. # this makes diffs much more stable
  2103. # otherwise, if we're trying to find a bandwidth cutoff, or we want to
  2104. # contact operators in priority order, sort by bandwidth (not yet
  2105. # implemented)
  2106. # otherwise, if we're contacting operators, sort by contact
  2107. candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
  2108. for x in candidates.fallbacks:
  2109. print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
  2110. if __name__ == "__main__":
  2111. main()