TorNet.py 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185
  1. #!/usr/bin/env python2
  2. #
  3. # Copyright 2011 Nick Mathewson, Michael Stone
  4. # Copyright 2013 The Tor Project
  5. #
  6. # You may do anything with this work that copyright law would normally
  7. # restrict, so long as you retain the above notice(s) and this license
  8. # in all redistributed copies and derived works. There is no warranty.
  9. from __future__ import print_function
  10. from __future__ import with_statement
  11. # Get verbose tracebacks, so we can diagnose better.
  12. import cgitb
  13. cgitb.enable(format="plain")
  14. import os
  15. import signal
  16. import subprocess
  17. import sys
  18. import re
  19. import errno
  20. import time
  21. import shutil
  22. import chutney.Templating
  23. import chutney.Traffic
  24. _BASE_ENVIRON = None
  25. _TORRC_OPTIONS = None
  26. _THE_NETWORK = None
  27. def mkdir_p(d, mode=511):
  28. """Create directory 'd' and all of its parents as needed. Unlike
  29. os.makedirs, does not give an error if d already exists.
  30. 511 is the decimal representation of the octal number 0777. Since
  31. python2 only supports 0777 and python3 only supports 0o777, we can use
  32. neither.
  33. """
  34. try:
  35. os.makedirs(d, mode=mode)
  36. except OSError as e:
  37. if e.errno == errno.EEXIST:
  38. return
  39. raise
  40. class Node(object):
  41. """A Node represents a Tor node or a set of Tor nodes. It's created
  42. in a network configuration file.
  43. This class is responsible for holding the user's selected node
  44. configuration, and figuring out how the node needs to be
  45. configured and launched.
  46. """
  47. # Fields:
  48. # _parent
  49. # _env
  50. # _builder
  51. # _controller
  52. ########
  53. # Users are expected to call these:
  54. def __init__(self, parent=None, **kwargs):
  55. self._parent = parent
  56. self._env = self._createEnviron(parent, kwargs)
  57. self._builder = None
  58. self._controller = None
  59. def getN(self, N):
  60. return [Node(self) for _ in range(N)]
  61. def specialize(self, **kwargs):
  62. return Node(parent=self, **kwargs)
  63. ######
  64. # Chutney uses these:
  65. def getBuilder(self):
  66. """Return a NodeBuilder instance to set up this node (that is, to
  67. write all the files that need to be in place so that this
  68. node can be run by a NodeController).
  69. """
  70. if self._builder is None:
  71. self._builder = LocalNodeBuilder(self._env)
  72. return self._builder
  73. def getController(self):
  74. """Return a NodeController instance to control this node (that is,
  75. to start it, stop it, see if it's running, etc.)
  76. """
  77. if self._controller is None:
  78. self._controller = LocalNodeController(self._env)
  79. return self._controller
  80. def setNodenum(self, num):
  81. """Assign a value to the 'nodenum' element of this node. Each node
  82. in a network gets its own nodenum.
  83. """
  84. self._env['nodenum'] = num
  85. #####
  86. # These are internal:
  87. def _createEnviron(self, parent, argdict):
  88. """Return an Environ that delegates to the parent node's Environ (if
  89. there is a parent node), or to the default environment.
  90. """
  91. if parent:
  92. parentenv = parent._env
  93. else:
  94. parentenv = self._getDefaultEnviron()
  95. return TorEnviron(parentenv, **argdict)
  96. def _getDefaultEnviron(self):
  97. """Return the default environment. Any variables that we can't find
  98. set for any particular node, we look for here.
  99. """
  100. return _BASE_ENVIRON
  101. class _NodeCommon(object):
  102. """Internal helper class for functionality shared by some NodeBuilders
  103. and some NodeControllers."""
  104. # XXXX maybe this should turn into a mixin.
  105. def __init__(self, env):
  106. self._env = env
  107. def expand(self, pat, includePath=(".",)):
  108. return chutney.Templating.Template(pat, includePath).format(self._env)
  109. def _getTorrcFname(self):
  110. """Return the name of the file where we'll be writing torrc"""
  111. return self.expand("${torrc_fname}")
  112. class NodeBuilder(_NodeCommon):
  113. """Abstract base class. A NodeBuilder is responsible for doing all the
  114. one-time prep needed to set up a node in a network.
  115. """
  116. def __init__(self, env):
  117. _NodeCommon.__init__(self, env)
  118. def checkConfig(self, net):
  119. """Try to format our torrc; raise an exception if we can't.
  120. """
  121. def preConfig(self, net):
  122. """Called on all nodes before any nodes configure: generates keys as
  123. needed.
  124. """
  125. def config(self, net):
  126. """Called to configure a node: creates a torrc file for it."""
  127. def postConfig(self, net):
  128. """Called on each nodes after all nodes configure."""
  129. class NodeController(_NodeCommon):
  130. """Abstract base class. A NodeController is responsible for running a
  131. node on the network.
  132. """
  133. def __init__(self, env):
  134. _NodeCommon.__init__(self, env)
  135. def check(self, listRunning=True, listNonRunning=False):
  136. """See if this node is running, stopped, or crashed. If it's running
  137. and listRunning is set, print a short statement. If it's
  138. stopped and listNonRunning is set, then print a short statement.
  139. If it's crashed, print a statement. Return True if the
  140. node is running, false otherwise.
  141. """
  142. def start(self):
  143. """Try to start this node; return True if we succeeded or it was
  144. already running, False if we failed."""
  145. def stop(self, sig=signal.SIGINT):
  146. """Try to stop this node by sending it the signal 'sig'."""
  147. class LocalNodeBuilder(NodeBuilder):
  148. # Environment members used:
  149. # torrc -- which torrc file to use
  150. # torrc_template_path -- path to search for torrc files and include files
  151. # authority -- bool -- are we an authority?
  152. # bridgeauthority -- bool -- are we a bridge authority?
  153. # relay -- bool -- are we a relay?
  154. # bridge -- bool -- are we a bridge?
  155. # hs -- bool -- are we a hidden service?
  156. # nodenum -- int -- set by chutney -- which unique node index is this?
  157. # dir -- path -- set by chutney -- data directory for this tor
  158. # tor_gencert -- path to tor_gencert binary
  159. # tor -- path to tor binary
  160. # auth_cert_lifetime -- lifetime of authority certs, in months.
  161. # ip -- IP to listen on (used only if authority or bridge)
  162. # ipv6_addr -- IPv6 address to listen on (used only if ipv6 bridge)
  163. # orport, dirport -- (used only if authority)
  164. # fingerprint -- used only if authority
  165. # dirserver_flags -- used only if authority
  166. # nick -- nickname of this router
  167. # Environment members set
  168. # fingerprint -- hex router key fingerprint
  169. # nodenum -- int -- set by chutney -- which unique node index is this?
  170. def __init__(self, env):
  171. NodeBuilder.__init__(self, env)
  172. self._env = env
  173. def _createTorrcFile(self, checkOnly=False):
  174. """Write the torrc file for this node, disabling any options
  175. that are not supported by env's tor binary using comments.
  176. If checkOnly, just make sure that the formatting is indeed
  177. possible.
  178. """
  179. fn_out = self._getTorrcFname()
  180. torrc_template = self._getTorrcTemplate()
  181. output = torrc_template.format(self._env)
  182. if checkOnly:
  183. # XXXX Is it time-consuming to format? If so, cache here.
  184. return
  185. # now filter the options we're about to write, commenting out
  186. # the options that the current tor binary doesn't support
  187. tor = self._env['tor']
  188. # find the options the current tor binary supports, and cache them
  189. if tor not in _TORRC_OPTIONS:
  190. # Note: some versions of tor (e.g. 0.2.4.23) require
  191. # --list-torrc-options to be the first argument
  192. cmdline = [
  193. tor,
  194. "--list-torrc-options",
  195. "--hush"]
  196. try:
  197. opts = subprocess.check_output(cmdline,
  198. bufsize=-1,
  199. universal_newlines=True)
  200. except OSError as e:
  201. # only catch file not found error
  202. if e.errno == errno.ENOENT:
  203. print ("Cannot find tor binary %r. Use "
  204. "CHUTNEY_TOR environment variable to set the "
  205. "path, or put the binary into $PATH." % tor)
  206. sys.exit(0)
  207. else:
  208. raise
  209. # check we received a list of options, and nothing else
  210. assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
  211. torrc_opts = opts.split()
  212. # cache the options for this tor binary's path
  213. _TORRC_OPTIONS[tor] = torrc_opts
  214. else:
  215. torrc_opts = _TORRC_OPTIONS[tor]
  216. # check if each option is supported before writing it
  217. # TODO: what about unsupported values?
  218. # e.g. tor 0.2.4.23 doesn't support TestingV3AuthInitialVoteDelay 2
  219. # but later version do. I say throw this one to the user.
  220. with open(fn_out, 'w') as f:
  221. # we need to do case-insensitive option comparison
  222. # even if this is a static whitelist,
  223. # so we convert to lowercase as close to the loop as possible
  224. lower_opts = [opt.lower() for opt in torrc_opts]
  225. # keep ends when splitting lines, so we can write them out
  226. # using writelines() without messing around with "\n"s
  227. for line in output.splitlines(True):
  228. # check if the first word on the line is a supported option,
  229. # preserving empty lines and comment lines
  230. sline = line.strip()
  231. if (len(sline) == 0
  232. or sline[0] == '#'
  233. or sline.split()[0].lower() in lower_opts):
  234. f.writelines([line])
  235. else:
  236. # well, this could get spammy
  237. # TODO: warn once per option per tor binary
  238. # TODO: print tor version?
  239. print (("The tor binary at %r does not support the "
  240. "option in the torrc line:\n"
  241. "%r") % (tor, line.strip()))
  242. # we could decide to skip these lines entirely
  243. # TODO: write tor version?
  244. f.writelines(["# " + tor + " unsupported: " + line])
  245. def _getTorrcTemplate(self):
  246. """Return the template used to write the torrc for this node."""
  247. template_path = self._env['torrc_template_path']
  248. return chutney.Templating.Template("$${include:$torrc}",
  249. includePath=template_path)
  250. def _getFreeVars(self):
  251. """Return a set of the free variables in the torrc template for this
  252. node.
  253. """
  254. template = self._getTorrcTemplate()
  255. return template.freevars(self._env)
  256. def checkConfig(self, net):
  257. """Try to format our torrc; raise an exception if we can't.
  258. """
  259. self._createTorrcFile(checkOnly=True)
  260. def preConfig(self, net):
  261. """Called on all nodes before any nodes configure: generates keys and
  262. hidden service directories as needed.
  263. """
  264. self._makeDataDir()
  265. if self._env['authority']:
  266. self._genAuthorityKey()
  267. if self._env['relay']:
  268. self._genRouterKey()
  269. if self._env['hs']:
  270. self._makeHiddenServiceDir()
  271. def config(self, net):
  272. """Called to configure a node: creates a torrc file for it."""
  273. self._createTorrcFile()
  274. # self._createScripts()
  275. def postConfig(self, net):
  276. """Called on each nodes after all nodes configure."""
  277. # self.net.addNode(self)
  278. pass
  279. def _makeDataDir(self):
  280. """Create the data directory (with keys subdirectory) for this node.
  281. """
  282. datadir = self._env['dir']
  283. mkdir_p(os.path.join(datadir, 'keys'))
  284. def _makeHiddenServiceDir(self):
  285. """Create the hidden service subdirectory for this node.
  286. The directory name is stored under the 'hs_directory' environment
  287. key. It is combined with the 'dir' data directory key to yield the
  288. path to the hidden service directory.
  289. 448 is the decimal representation of the octal number 0700. Since
  290. python2 only supports 0700 and python3 only supports 0o700, we can
  291. use neither.
  292. """
  293. datadir = self._env['dir']
  294. mkdir_p(os.path.join(datadir, self._env['hs_directory']), 448)
  295. def _genAuthorityKey(self):
  296. """Generate an authority identity and signing key for this authority,
  297. if they do not already exist."""
  298. datadir = self._env['dir']
  299. tor_gencert = self._env['tor_gencert']
  300. lifetime = self._env['auth_cert_lifetime']
  301. idfile = os.path.join(datadir, 'keys', "authority_identity_key")
  302. skfile = os.path.join(datadir, 'keys', "authority_signing_key")
  303. certfile = os.path.join(datadir, 'keys', "authority_certificate")
  304. addr = self.expand("${ip}:${dirport}")
  305. passphrase = self._env['auth_passphrase']
  306. if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
  307. return
  308. cmdline = [
  309. tor_gencert,
  310. '--create-identity-key',
  311. '--passphrase-fd', '0',
  312. '-i', idfile,
  313. '-s', skfile,
  314. '-c', certfile,
  315. '-m', str(lifetime),
  316. '-a', addr]
  317. print("Creating identity key %s for %s with %s" % (
  318. idfile, self._env['nick'], " ".join(cmdline)))
  319. try:
  320. p = subprocess.Popen(cmdline, stdin=subprocess.PIPE)
  321. except OSError as e:
  322. # only catch file not found error
  323. if e.errno == errno.ENOENT:
  324. print("Cannot find tor-gencert binary %r. Use "
  325. "CHUTNEY_TOR_GENCERT environment variable to set the "
  326. "path, or put the binary into $PATH." % tor_gencert)
  327. sys.exit(0)
  328. else:
  329. raise
  330. p.communicate(passphrase + "\n")
  331. assert p.returncode == 0 # XXXX BAD!
  332. def _genRouterKey(self):
  333. """Generate an identity key for this router, unless we already have,
  334. and set up the 'fingerprint' entry in the Environ.
  335. """
  336. datadir = self._env['dir']
  337. tor = self._env['tor']
  338. torrc = self._getTorrcFname()
  339. cmdline = [
  340. tor,
  341. "--quiet",
  342. "--ignore-missing-torrc",
  343. "-f", torrc,
  344. "--list-fingerprint",
  345. "--orport", "1",
  346. "--datadirectory", datadir]
  347. try:
  348. p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
  349. except OSError as e:
  350. # only catch file not found error
  351. if e.errno == errno.ENOENT:
  352. print("Cannot find tor binary %r. Use "
  353. "CHUTNEY_TOR environment variable to set the "
  354. "path, or put the binary into $PATH." % tor)
  355. sys.exit(0)
  356. else:
  357. raise
  358. stdout, stderr = p.communicate()
  359. fingerprint = "".join((stdout.rstrip().split('\n')[-1]).split()[1:])
  360. if not re.match(r'^[A-F0-9]{40}$', fingerprint):
  361. print (("Error when calling %r. It gave %r as a fingerprint "
  362. " and %r on stderr.")%(" ".join(cmdline), stdout, stderr))
  363. sys.exit(1)
  364. self._env['fingerprint'] = fingerprint
  365. def _getAltAuthLines(self, hasbridgeauth=False):
  366. """Return a combination of AlternateDirAuthority,
  367. AlternateHSAuthority and AlternateBridgeAuthority lines for
  368. this Node, appropriately. Non-authorities return ""."""
  369. if not self._env['authority']:
  370. return ""
  371. datadir = self._env['dir']
  372. certfile = os.path.join(datadir, 'keys', "authority_certificate")
  373. v3id = None
  374. with open(certfile, 'r') as f:
  375. for line in f:
  376. if line.startswith("fingerprint"):
  377. v3id = line.split()[1].strip()
  378. break
  379. assert v3id is not None
  380. if self._env['bridgeauthority']:
  381. # Bridge authorities return AlternateBridgeAuthority with
  382. # the 'bridge' flag set.
  383. options = ("AlternateBridgeAuthority",)
  384. self._env['dirserver_flags'] += " bridge"
  385. else:
  386. # Directory authorities return AlternateDirAuthority with
  387. # the 'hs' and 'v3ident' flags set.
  388. # XXXX This next line is needed for 'bridges' but breaks
  389. # 'basic'
  390. if hasbridgeauth:
  391. options = ("AlternateDirAuthority",)
  392. else:
  393. options = ("DirAuthority",)
  394. self._env['dirserver_flags'] += " hs v3ident=%s" % v3id
  395. authlines = ""
  396. for authopt in options:
  397. authlines += "%s %s orport=%s %s %s:%s %s\n" % (
  398. authopt, self._env['nick'], self._env['orport'],
  399. self._env['dirserver_flags'], self._env['ip'],
  400. self._env['dirport'], self._env['fingerprint'])
  401. return authlines
  402. def _getBridgeLines(self):
  403. """Return potential Bridge line for this Node. Non-bridge
  404. relays return "".
  405. """
  406. if not self._env['bridge']:
  407. return ""
  408. bridgelines = "Bridge %s:%s\n" % (self._env['ip'],
  409. self._env['orport'])
  410. if self._env['ipv6_addr'] is not None:
  411. bridgelines += "Bridge %s:%s\n" % (self._env['ipv6_addr'],
  412. self._env['orport'])
  413. return bridgelines
  414. class LocalNodeController(NodeController):
  415. def __init__(self, env):
  416. NodeController.__init__(self, env)
  417. self._env = env
  418. def getPid(self):
  419. """Assuming that this node has its pidfile in ${dir}/pid, return
  420. the pid of the running process, or None if there is no pid in the
  421. file.
  422. """
  423. pidfile = os.path.join(self._env['dir'], 'pid')
  424. if not os.path.exists(pidfile):
  425. return None
  426. with open(pidfile, 'r') as f:
  427. return int(f.read())
  428. def isRunning(self, pid=None):
  429. """Return true iff this node is running. (If 'pid' is provided, we
  430. assume that the pid provided is the one of this node. Otherwise
  431. we call getPid().
  432. """
  433. if pid is None:
  434. pid = self.getPid()
  435. if pid is None:
  436. return False
  437. try:
  438. os.kill(pid, 0) # "kill 0" == "are you there?"
  439. except OSError as e:
  440. if e.errno == errno.ESRCH:
  441. return False
  442. raise
  443. # okay, so the process exists. Say "True" for now.
  444. # XXXX check if this is really tor!
  445. return True
  446. def check(self, listRunning=True, listNonRunning=False):
  447. """See if this node is running, stopped, or crashed. If it's running
  448. and listRunning is set, print a short statement. If it's
  449. stopped and listNonRunning is set, then print a short statement.
  450. If it's crashed, print a statement. Return True if the
  451. node is running, false otherwise.
  452. """
  453. # XXX Split this into "check" and "print" parts.
  454. pid = self.getPid()
  455. nick = self._env['nick']
  456. datadir = self._env['dir']
  457. corefile = "core.%s" % pid
  458. if self.isRunning(pid):
  459. if listRunning:
  460. print("%s is running with PID %s" % (nick, pid))
  461. return True
  462. elif os.path.exists(os.path.join(datadir, corefile)):
  463. if listNonRunning:
  464. print("%s seems to have crashed, and left core file %s" % (
  465. nick, corefile))
  466. return False
  467. else:
  468. if listNonRunning:
  469. print("%s is stopped" % nick)
  470. return False
  471. def hup(self):
  472. """Send a SIGHUP to this node, if it's running."""
  473. pid = self.getPid()
  474. nick = self._env['nick']
  475. if self.isRunning(pid):
  476. print("Sending sighup to %s" % nick)
  477. os.kill(pid, signal.SIGHUP)
  478. return True
  479. else:
  480. print("%s is not running" % nick)
  481. return False
  482. def start(self):
  483. """Try to start this node; return True if we succeeded or it was
  484. already running, False if we failed."""
  485. if self.isRunning():
  486. print("%s is already running" % self._env['nick'])
  487. return True
  488. tor_path = self._env['tor']
  489. torrc = self._getTorrcFname()
  490. cmdline = [
  491. tor_path,
  492. "--quiet",
  493. "-f", torrc,
  494. ]
  495. try:
  496. p = subprocess.Popen(cmdline)
  497. except OSError as e:
  498. # only catch file not found error
  499. if e.errno == errno.ENOENT:
  500. print("Cannot find tor binary %r. Use CHUTNEY_TOR "
  501. "environment variable to set the path, or put the "
  502. "binary into $PATH." % tor_path)
  503. sys.exit(0)
  504. else:
  505. raise
  506. if self.waitOnLaunch():
  507. # this requires that RunAsDaemon is set
  508. p.wait()
  509. else:
  510. # this does not require RunAsDaemon to be set, but is slower.
  511. #
  512. # poll() only catches failures before the call itself
  513. # so let's sleep a little first
  514. # this does, of course, slow down process launch
  515. # which can require an adjustment to the voting interval
  516. #
  517. # avoid writing a newline or space when polling
  518. # so output comes out neatly
  519. sys.stdout.write('.')
  520. sys.stdout.flush()
  521. time.sleep(self._env['poll_launch_time'])
  522. p.poll()
  523. if p.returncode != None and p.returncode != 0:
  524. if self._env['poll_launch_time'] is None:
  525. print("Couldn't launch %s (%s): %s" % (self._env['nick'],
  526. " ".join(cmdline),
  527. p.returncode))
  528. else:
  529. print("Couldn't poll %s (%s) "
  530. "after waiting %s seconds for launch"
  531. ": %s" % (self._env['nick'],
  532. " ".join(cmdline),
  533. self._env['poll_launch_time'],
  534. p.returncode))
  535. return False
  536. return True
  537. def stop(self, sig=signal.SIGINT):
  538. """Try to stop this node by sending it the signal 'sig'."""
  539. pid = self.getPid()
  540. if not self.isRunning(pid):
  541. print("%s is not running" % self._env['nick'])
  542. return
  543. os.kill(pid, sig)
  544. def cleanup_lockfile(self):
  545. lf = self._env['lockfile']
  546. if not self.isRunning() and os.path.exists(lf):
  547. print('Removing stale lock file for {0} ...'.format(
  548. self._env['nick']))
  549. os.remove(lf)
  550. def waitOnLaunch(self):
  551. """Check whether we can wait() for the tor process to launch"""
  552. # TODO: is this the best place for this code?
  553. # RunAsDaemon default is 0
  554. runAsDaemon = False
  555. with open(self._getTorrcFname(), 'r') as f:
  556. for line in f.readlines():
  557. stline = line.strip()
  558. # if the line isn't all whitespace or blank
  559. if len(stline) > 0:
  560. splline = stline.split()
  561. # if the line has at least two tokens on it
  562. if (len(splline) > 0
  563. and splline[0].lower() == "RunAsDaemon".lower()
  564. and splline[1] == "1"):
  565. # use the RunAsDaemon value from the torrc
  566. # TODO: multiple values?
  567. runAsDaemon = True
  568. if runAsDaemon:
  569. # we must use wait() instead of poll()
  570. self._env['poll_launch_time'] = None
  571. return True;
  572. else:
  573. # we must use poll() instead of wait()
  574. if self._env['poll_launch_time'] is None:
  575. self._env['poll_launch_time'] = self._env['poll_launch_time_default']
  576. return False;
  577. DEFAULTS = {
  578. 'authority': False,
  579. 'bridgeauthority': False,
  580. 'hasbridgeauth': False,
  581. 'relay': False,
  582. 'bridge': False,
  583. 'hs': False,
  584. 'hs_directory': 'hidden_service',
  585. 'hs-hostname': None,
  586. 'connlimit': 60,
  587. 'net_base_dir': os.environ.get('CHUTNEY_DATA_DIR', 'net'),
  588. 'tor': os.environ.get('CHUTNEY_TOR', 'tor'),
  589. 'tor-gencert': os.environ.get('CHUTNEY_TOR_GENCERT', None),
  590. 'auth_cert_lifetime': 12,
  591. 'ip': '127.0.0.1',
  592. 'ipv6_addr': None,
  593. 'dirserver_flags': 'no-v2',
  594. 'chutney_dir': '.',
  595. 'torrc_fname': '${dir}/torrc',
  596. 'orport_base': 5000,
  597. 'dirport_base': 7000,
  598. 'controlport_base': 8000,
  599. 'socksport_base': 9000,
  600. 'authorities': "AlternateDirAuthority bleargh bad torrc file!",
  601. 'bridges': "Bridge bleargh bad torrc file!",
  602. 'core': True,
  603. # poll_launch_time: None means wait on launch (requires RunAsDaemon),
  604. # otherwise, poll after that many seconds (can be fractional/decimal)
  605. 'poll_launch_time': None,
  606. # Used when poll_launch_time is None, but RunAsDaemon is not set
  607. # Set low so that we don't interfere with the voting interval
  608. 'poll_launch_time_default': 0.1,
  609. # the number of bytes of random data we send on each connection
  610. 'data_bytes': int(os.environ.get('CHUTNEY_DATA_BYTES', 10 * 1024)),
  611. # the number of times each client will connect
  612. 'connection_count': int(os.environ.get('CHUTNEY_CONNECTIONS', 1)),
  613. # Do we want every client to connect to every HS, or one client
  614. # to connect to each HS?
  615. # (Clients choose an exit at random, so this doesn't apply to exits.)
  616. 'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
  617. }
  618. class TorEnviron(chutney.Templating.Environ):
  619. """Subclass of chutney.Templating.Environ to implement commonly-used
  620. substitutions.
  621. Environment fields provided:
  622. orport, controlport, socksport, dirport:
  623. dir:
  624. nick:
  625. tor_gencert:
  626. auth_passphrase:
  627. torrc_template_path:
  628. hs_hostname:
  629. Environment fields used:
  630. nodenum
  631. tag
  632. orport_base, controlport_base, socksport_base, dirport_base
  633. tor-gencert (note hyphen)
  634. chutney_dir
  635. tor
  636. dir
  637. hs_directory
  638. nick (debugging only)
  639. hs-hostname (note hyphen)
  640. XXXX document the above. Or document all fields in one place?
  641. """
  642. def __init__(self, parent=None, **kwargs):
  643. chutney.Templating.Environ.__init__(self, parent=parent, **kwargs)
  644. def _get_orport(self, my):
  645. return my['orport_base'] + my['nodenum']
  646. def _get_controlport(self, my):
  647. return my['controlport_base'] + my['nodenum']
  648. def _get_socksport(self, my):
  649. return my['socksport_base'] + my['nodenum']
  650. def _get_dirport(self, my):
  651. return my['dirport_base'] + my['nodenum']
  652. def _get_dir(self, my):
  653. return os.path.abspath(os.path.join(my['net_base_dir'],
  654. "nodes",
  655. "%03d%s" % (
  656. my['nodenum'], my['tag'])))
  657. def _get_nick(self, my):
  658. return "test%03d%s" % (my['nodenum'], my['tag'])
  659. def _get_tor_gencert(self, my):
  660. return my['tor-gencert'] or '{0}-gencert'.format(my['tor'])
  661. def _get_auth_passphrase(self, my):
  662. return self['nick'] # OMG TEH SECURE!
  663. def _get_torrc_template_path(self, my):
  664. return [os.path.join(my['chutney_dir'], 'torrc_templates')]
  665. def _get_lockfile(self, my):
  666. return os.path.join(self['dir'], 'lock')
  667. # A hs generates its key on first run,
  668. # so check for it at the last possible moment,
  669. # but cache it in memory to avoid repeatedly reading the file
  670. # XXXX - this is not like the other functions in this class,
  671. # as it reads from a file created by the hidden service
  672. def _get_hs_hostname(self, my):
  673. if my['hs-hostname'] is None:
  674. datadir = my['dir']
  675. # a file containing a single line with the hs' .onion address
  676. hs_hostname_file = os.path.join(datadir,
  677. my['hs_directory'],
  678. 'hostname')
  679. try:
  680. with open(hs_hostname_file, 'r') as hostnamefp:
  681. hostname = hostnamefp.read()
  682. # the hostname file ends with a newline
  683. hostname = hostname.strip()
  684. my['hs-hostname'] = hostname
  685. except IOError as e:
  686. print("Error: hs %r error %d: %r opening hostname file '%r'"
  687. %(my['nick'], e.errno, e.strerror, hs_hostname_file))
  688. return my['hs-hostname']
  689. class Network(object):
  690. """A network of Tor nodes, plus functions to manipulate them
  691. """
  692. def __init__(self, defaultEnviron):
  693. self._nodes = []
  694. self._dfltEnv = defaultEnviron
  695. self._nextnodenum = 0
  696. def _addNode(self, n):
  697. n.setNodenum(self._nextnodenum)
  698. self._nextnodenum += 1
  699. self._nodes.append(n)
  700. def move_aside_nodes(self):
  701. nodesdir = os.path.join(os.getcwd(),'net','nodes')
  702. if not os.path.exists(nodesdir):
  703. return
  704. newdir = newdirbase = "%s.%d" % (nodesdir, time.time())
  705. i = 0
  706. while os.path.exists(newdir):
  707. i += 1
  708. newdir = "%s.%d" %(newdirbase, i)
  709. print ("NOTE: renaming %r to %r"%(nodesdir, newdir))
  710. os.rename(nodesdir, newdir)
  711. def _checkConfig(self):
  712. for n in self._nodes:
  713. n.getBuilder().checkConfig(self)
  714. def configure(self):
  715. # shutil.rmtree(os.path.join(os.getcwd(),'net','nodes'),ignore_errors=True)
  716. self.move_aside_nodes()
  717. network = self
  718. altauthlines = []
  719. bridgelines = []
  720. builders = [n.getBuilder() for n in self._nodes]
  721. self._checkConfig()
  722. # XXX don't change node names or types or count if anything is
  723. # XXX running!
  724. for b in builders:
  725. b.preConfig(network)
  726. altauthlines.append(b._getAltAuthLines(
  727. self._dfltEnv['hasbridgeauth']))
  728. bridgelines.append(b._getBridgeLines())
  729. self._dfltEnv['authorities'] = "".join(altauthlines)
  730. self._dfltEnv['bridges'] = "".join(bridgelines)
  731. for b in builders:
  732. b.config(network)
  733. for b in builders:
  734. b.postConfig(network)
  735. def status(self):
  736. statuses = [n.getController().check() for n in self._nodes]
  737. n_ok = len([x for x in statuses if x])
  738. print("%d/%d nodes are running" % (n_ok, len(self._nodes)))
  739. return n_ok == len(self._nodes)
  740. def restart(self):
  741. self.stop()
  742. self.start()
  743. def start(self):
  744. if self._dfltEnv['poll_launch_time'] is not None:
  745. # format polling correctly - avoid printing a newline
  746. sys.stdout.write("Starting nodes")
  747. sys.stdout.flush()
  748. else:
  749. print("Starting nodes")
  750. rv = all([n.getController().start() for n in self._nodes])
  751. # now print a newline unconditionally - this stops poll()ing
  752. # output from being squashed together, at the cost of a blank
  753. # line in wait()ing output
  754. print("")
  755. return rv
  756. def hup(self):
  757. print("Sending SIGHUP to nodes")
  758. return all([n.getController().hup() for n in self._nodes])
  759. def stop(self):
  760. controllers = [n.getController() for n in self._nodes]
  761. for sig, desc in [(signal.SIGINT, "SIGINT"),
  762. (signal.SIGINT, "another SIGINT"),
  763. (signal.SIGKILL, "SIGKILL")]:
  764. print("Sending %s to nodes" % desc)
  765. for c in controllers:
  766. if c.isRunning():
  767. c.stop(sig=sig)
  768. print("Waiting for nodes to finish.")
  769. for n in range(15):
  770. time.sleep(1)
  771. if all(not c.isRunning() for c in controllers):
  772. # check for stale lock file when Tor crashes
  773. for c in controllers:
  774. c.cleanup_lockfile()
  775. return
  776. sys.stdout.write(".")
  777. sys.stdout.flush()
  778. for c in controllers:
  779. c.check(listNonRunning=False)
  780. def verify(self):
  781. print("Verifying data transmission:")
  782. status = self._verify_traffic()
  783. print("Transmission: %s" % ("Success" if status else "Failure"))
  784. if not status:
  785. # TODO: allow the debug flag to be passed as an argument to
  786. # src/test/test-network.sh and chutney
  787. print("Set 'debug_flag = True' in Traffic.py to diagnose.")
  788. return status
  789. def _verify_traffic(self):
  790. """Verify (parts of) the network by sending traffic through it
  791. and verify what is received."""
  792. LISTEN_PORT = 4747 # FIXME: Do better! Note the default exit policy.
  793. # HSs must have a HiddenServiceDir with
  794. # "HiddenServicePort <HS_PORT> 127.0.0.1:<LISTEN_PORT>"
  795. HS_PORT = 5858
  796. # The amount of data to send between each source-sink pair,
  797. # each time the source connects.
  798. # We create a source-sink pair for each (bridge) client to an exit,
  799. # and a source-sink pair for a (bridge) client to each hidden service
  800. DATALEN = self._dfltEnv['data_bytes']
  801. # Print a dot each time a sink verifies this much data
  802. DOTDATALEN = 5 * 1024 * 1024 # Octets.
  803. TIMEOUT = 3 # Seconds.
  804. # Calculate the amount of random data we should use
  805. randomlen = self._calculate_randomlen(DATALEN)
  806. reps = self._calculate_reps(DATALEN, randomlen)
  807. # sanity check
  808. if reps == 0:
  809. DATALEN = 0
  810. # Get the random data
  811. if randomlen > 0:
  812. # print a dot after every DOTDATALEN data is verified, rounding up
  813. dot_reps = self._calculate_reps(DOTDATALEN, randomlen)
  814. # make sure we get at least one dot per transmission
  815. dot_reps = min(reps, dot_reps)
  816. with open('/dev/urandom', 'r') as randfp:
  817. tmpdata = randfp.read(randomlen)
  818. else:
  819. dot_reps = 0
  820. tmpdata = {}
  821. # now make the connections
  822. bind_to = ('127.0.0.1', LISTEN_PORT)
  823. tt = chutney.Traffic.TrafficTester(bind_to,
  824. tmpdata,
  825. TIMEOUT,
  826. reps,
  827. dot_reps)
  828. client_list = filter(lambda n:
  829. n._env['tag'] == 'c' or n._env['tag'] == 'bc',
  830. self._nodes)
  831. exit_list = filter(lambda n:
  832. ('exit' in n._env.keys()) and n._env['exit'] == 1,
  833. self._nodes)
  834. hs_list = filter(lambda n:
  835. n._env['tag'] == 'h',
  836. self._nodes)
  837. if len(client_list) == 0:
  838. print(" Unable to verify network: no client nodes available")
  839. return False
  840. if len(exit_list) == 0 and len(hs_list) == 0:
  841. print(" Unable to verify network: no exit/hs nodes available")
  842. print(" Exit nodes must be declared 'relay=1, exit=1'")
  843. print(" HS nodes must be declared 'tag=\"hs\"'")
  844. return False
  845. print("Connecting:")
  846. # the number of tor nodes in paths which will send DATALEN data
  847. # if a node is used in two paths, we count it twice
  848. # this is a lower bound, as cannabilised circuits are one node longer
  849. total_path_node_count = 0
  850. total_path_node_count += self._configure_exits(tt, bind_to,
  851. tmpdata, reps,
  852. client_list, exit_list,
  853. LISTEN_PORT)
  854. total_path_node_count += self._configure_hs(tt,
  855. tmpdata, reps,
  856. client_list, hs_list,
  857. HS_PORT,
  858. LISTEN_PORT)
  859. print("Transmitting Data:")
  860. start_time = time.clock()
  861. status = tt.run()
  862. end_time = time.clock()
  863. # if we fail, don't report the bandwidth
  864. if not status:
  865. return status
  866. # otherwise, report bandwidth used, if sufficient data was transmitted
  867. self._report_bandwidth(DATALEN, total_path_node_count,
  868. start_time, end_time)
  869. return status
  870. # In order to performance test a tor network, we need to transmit
  871. # several hundred megabytes of data or more. Passing around this
  872. # much data in Python has its own performance impacts, so we provide
  873. # a smaller amount of random data instead, and repeat it to DATALEN
  874. def _calculate_randomlen(self, datalen):
  875. MAX_RANDOMLEN = 128 * 1024 # Octets.
  876. if datalen > MAX_RANDOMLEN:
  877. return MAX_RANDOMLEN
  878. else:
  879. return datalen
  880. def _calculate_reps(self, datalen, replen):
  881. # sanity checks
  882. if datalen == 0 or replen == 0:
  883. return 0
  884. # effectively rounds datalen up to the nearest replen
  885. if replen < datalen:
  886. return (datalen + replen - 1) / replen
  887. else:
  888. return 1
  889. # if there are any exits, each client / bridge client transmits
  890. # via 4 nodes (including the client) to an arbitrary exit
  891. # Each client binds directly to 127.0.0.1:LISTEN_PORT via an Exit relay
  892. def _configure_exits(self, tt, bind_to,
  893. tmpdata, reps,
  894. client_list, exit_list,
  895. LISTEN_PORT):
  896. CLIENT_EXIT_PATH_NODES = 4
  897. connection_count = self._dfltEnv['connection_count']
  898. exit_path_node_count = 0
  899. if len(exit_list) > 0:
  900. exit_path_node_count += (len(client_list)
  901. * CLIENT_EXIT_PATH_NODES
  902. * connection_count)
  903. for op in client_list:
  904. print(" Exit to %s:%d via client %s:%s"
  905. % ('127.0.0.1', LISTEN_PORT,
  906. 'localhost', op._env['socksport']))
  907. for i in range(connection_count):
  908. tt.add(chutney.Traffic.Source(tt,
  909. bind_to,
  910. tmpdata,
  911. ('localhost',
  912. int(op._env['socksport'])),
  913. reps))
  914. return exit_path_node_count
  915. # The HS redirects .onion connections made to hs_hostname:HS_PORT
  916. # to the Traffic Tester's 127.0.0.1:LISTEN_PORT
  917. # an arbitrary client / bridge client transmits via 8 nodes
  918. # (including the client and hs) to each hidden service
  919. # Instead of binding directly to LISTEN_PORT via an Exit relay,
  920. # we bind to hs_hostname:HS_PORT via a hidden service connection
  921. def _configure_hs(self, tt,
  922. tmpdata, reps,
  923. client_list, hs_list,
  924. HS_PORT,
  925. LISTEN_PORT):
  926. CLIENT_HS_PATH_NODES = 8
  927. connection_count = self._dfltEnv['connection_count']
  928. hs_path_node_count = (len(hs_list)
  929. * CLIENT_HS_PATH_NODES
  930. * connection_count)
  931. # Each client in hs_client_list connects to each hs
  932. if self._dfltEnv['hs_multi_client']:
  933. hs_client_list = client_list
  934. hs_path_node_count *= len(client_list)
  935. else:
  936. # only use the first client in the list
  937. hs_client_list = client_list[:1]
  938. # Setup the connections from each client in hs_client_list to each hs
  939. for hs in hs_list:
  940. hs_bind_to = (hs._env['hs_hostname'], HS_PORT)
  941. for client in hs_client_list:
  942. print(" HS to %s:%d (%s:%d) via client %s:%s"
  943. % (hs._env['hs_hostname'], HS_PORT,
  944. '127.0.0.1', LISTEN_PORT,
  945. 'localhost', client._env['socksport']))
  946. for i in range(connection_count):
  947. tt.add(chutney.Traffic.Source(tt,
  948. hs_bind_to,
  949. tmpdata,
  950. ('localhost',
  951. int(client._env['socksport'])),
  952. reps))
  953. return hs_path_node_count
  954. # calculate the single stream bandwidth and overall tor bandwidth
  955. # the single stream bandwidth is the bandwidth of the
  956. # slowest stream of all the simultaneously transmitted streams
  957. # the overall bandwidth estimates the simultaneous bandwidth between
  958. # all tor nodes over all simultaneous streams, assuming:
  959. # * minimum path lengths (no cannibalized circuits)
  960. # * unlimited network bandwidth (that is, localhost)
  961. # * tor performance is CPU-limited
  962. # This be used to estimate the bandwidth capacity of a CPU-bound
  963. # tor relay running on this machine
  964. def _report_bandwidth(self, data_length, total_path_node_count,
  965. start_time, end_time):
  966. # otherwise, if we sent at least 5 MB cumulative total, and
  967. # it took us at least a second to send, report bandwidth
  968. MIN_BWDATA = 5 * 1024 * 1024 # Octets.
  969. MIN_ELAPSED_TIME = 1.0 # Seconds.
  970. cumulative_data_sent = total_path_node_count * data_length
  971. elapsed_time = end_time - start_time
  972. if (cumulative_data_sent >= MIN_BWDATA
  973. and elapsed_time >= MIN_ELAPSED_TIME):
  974. # Report megabytes per second
  975. BWDIVISOR = 1024*1024
  976. single_stream_bandwidth = (data_length
  977. / elapsed_time
  978. / BWDIVISOR)
  979. overall_bandwidth = (cumulative_data_sent
  980. / elapsed_time
  981. / BWDIVISOR)
  982. print("Single Stream Bandwidth: %.2f MBytes/s"
  983. % single_stream_bandwidth)
  984. print("Overall tor Bandwidth: %.2f MBytes/s"
  985. % overall_bandwidth)
  986. def ConfigureNodes(nodelist):
  987. network = _THE_NETWORK
  988. for n in nodelist:
  989. network._addNode(n)
  990. if n._env['bridgeauthority']:
  991. network._dfltEnv['hasbridgeauth'] = True
  992. def usage(network):
  993. return "\n".join(["Usage: chutney {command} {networkfile}",
  994. "Known commands are: %s" % (
  995. " ".join(x for x in dir(network)
  996. if not x.startswith("_")))])
  997. def exit_on_error(err_msg):
  998. print ("Error: {0}\n".format(err_msg))
  999. print (usage(_THE_NETWORK))
  1000. sys.exit(1)
  1001. def runConfigFile(verb, data):
  1002. _GLOBALS = dict(_BASE_ENVIRON=_BASE_ENVIRON,
  1003. Node=Node,
  1004. ConfigureNodes=ConfigureNodes,
  1005. _THE_NETWORK=_THE_NETWORK)
  1006. exec(data, _GLOBALS)
  1007. network = _GLOBALS['_THE_NETWORK']
  1008. if not hasattr(network, verb):
  1009. print(usage(network))
  1010. print("Error: I don't know how to %s." % verb)
  1011. return
  1012. return getattr(network, verb)()
  1013. def parseArgs():
  1014. if len(sys.argv) < 3:
  1015. exit_on_error("Not enough arguments given.")
  1016. if not os.path.isfile(sys.argv[2]):
  1017. exit_on_error("Cannot find networkfile: {0}.".format(sys.argv[2]))
  1018. return {'network_cfg': sys.argv[2], 'action': sys.argv[1]}
  1019. def main():
  1020. global _BASE_ENVIRON
  1021. global _TORRC_OPTIONS
  1022. global _THE_NETWORK
  1023. _BASE_ENVIRON = TorEnviron(chutney.Templating.Environ(**DEFAULTS))
  1024. # _TORRC_OPTIONS gets initialised on demand as a map of
  1025. # "/path/to/tor" => ["SupportedOption1", "SupportedOption2", ...]
  1026. # Or it can be pre-populated as a static whitelist of options
  1027. _TORRC_OPTIONS = dict()
  1028. _THE_NETWORK = Network(_BASE_ENVIRON)
  1029. args = parseArgs()
  1030. f = open(args['network_cfg'])
  1031. result = runConfigFile(args['action'], f)
  1032. if result is False:
  1033. return -1
  1034. return 0
  1035. if __name__ == '__main__':
  1036. sys.exit(main())