TorNet.py 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161
  1. #!/usr/bin/env python2
  2. #
  3. # Copyright 2011 Nick Mathewson, Michael Stone
  4. # Copyright 2013 The Tor Project
  5. #
  6. # You may do anything with this work that copyright law would normally
  7. # restrict, so long as you retain the above notice(s) and this license
  8. # in all redistributed copies and derived works. There is no warranty.
  9. from __future__ import print_function
  10. from __future__ import with_statement
  11. import cgitb
  12. import os
  13. import signal
  14. import subprocess
  15. import sys
  16. import re
  17. import errno
  18. import time
  19. import shutil
  20. import importlib
  21. import chutney.Templating
  22. import chutney.Traffic
  23. _BASE_ENVIRON = None
  24. _TORRC_OPTIONS = None
  25. _THE_NETWORK = None
  26. # Get verbose tracebacks, so we can diagnose better.
  27. cgitb.enable(format="plain")
  28. def mkdir_p(d, mode=448):
  29. """Create directory 'd' and all of its parents as needed. Unlike
  30. os.makedirs, does not give an error if d already exists.
  31. 448 is the decimal representation of the octal number 0700. Since
  32. python2 only supports 0700 and python3 only supports 0o700, we can use
  33. neither.
  34. """
  35. try:
  36. os.makedirs(d, mode=mode)
  37. except OSError as e:
  38. if e.errno == errno.EEXIST:
  39. return
  40. raise
  41. def get_absolute_chutney_path():
  42. # use the current directory as the default
  43. # (./chutney already sets CHUTNEY_PATH using the path to the script)
  44. # use tools/test-network.sh if you want chutney to try really hard to find
  45. # itself
  46. relative_chutney_path = os.environ.get('CHUTNEY_PATH', os.getcwd())
  47. return os.path.abspath(relative_chutney_path)
  48. def get_absolute_net_path():
  49. # use the chutney path as the default
  50. absolute_chutney_path = get_absolute_chutney_path()
  51. relative_net_path = os.environ.get('CHUTNEY_DATA_DIR', 'net')
  52. # but what is it relative to?
  53. # let's check if it's in CHUTNEY_PATH first, to preserve
  54. # backwards-compatible behaviour
  55. chutney_net_path = os.path.join(absolute_chutney_path, relative_net_path)
  56. if os.path.isdir(chutney_net_path):
  57. return chutney_net_path
  58. # ok, it's relative to the current directory, whatever that is
  59. return os.path.abspath(relative_net_path)
  60. def get_absolute_nodes_path():
  61. # there's no way to customise this: we really don't need more options
  62. return os.path.join(get_absolute_net_path(), 'nodes')
  63. def get_new_absolute_nodes_path(now=time.time()):
  64. # automatically chosen to prevent path collisions, and result in an ordered
  65. # series of directory path names
  66. # should only be called by 'chutney configure', all other chutney commands
  67. # should use get_absolute_nodes_path()
  68. nodesdir = get_absolute_nodes_path()
  69. newdir = newdirbase = "%s.%d" % (nodesdir, now)
  70. # if the time is the same, fall back to a simple integer count
  71. # (this is very unlikely to happen unless the clock changes: it's not
  72. # possible to run multiple chutney networks at the same time)
  73. i = 0
  74. while os.path.exists(newdir):
  75. i += 1
  76. newdir = "%s.%d" % (newdirbase, i)
  77. return newdir
  78. class Node(object):
  79. """A Node represents a Tor node or a set of Tor nodes. It's created
  80. in a network configuration file.
  81. This class is responsible for holding the user's selected node
  82. configuration, and figuring out how the node needs to be
  83. configured and launched.
  84. """
  85. # Fields:
  86. # _parent
  87. # _env
  88. # _builder
  89. # _controller
  90. ########
  91. # Users are expected to call these:
  92. def __init__(self, parent=None, **kwargs):
  93. self._parent = parent
  94. self._env = self._createEnviron(parent, kwargs)
  95. self._builder = None
  96. self._controller = None
  97. def getN(self, N):
  98. return [Node(self) for _ in range(N)]
  99. def specialize(self, **kwargs):
  100. return Node(parent=self, **kwargs)
  101. ######
  102. # Chutney uses these:
  103. def getBuilder(self):
  104. """Return a NodeBuilder instance to set up this node (that is, to
  105. write all the files that need to be in place so that this
  106. node can be run by a NodeController).
  107. """
  108. if self._builder is None:
  109. self._builder = LocalNodeBuilder(self._env)
  110. return self._builder
  111. def getController(self):
  112. """Return a NodeController instance to control this node (that is,
  113. to start it, stop it, see if it's running, etc.)
  114. """
  115. if self._controller is None:
  116. self._controller = LocalNodeController(self._env)
  117. return self._controller
  118. def setNodenum(self, num):
  119. """Assign a value to the 'nodenum' element of this node. Each node
  120. in a network gets its own nodenum.
  121. """
  122. self._env['nodenum'] = num
  123. #####
  124. # These are internal:
  125. def _createEnviron(self, parent, argdict):
  126. """Return an Environ that delegates to the parent node's Environ (if
  127. there is a parent node), or to the default environment.
  128. """
  129. if parent:
  130. parentenv = parent._env
  131. else:
  132. parentenv = self._getDefaultEnviron()
  133. return TorEnviron(parentenv, **argdict)
  134. def _getDefaultEnviron(self):
  135. """Return the default environment. Any variables that we can't find
  136. set for any particular node, we look for here.
  137. """
  138. return _BASE_ENVIRON
  139. class _NodeCommon(object):
  140. """Internal helper class for functionality shared by some NodeBuilders
  141. and some NodeControllers."""
  142. # XXXX maybe this should turn into a mixin.
  143. def __init__(self, env):
  144. self._env = env
  145. def expand(self, pat, includePath=(".",)):
  146. return chutney.Templating.Template(pat, includePath).format(self._env)
  147. def _getTorrcFname(self):
  148. """Return the name of the file where we'll be writing torrc"""
  149. return self.expand("${torrc_fname}")
  150. class NodeBuilder(_NodeCommon):
  151. """Abstract base class. A NodeBuilder is responsible for doing all the
  152. one-time prep needed to set up a node in a network.
  153. """
  154. def __init__(self, env):
  155. _NodeCommon.__init__(self, env)
  156. def checkConfig(self, net):
  157. """Try to format our torrc; raise an exception if we can't.
  158. """
  159. def preConfig(self, net):
  160. """Called on all nodes before any nodes configure: generates keys as
  161. needed.
  162. """
  163. def config(self, net):
  164. """Called to configure a node: creates a torrc file for it."""
  165. def postConfig(self, net):
  166. """Called on each nodes after all nodes configure."""
  167. class NodeController(_NodeCommon):
  168. """Abstract base class. A NodeController is responsible for running a
  169. node on the network.
  170. """
  171. def __init__(self, env):
  172. _NodeCommon.__init__(self, env)
  173. def check(self, listRunning=True, listNonRunning=False):
  174. """See if this node is running, stopped, or crashed. If it's running
  175. and listRunning is set, print a short statement. If it's
  176. stopped and listNonRunning is set, then print a short statement.
  177. If it's crashed, print a statement. Return True if the
  178. node is running, false otherwise.
  179. """
  180. def start(self):
  181. """Try to start this node; return True if we succeeded or it was
  182. already running, False if we failed."""
  183. def stop(self, sig=signal.SIGINT):
  184. """Try to stop this node by sending it the signal 'sig'."""
  185. class LocalNodeBuilder(NodeBuilder):
  186. # Environment members used:
  187. # torrc -- which torrc file to use
  188. # torrc_template_path -- path to search for torrc files and include files
  189. # authority -- bool -- are we an authority?
  190. # bridgeauthority -- bool -- are we a bridge authority?
  191. # relay -- bool -- are we a relay?
  192. # bridge -- bool -- are we a bridge?
  193. # hs -- bool -- are we a hidden service?
  194. # nodenum -- int -- set by chutney -- which unique node index is this?
  195. # dir -- path -- set by chutney -- data directory for this tor
  196. # tor_gencert -- path to tor_gencert binary
  197. # tor -- path to tor binary
  198. # auth_cert_lifetime -- lifetime of authority certs, in months.
  199. # ip -- IP to listen on
  200. # ipv6_addr -- IPv6 address to listen on
  201. # orport, dirport -- used on authorities, relays, and bridges
  202. # fingerprint -- used only if authority
  203. # dirserver_flags -- used only if authority
  204. # nick -- nickname of this router
  205. # Environment members set
  206. # fingerprint -- hex router key fingerprint
  207. # nodenum -- int -- set by chutney -- which unique node index is this?
  208. def __init__(self, env):
  209. NodeBuilder.__init__(self, env)
  210. self._env = env
  211. def _createTorrcFile(self, checkOnly=False):
  212. """Write the torrc file for this node, disabling any options
  213. that are not supported by env's tor binary using comments.
  214. If checkOnly, just make sure that the formatting is indeed
  215. possible.
  216. """
  217. fn_out = self._getTorrcFname()
  218. torrc_template = self._getTorrcTemplate()
  219. output = torrc_template.format(self._env)
  220. if checkOnly:
  221. # XXXX Is it time-consuming to format? If so, cache here.
  222. return
  223. # now filter the options we're about to write, commenting out
  224. # the options that the current tor binary doesn't support
  225. tor = self._env['tor']
  226. # find the options the current tor binary supports, and cache them
  227. if tor not in _TORRC_OPTIONS:
  228. # Note: some versions of tor (e.g. 0.2.4.23) require
  229. # --list-torrc-options to be the first argument
  230. cmdline = [
  231. tor,
  232. "--list-torrc-options",
  233. "--hush"]
  234. try:
  235. opts = subprocess.check_output(cmdline, bufsize=-1,
  236. universal_newlines=True)
  237. except OSError as e:
  238. # only catch file not found error
  239. if e.errno == errno.ENOENT:
  240. print("Cannot find tor binary %r. Use "
  241. "CHUTNEY_TOR environment variable to set the "
  242. "path, or put the binary into $PATH." % tor)
  243. sys.exit(0)
  244. else:
  245. raise
  246. # check we received a list of options, and nothing else
  247. assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
  248. torrc_opts = opts.split()
  249. # cache the options for this tor binary's path
  250. _TORRC_OPTIONS[tor] = torrc_opts
  251. else:
  252. torrc_opts = _TORRC_OPTIONS[tor]
  253. # check if each option is supported before writing it
  254. # TODO: what about unsupported values?
  255. # e.g. tor 0.2.4.23 doesn't support TestingV3AuthInitialVoteDelay 2
  256. # but later version do. I say throw this one to the user.
  257. with open(fn_out, 'w') as f:
  258. # we need to do case-insensitive option comparison
  259. # even if this is a static whitelist,
  260. # so we convert to lowercase as close to the loop as possible
  261. lower_opts = [opt.lower() for opt in torrc_opts]
  262. # keep ends when splitting lines, so we can write them out
  263. # using writelines() without messing around with "\n"s
  264. for line in output.splitlines(True):
  265. # check if the first word on the line is a supported option,
  266. # preserving empty lines and comment lines
  267. sline = line.strip()
  268. if (len(sline) == 0 or
  269. sline[0] == '#' or
  270. sline.split()[0].lower() in lower_opts):
  271. f.writelines([line])
  272. else:
  273. # well, this could get spammy
  274. # TODO: warn once per option per tor binary
  275. # TODO: print tor version?
  276. print(("The tor binary at %r does not support the "
  277. "option in the torrc line:\n"
  278. "%r") % (tor, line.strip()))
  279. # we could decide to skip these lines entirely
  280. # TODO: write tor version?
  281. f.writelines(["# " + tor + " unsupported: " + line])
  282. def _getTorrcTemplate(self):
  283. """Return the template used to write the torrc for this node."""
  284. template_path = self._env['torrc_template_path']
  285. return chutney.Templating.Template("$${include:$torrc}",
  286. includePath=template_path)
  287. def _getFreeVars(self):
  288. """Return a set of the free variables in the torrc template for this
  289. node.
  290. """
  291. template = self._getTorrcTemplate()
  292. return template.freevars(self._env)
  293. def checkConfig(self, net):
  294. """Try to format our torrc; raise an exception if we can't.
  295. """
  296. self._createTorrcFile(checkOnly=True)
  297. def preConfig(self, net):
  298. """Called on all nodes before any nodes configure: generates keys and
  299. hidden service directories as needed.
  300. """
  301. self._makeDataDir()
  302. if self._env['authority']:
  303. self._genAuthorityKey()
  304. if self._env['relay']:
  305. self._genRouterKey()
  306. if self._env['hs']:
  307. self._makeHiddenServiceDir()
  308. def config(self, net):
  309. """Called to configure a node: creates a torrc file for it."""
  310. self._createTorrcFile()
  311. # self._createScripts()
  312. def postConfig(self, net):
  313. """Called on each nodes after all nodes configure."""
  314. # self.net.addNode(self)
  315. pass
  316. def _makeDataDir(self):
  317. """Create the data directory (with keys subdirectory) for this node.
  318. """
  319. datadir = self._env['dir']
  320. mkdir_p(os.path.join(datadir, 'keys'))
  321. def _makeHiddenServiceDir(self):
  322. """Create the hidden service subdirectory for this node.
  323. The directory name is stored under the 'hs_directory' environment
  324. key. It is combined with the 'dir' data directory key to yield the
  325. path to the hidden service directory.
  326. """
  327. datadir = self._env['dir']
  328. mkdir_p(os.path.join(datadir, self._env['hs_directory']))
  329. def _genAuthorityKey(self):
  330. """Generate an authority identity and signing key for this authority,
  331. if they do not already exist."""
  332. datadir = self._env['dir']
  333. tor_gencert = self._env['tor_gencert']
  334. lifetime = self._env['auth_cert_lifetime']
  335. idfile = os.path.join(datadir, 'keys', "authority_identity_key")
  336. skfile = os.path.join(datadir, 'keys', "authority_signing_key")
  337. certfile = os.path.join(datadir, 'keys', "authority_certificate")
  338. addr = self.expand("${ip}:${dirport}")
  339. passphrase = self._env['auth_passphrase']
  340. if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
  341. return
  342. cmdline = [
  343. tor_gencert,
  344. '--create-identity-key',
  345. '--passphrase-fd', '0',
  346. '-i', idfile,
  347. '-s', skfile,
  348. '-c', certfile,
  349. '-m', str(lifetime),
  350. '-a', addr]
  351. print("Creating identity key %s for %s with %s" % (
  352. idfile, self._env['nick'], " ".join(cmdline)))
  353. try:
  354. p = subprocess.Popen(cmdline, stdin=subprocess.PIPE)
  355. except OSError as e:
  356. # only catch file not found error
  357. if e.errno == errno.ENOENT:
  358. print("Cannot find tor-gencert binary %r. Use "
  359. "CHUTNEY_TOR_GENCERT environment variable to set the "
  360. "path, or put the binary into $PATH." % tor_gencert)
  361. sys.exit(0)
  362. else:
  363. raise
  364. p.communicate(passphrase + "\n")
  365. assert p.returncode == 0 # XXXX BAD!
  366. def _genRouterKey(self):
  367. """Generate an identity key for this router, unless we already have,
  368. and set up the 'fingerprint' entry in the Environ.
  369. """
  370. datadir = self._env['dir']
  371. tor = self._env['tor']
  372. torrc = self._getTorrcFname()
  373. cmdline = [
  374. tor,
  375. "--quiet",
  376. "--ignore-missing-torrc",
  377. "-f", torrc,
  378. "--list-fingerprint",
  379. "--orport", "1",
  380. "--datadirectory", datadir]
  381. try:
  382. p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
  383. except OSError as e:
  384. # only catch file not found error
  385. if e.errno == errno.ENOENT:
  386. print("Cannot find tor binary %r. Use "
  387. "CHUTNEY_TOR environment variable to set the "
  388. "path, or put the binary into $PATH." % tor)
  389. sys.exit(0)
  390. else:
  391. raise
  392. stdout, stderr = p.communicate()
  393. fingerprint = "".join((stdout.rstrip().split('\n')[-1]).split()[1:])
  394. if not re.match(r'^[A-F0-9]{40}$', fingerprint):
  395. print(("Error when calling %r. It gave %r as a fingerprint "
  396. " and %r on stderr.") % (" ".join(cmdline), stdout, stderr))
  397. sys.exit(1)
  398. self._env['fingerprint'] = fingerprint
  399. def _getAltAuthLines(self, hasbridgeauth=False):
  400. """Return a combination of AlternateDirAuthority,
  401. AlternateHSAuthority and AlternateBridgeAuthority lines for
  402. this Node, appropriately. Non-authorities return ""."""
  403. if not self._env['authority']:
  404. return ""
  405. datadir = self._env['dir']
  406. certfile = os.path.join(datadir, 'keys', "authority_certificate")
  407. v3id = None
  408. with open(certfile, 'r') as f:
  409. for line in f:
  410. if line.startswith("fingerprint"):
  411. v3id = line.split()[1].strip()
  412. break
  413. assert v3id is not None
  414. if self._env['bridgeauthority']:
  415. # Bridge authorities return AlternateBridgeAuthority with
  416. # the 'bridge' flag set.
  417. options = ("AlternateBridgeAuthority",)
  418. self._env['dirserver_flags'] += " bridge"
  419. else:
  420. # Directory authorities return AlternateDirAuthority with
  421. # the 'hs' and 'v3ident' flags set.
  422. # XXXX This next line is needed for 'bridges' but breaks
  423. # 'basic'
  424. if hasbridgeauth:
  425. options = ("AlternateDirAuthority",)
  426. else:
  427. options = ("DirAuthority",)
  428. self._env['dirserver_flags'] += " hs v3ident=%s" % v3id
  429. authlines = ""
  430. for authopt in options:
  431. authlines += "%s %s orport=%s" % (
  432. authopt, self._env['nick'], self._env['orport'])
  433. # It's ok to give an authority's IPv6 address to an IPv4-only
  434. # client or relay: it will and must ignore it
  435. if self._env['ipv6_addr'] is not None:
  436. authlines += " ipv6=%s:%s" % (self._env['ipv6_addr'],
  437. self._env['orport'])
  438. authlines += " %s %s:%s %s\n" % (
  439. self._env['dirserver_flags'], self._env['ip'],
  440. self._env['dirport'], self._env['fingerprint'])
  441. return authlines
  442. def _getBridgeLines(self):
  443. """Return potential Bridge line for this Node. Non-bridge
  444. relays return "".
  445. """
  446. if not self._env['bridge']:
  447. return ""
  448. bridgelines = "Bridge %s:%s\n" % (self._env['ip'],
  449. self._env['orport'])
  450. if self._env['ipv6_addr'] is not None:
  451. bridgelines += "Bridge %s:%s\n" % (self._env['ipv6_addr'],
  452. self._env['orport'])
  453. return bridgelines
  454. class LocalNodeController(NodeController):
  455. def __init__(self, env):
  456. NodeController.__init__(self, env)
  457. self._env = env
  458. def getPid(self):
  459. """Assuming that this node has its pidfile in ${dir}/pid, return
  460. the pid of the running process, or None if there is no pid in the
  461. file.
  462. """
  463. pidfile = os.path.join(self._env['dir'], 'pid')
  464. if not os.path.exists(pidfile):
  465. return None
  466. with open(pidfile, 'r') as f:
  467. return int(f.read())
  468. def isRunning(self, pid=None):
  469. """Return true iff this node is running. (If 'pid' is provided, we
  470. assume that the pid provided is the one of this node. Otherwise
  471. we call getPid().
  472. """
  473. if pid is None:
  474. pid = self.getPid()
  475. if pid is None:
  476. return False
  477. try:
  478. os.kill(pid, 0) # "kill 0" == "are you there?"
  479. except OSError as e:
  480. if e.errno == errno.ESRCH:
  481. return False
  482. raise
  483. # okay, so the process exists. Say "True" for now.
  484. # XXXX check if this is really tor!
  485. return True
  486. def check(self, listRunning=True, listNonRunning=False):
  487. """See if this node is running, stopped, or crashed. If it's running
  488. and listRunning is set, print a short statement. If it's
  489. stopped and listNonRunning is set, then print a short statement.
  490. If it's crashed, print a statement. Return True if the
  491. node is running, false otherwise.
  492. """
  493. # XXX Split this into "check" and "print" parts.
  494. pid = self.getPid()
  495. nick = self._env['nick']
  496. datadir = self._env['dir']
  497. corefile = "core.%s" % pid
  498. if self.isRunning(pid):
  499. if listRunning:
  500. print("%s is running with PID %s" % (nick, pid))
  501. return True
  502. elif os.path.exists(os.path.join(datadir, corefile)):
  503. if listNonRunning:
  504. print("%s seems to have crashed, and left core file %s" % (
  505. nick, corefile))
  506. return False
  507. else:
  508. if listNonRunning:
  509. print("%s is stopped" % nick)
  510. return False
  511. def hup(self):
  512. """Send a SIGHUP to this node, if it's running."""
  513. pid = self.getPid()
  514. nick = self._env['nick']
  515. if self.isRunning(pid):
  516. print("Sending sighup to %s" % nick)
  517. os.kill(pid, signal.SIGHUP)
  518. return True
  519. else:
  520. print("%s is not running" % nick)
  521. return False
  522. def start(self):
  523. """Try to start this node; return True if we succeeded or it was
  524. already running, False if we failed."""
  525. if self.isRunning():
  526. print("%s is already running" % self._env['nick'])
  527. return True
  528. tor_path = self._env['tor']
  529. torrc = self._getTorrcFname()
  530. cmdline = [
  531. tor_path,
  532. "--quiet",
  533. "-f", torrc,
  534. ]
  535. try:
  536. p = subprocess.Popen(cmdline)
  537. except OSError as e:
  538. # only catch file not found error
  539. if e.errno == errno.ENOENT:
  540. print("Cannot find tor binary %r. Use CHUTNEY_TOR "
  541. "environment variable to set the path, or put the "
  542. "binary into $PATH." % tor_path)
  543. sys.exit(0)
  544. else:
  545. raise
  546. if self.waitOnLaunch():
  547. # this requires that RunAsDaemon is set
  548. p.wait()
  549. else:
  550. # this does not require RunAsDaemon to be set, but is slower.
  551. #
  552. # poll() only catches failures before the call itself
  553. # so let's sleep a little first
  554. # this does, of course, slow down process launch
  555. # which can require an adjustment to the voting interval
  556. #
  557. # avoid writing a newline or space when polling
  558. # so output comes out neatly
  559. sys.stdout.write('.')
  560. sys.stdout.flush()
  561. time.sleep(self._env['poll_launch_time'])
  562. p.poll()
  563. if p.returncode is not None and p.returncode != 0:
  564. if self._env['poll_launch_time'] is None:
  565. print("Couldn't launch %s (%s): %s" % (self._env['nick'],
  566. " ".join(cmdline),
  567. p.returncode))
  568. else:
  569. print("Couldn't poll %s (%s) "
  570. "after waiting %s seconds for launch"
  571. ": %s" % (self._env['nick'],
  572. " ".join(cmdline),
  573. self._env['poll_launch_time'],
  574. p.returncode))
  575. return False
  576. return True
  577. def stop(self, sig=signal.SIGINT):
  578. """Try to stop this node by sending it the signal 'sig'."""
  579. pid = self.getPid()
  580. if not self.isRunning(pid):
  581. print("%s is not running" % self._env['nick'])
  582. return
  583. os.kill(pid, sig)
  584. def cleanup_lockfile(self):
  585. lf = self._env['lockfile']
  586. if not self.isRunning() and os.path.exists(lf):
  587. print('Removing stale lock file for {0} ...'.format(
  588. self._env['nick']))
  589. os.remove(lf)
  590. def waitOnLaunch(self):
  591. """Check whether we can wait() for the tor process to launch"""
  592. # TODO: is this the best place for this code?
  593. # RunAsDaemon default is 0
  594. runAsDaemon = False
  595. with open(self._getTorrcFname(), 'r') as f:
  596. for line in f.readlines():
  597. stline = line.strip()
  598. # if the line isn't all whitespace or blank
  599. if len(stline) > 0:
  600. splline = stline.split()
  601. # if the line has at least two tokens on it
  602. if (len(splline) > 0 and
  603. splline[0].lower() == "RunAsDaemon".lower() and
  604. splline[1] == "1"):
  605. # use the RunAsDaemon value from the torrc
  606. # TODO: multiple values?
  607. runAsDaemon = True
  608. if runAsDaemon:
  609. # we must use wait() instead of poll()
  610. self._env['poll_launch_time'] = None
  611. return True
  612. else:
  613. # we must use poll() instead of wait()
  614. if self._env['poll_launch_time'] is None:
  615. self._env['poll_launch_time'] = \
  616. self._env['poll_launch_time_default']
  617. return False
  618. # XXX: document these options
  619. DEFAULTS = {
  620. 'authority': False,
  621. 'bridgeauthority': False,
  622. 'hasbridgeauth': False,
  623. 'relay': False,
  624. 'bridge': False,
  625. 'hs': False,
  626. 'hs_directory': 'hidden_service',
  627. 'hs-hostname': None,
  628. 'connlimit': 60,
  629. 'net_base_dir': get_absolute_net_path(),
  630. 'tor': os.environ.get('CHUTNEY_TOR', 'tor'),
  631. 'tor-gencert': os.environ.get('CHUTNEY_TOR_GENCERT', None),
  632. 'auth_cert_lifetime': 12,
  633. 'ip': os.environ.get('CHUTNEY_LISTEN_ADDRESS', '127.0.0.1'),
  634. # Pre-0.2.8 clients will fail to parse ipv6 auth lines,
  635. # so we default to ipv6_addr None
  636. 'ipv6_addr': os.environ.get('CHUTNEY_LISTEN_ADDRESS_V6', None),
  637. 'dirserver_flags': 'no-v2',
  638. 'chutney_dir': get_absolute_chutney_path(),
  639. 'torrc_fname': '${dir}/torrc',
  640. 'orport_base': 5000,
  641. 'dirport_base': 7000,
  642. 'controlport_base': 8000,
  643. 'socksport_base': 9000,
  644. 'authorities': "AlternateDirAuthority bleargh bad torrc file!",
  645. 'bridges': "Bridge bleargh bad torrc file!",
  646. 'core': True,
  647. # poll_launch_time: None means wait on launch (requires RunAsDaemon),
  648. # otherwise, poll after that many seconds (can be fractional/decimal)
  649. 'poll_launch_time': None,
  650. # Used when poll_launch_time is None, but RunAsDaemon is not set
  651. # Set low so that we don't interfere with the voting interval
  652. 'poll_launch_time_default': 0.1,
  653. # the number of bytes of random data we send on each connection
  654. 'data_bytes': int(os.environ.get('CHUTNEY_DATA_BYTES', 10 * 1024)),
  655. # the number of times each client will connect
  656. 'connection_count': int(os.environ.get('CHUTNEY_CONNECTIONS', 1)),
  657. # Do we want every client to connect to every HS, or one client
  658. # to connect to each HS?
  659. # (Clients choose an exit at random, so this doesn't apply to exits.)
  660. 'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
  661. # How long should verify (and similar commands) wait for a successful
  662. # outcome? (seconds)
  663. # We check BOOTSTRAP_TIME for compatibility with old versions of
  664. # test-network.sh
  665. 'bootstrap_time': int(os.environ.get('CHUTNEY_BOOTSTRAP_TIME',
  666. os.environ.get('BOOTSTRAP_TIME',
  667. 60))),
  668. # the PID of the controlling script (for __OwningControllerProcess)
  669. 'controlling_pid': (int(os.environ.get('CHUTNEY_CONTROLLING_PID', 0))
  670. if 'CHUTNEY_CONTROLLING_PID' in os.environ
  671. else None),
  672. # a DNS config file (for ServerDNSResolvConfFile)
  673. 'dns_conf': (os.environ.get('CHUTNEY_DNS_CONF', '/etc/resolv.conf')
  674. if 'CHUTNEY_DNS_CONF' in os.environ
  675. else None),
  676. }
  677. class TorEnviron(chutney.Templating.Environ):
  678. """Subclass of chutney.Templating.Environ to implement commonly-used
  679. substitutions.
  680. Environment fields provided:
  681. orport, controlport, socksport, dirport: *Port torrc option
  682. dir: DataDirectory torrc option
  683. nick: Nickname torrc option
  684. tor_gencert: name or path of the tor-gencert binary
  685. auth_passphrase: obsoleted by CookieAuthentication
  686. torrc_template_path: path to chutney torrc_templates directory
  687. hs_hostname: the hostname of the key generated by a hidden service
  688. owning_controller_process: the __OwningControllerProcess torrc line,
  689. disabled if tor should continue after the script exits
  690. server_dns_resolv_conf: the ServerDNSResolvConfFile torrc line,
  691. disabled if tor should use the default DNS conf.
  692. If the dns_conf file is missing, this option is also disabled:
  693. otherwise, exits would not work due to tor bug #21900.
  694. Environment fields used:
  695. nodenum: chutney's internal node number for the node
  696. tag: a short text string that represents the type of node
  697. orport_base, controlport_base, socksport_base, dirport_base: the
  698. initial port numbers used by nodenum 0. Each additional node adds
  699. 1 to the port numbers.
  700. tor-gencert (note hyphen): name or path of the tor-gencert binary (if
  701. present)
  702. chutney_dir: directory of the chutney source code
  703. tor: name or path of the tor binary
  704. net_base_dir: path to the chutney net directory
  705. hs_directory: name of the hidden service directory
  706. nick: Nickname torrc option (debugging only)
  707. hs-hostname (note hyphen): cached hidden service hostname value
  708. controlling_pid: the PID of the controlling process. After this
  709. process exits, the child tor processes will exit
  710. dns_conf: the path to a DNS config file for Tor Exits. If this file
  711. is empty or unreadable, Tor will try 127.0.0.1:53.
  712. """
  713. def __init__(self, parent=None, **kwargs):
  714. chutney.Templating.Environ.__init__(self, parent=parent, **kwargs)
  715. def _get_orport(self, my):
  716. return my['orport_base'] + my['nodenum']
  717. def _get_controlport(self, my):
  718. return my['controlport_base'] + my['nodenum']
  719. def _get_socksport(self, my):
  720. return my['socksport_base'] + my['nodenum']
  721. def _get_dirport(self, my):
  722. return my['dirport_base'] + my['nodenum']
  723. def _get_dir(self, my):
  724. return os.path.abspath(os.path.join(my['net_base_dir'],
  725. "nodes",
  726. "%03d%s" % (
  727. my['nodenum'], my['tag'])))
  728. def _get_nick(self, my):
  729. return "test%03d%s" % (my['nodenum'], my['tag'])
  730. def _get_tor_gencert(self, my):
  731. return my['tor-gencert'] or '{0}-gencert'.format(my['tor'])
  732. def _get_auth_passphrase(self, my):
  733. return self['nick'] # OMG TEH SECURE!
  734. def _get_torrc_template_path(self, my):
  735. return [os.path.join(my['chutney_dir'], 'torrc_templates')]
  736. def _get_lockfile(self, my):
  737. return os.path.join(self['dir'], 'lock')
  738. # A hs generates its key on first run,
  739. # so check for it at the last possible moment,
  740. # but cache it in memory to avoid repeatedly reading the file
  741. # XXXX - this is not like the other functions in this class,
  742. # as it reads from a file created by the hidden service
  743. def _get_hs_hostname(self, my):
  744. if my['hs-hostname'] is None:
  745. datadir = my['dir']
  746. # a file containing a single line with the hs' .onion address
  747. hs_hostname_file = os.path.join(datadir, my['hs_directory'],
  748. 'hostname')
  749. try:
  750. with open(hs_hostname_file, 'r') as hostnamefp:
  751. hostname = hostnamefp.read()
  752. # the hostname file ends with a newline
  753. hostname = hostname.strip()
  754. my['hs-hostname'] = hostname
  755. except IOError as e:
  756. print("Error: hs %r error %d: %r opening hostname file '%r'" %
  757. (my['nick'], e.errno, e.strerror, hs_hostname_file))
  758. return my['hs-hostname']
  759. def _get_owning_controller_process(self, my):
  760. cpid = my['controlling_pid']
  761. if cpid is None:
  762. cpid = 0
  763. ocp_line = ('__OwningControllerProcess %d' % (cpid))
  764. # if we want to leave the network running, or controlling_pid is 1
  765. # (or invalid)
  766. if (os.environ.get('CHUTNEY_START_TIME', 0) < 0 or
  767. os.environ.get('CHUTNEY_BOOTSTRAP_TIME', 0) < 0 or
  768. os.environ.get('CHUTNEY_STOP_TIME', 0) < 0 or
  769. cpid <= 1):
  770. return '#' + ocp_line
  771. else:
  772. return ocp_line
  773. # the default resolv.conf path is set at compile time
  774. # there's no easy way to get it out of tor, so we use the typical value
  775. DEFAULT_DNS_RESOLV_CONF = "/etc/resolv.conf"
  776. # if we can't find the specified file, use this one as a substitute
  777. OFFLINE_DNS_RESOLV_CONF = "/dev/null"
  778. def _get_server_dns_resolv_conf(self, my):
  779. if my['dns_conf'] == "":
  780. # if the user asked for tor's default
  781. return "#ServerDNSResolvConfFile using tor's compile-time default"
  782. elif my['dns_conf'] is None:
  783. # if there is no DNS conf file set
  784. print("CHUTNEY_DNS_CONF not specified, using '%s'."
  785. % (DEFAULT_DNS_RESOLV_CONF))
  786. dns_conf = DEFAULT_DNS_RESOLV_CONF
  787. else:
  788. dns_conf = my['dns_conf']
  789. dns_conf = os.path.abspath(my['dns_conf'])
  790. # work around Tor bug #21900, where exits fail when the DNS conf
  791. # file does not exist, or is a broken symlink
  792. # (os.path.exists returns False for broken symbolic links)
  793. if not os.path.exists(dns_conf):
  794. # Issue a warning so the user notices
  795. print("CHUTNEY_DNS_CONF '%s' does not exist, using '%s'."
  796. % (dns_conf, OFFLINE_DNS_RESOLV_CONF))
  797. dns_conf = OFFLINE_DNS_RESOLV_CONF
  798. return "ServerDNSResolvConfFile %s" % (dns_conf)
  799. class Network(object):
  800. """A network of Tor nodes, plus functions to manipulate them
  801. """
  802. def __init__(self, defaultEnviron):
  803. self._nodes = []
  804. self._dfltEnv = defaultEnviron
  805. self._nextnodenum = 0
  806. def _addNode(self, n):
  807. n.setNodenum(self._nextnodenum)
  808. self._nextnodenum += 1
  809. self._nodes.append(n)
  810. def move_aside_nodes_dir(self):
  811. """Move aside the nodes directory, if it exists and is not a link.
  812. Used for backwards-compatibility only: nodes is created as a link to
  813. a new directory with a unique name in the current implementation.
  814. """
  815. nodesdir = get_absolute_nodes_path()
  816. # only move the directory if it exists
  817. if not os.path.exists(nodesdir):
  818. return
  819. # and if it's not a link
  820. if os.path.islink(nodesdir):
  821. return
  822. # subtract 1 second to avoid collisions and get the correct ordering
  823. newdir = get_new_absolute_nodes_path(time.time() - 1)
  824. print("NOTE: renaming %r to %r" % (nodesdir, newdir))
  825. os.rename(nodesdir, newdir)
  826. def create_new_nodes_dir(self):
  827. """Create a new directory with a unique name, and symlink it to nodes
  828. """
  829. # for backwards compatibility, move aside the old nodes directory
  830. # (if it's not a link)
  831. self.move_aside_nodes_dir()
  832. # the unique directory we'll create
  833. newnodesdir = get_new_absolute_nodes_path()
  834. # the canonical name we'll link it to
  835. nodeslink = get_absolute_nodes_path()
  836. # this path should be unique and should not exist
  837. if os.path.exists(newnodesdir):
  838. raise RuntimeError(
  839. 'get_new_absolute_nodes_path returned a path that exists')
  840. # if this path exists, it must be a link
  841. if os.path.exists(nodeslink) and not os.path.islink(nodeslink):
  842. raise RuntimeError(
  843. 'get_absolute_nodes_path returned a path that exists and is not a link')
  844. # create the new, uniquely named directory, and link it to nodes
  845. print("NOTE: creating %r, linking to %r" % (newnodesdir, nodeslink))
  846. # this gets created with mode 0700, that's probably ok
  847. mkdir_p(newnodesdir)
  848. try:
  849. os.unlink(nodeslink)
  850. except OSError as e:
  851. # it's ok if the link doesn't exist, we're just about to make it
  852. if e.errno == errno.ENOENT:
  853. pass
  854. else:
  855. raise
  856. os.symlink(newnodesdir, nodeslink)
  857. def _checkConfig(self):
  858. for n in self._nodes:
  859. n.getBuilder().checkConfig(self)
  860. def configure(self):
  861. self.create_new_nodes_dir()
  862. network = self
  863. altauthlines = []
  864. bridgelines = []
  865. builders = [n.getBuilder() for n in self._nodes]
  866. self._checkConfig()
  867. # XXX don't change node names or types or count if anything is
  868. # XXX running!
  869. for b in builders:
  870. b.preConfig(network)
  871. altauthlines.append(b._getAltAuthLines(
  872. self._dfltEnv['hasbridgeauth']))
  873. bridgelines.append(b._getBridgeLines())
  874. self._dfltEnv['authorities'] = "".join(altauthlines)
  875. self._dfltEnv['bridges'] = "".join(bridgelines)
  876. for b in builders:
  877. b.config(network)
  878. for b in builders:
  879. b.postConfig(network)
  880. def status(self):
  881. statuses = [n.getController().check() for n in self._nodes]
  882. n_ok = len([x for x in statuses if x])
  883. print("%d/%d nodes are running" % (n_ok, len(self._nodes)))
  884. return n_ok == len(self._nodes)
  885. def restart(self):
  886. self.stop()
  887. self.start()
  888. def start(self):
  889. if self._dfltEnv['poll_launch_time'] is not None:
  890. # format polling correctly - avoid printing a newline
  891. sys.stdout.write("Starting nodes")
  892. sys.stdout.flush()
  893. else:
  894. print("Starting nodes")
  895. rv = all([n.getController().start() for n in self._nodes])
  896. # now print a newline unconditionally - this stops poll()ing
  897. # output from being squashed together, at the cost of a blank
  898. # line in wait()ing output
  899. print("")
  900. return rv
  901. def hup(self):
  902. print("Sending SIGHUP to nodes")
  903. return all([n.getController().hup() for n in self._nodes])
  904. def stop(self):
  905. controllers = [n.getController() for n in self._nodes]
  906. for sig, desc in [(signal.SIGINT, "SIGINT"),
  907. (signal.SIGINT, "another SIGINT"),
  908. (signal.SIGKILL, "SIGKILL")]:
  909. print("Sending %s to nodes" % desc)
  910. for c in controllers:
  911. if c.isRunning():
  912. c.stop(sig=sig)
  913. print("Waiting for nodes to finish.")
  914. wrote_dot = False
  915. for n in range(15):
  916. time.sleep(1)
  917. if all(not c.isRunning() for c in controllers):
  918. # make the output clearer by adding a newline
  919. if wrote_dot:
  920. sys.stdout.write("\n")
  921. sys.stdout.flush()
  922. # check for stale lock file when Tor crashes
  923. for c in controllers:
  924. c.cleanup_lockfile()
  925. return
  926. sys.stdout.write(".")
  927. wrote_dot = True
  928. sys.stdout.flush()
  929. for c in controllers:
  930. c.check(listNonRunning=False)
  931. # make the output clearer by adding a newline
  932. if wrote_dot:
  933. sys.stdout.write("\n")
  934. sys.stdout.flush()
  935. def ConfigureNodes(nodelist):
  936. network = _THE_NETWORK
  937. for n in nodelist:
  938. network._addNode(n)
  939. if n._env['bridgeauthority']:
  940. network._dfltEnv['hasbridgeauth'] = True
  941. def getTests():
  942. tests = []
  943. chutney_path = get_absolute_chutney_path()
  944. if len(chutney_path) > 0 and chutney_path[-1] != '/':
  945. chutney_path += "/"
  946. for x in os.listdir(chutney_path + "scripts/chutney_tests/"):
  947. if not x.startswith("_") and os.path.splitext(x)[1] == ".py":
  948. tests.append(os.path.splitext(x)[0])
  949. return tests
  950. def usage(network):
  951. return "\n".join(["Usage: chutney {command/test} {networkfile}",
  952. "Known commands are: %s" % (
  953. " ".join(x for x in dir(network)
  954. if not x.startswith("_"))),
  955. "Known tests are: %s" % (
  956. " ".join(getTests()))
  957. ])
  958. def exit_on_error(err_msg):
  959. print("Error: {0}\n".format(err_msg))
  960. print(usage(_THE_NETWORK))
  961. sys.exit(1)
  962. def runConfigFile(verb, data):
  963. _GLOBALS = dict(_BASE_ENVIRON=_BASE_ENVIRON,
  964. Node=Node,
  965. ConfigureNodes=ConfigureNodes,
  966. _THE_NETWORK=_THE_NETWORK)
  967. exec(data, _GLOBALS)
  968. network = _GLOBALS['_THE_NETWORK']
  969. # let's check if the verb is a valid test and run it
  970. if verb in getTests():
  971. test_module = importlib.import_module("chutney_tests.{}".format(verb))
  972. try:
  973. return test_module.run_test(network)
  974. except AttributeError:
  975. print("Test {!r} has no 'run_test(network)' function".format(verb))
  976. return False
  977. # tell the user we don't know what their verb meant
  978. if not hasattr(network, verb):
  979. print(usage(network))
  980. print("Error: I don't know how to %s." % verb)
  981. return
  982. return getattr(network, verb)()
  983. def parseArgs():
  984. if len(sys.argv) < 3:
  985. exit_on_error("Not enough arguments given.")
  986. if not os.path.isfile(sys.argv[2]):
  987. exit_on_error("Cannot find networkfile: {0}.".format(sys.argv[2]))
  988. return {'network_cfg': sys.argv[2], 'action': sys.argv[1]}
  989. def main():
  990. global _BASE_ENVIRON
  991. global _TORRC_OPTIONS
  992. global _THE_NETWORK
  993. _BASE_ENVIRON = TorEnviron(chutney.Templating.Environ(**DEFAULTS))
  994. # _TORRC_OPTIONS gets initialised on demand as a map of
  995. # "/path/to/tor" => ["SupportedOption1", "SupportedOption2", ...]
  996. # Or it can be pre-populated as a static whitelist of options
  997. _TORRC_OPTIONS = dict()
  998. _THE_NETWORK = Network(_BASE_ENVIRON)
  999. args = parseArgs()
  1000. f = open(args['network_cfg'])
  1001. result = runConfigFile(args['action'], f)
  1002. if result is False:
  1003. return -1
  1004. return 0
  1005. if __name__ == '__main__':
  1006. sys.exit(main())