1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165 |
- #!/usr/bin/env python2
- #
- # Copyright 2011 Nick Mathewson, Michael Stone
- # Copyright 2013 The Tor Project
- #
- # You may do anything with this work that copyright law would normally
- # restrict, so long as you retain the above notice(s) and this license
- # in all redistributed copies and derived works. There is no warranty.
- from __future__ import print_function
- from __future__ import with_statement
- import cgitb
- import os
- import signal
- import subprocess
- import sys
- import re
- import errno
- import time
- import shutil
- import chutney.Templating
- import chutney.Traffic
- _BASE_ENVIRON = None
- _TORRC_OPTIONS = None
- _THE_NETWORK = None
- # Get verbose tracebacks, so we can diagnose better.
- cgitb.enable(format="plain")
- def mkdir_p(d, mode=511):
- """Create directory 'd' and all of its parents as needed. Unlike
- os.makedirs, does not give an error if d already exists.
- 511 is the decimal representation of the octal number 0777. Since
- python2 only supports 0777 and python3 only supports 0o777, we can use
- neither.
- """
- try:
- os.makedirs(d, mode=mode)
- except OSError as e:
- if e.errno == errno.EEXIST:
- return
- raise
- class Node(object):
- """A Node represents a Tor node or a set of Tor nodes. It's created
- in a network configuration file.
- This class is responsible for holding the user's selected node
- configuration, and figuring out how the node needs to be
- configured and launched.
- """
- # Fields:
- # _parent
- # _env
- # _builder
- # _controller
- ########
- # Users are expected to call these:
- def __init__(self, parent=None, **kwargs):
- self._parent = parent
- self._env = self._createEnviron(parent, kwargs)
- self._builder = None
- self._controller = None
- def getN(self, N):
- return [Node(self) for _ in range(N)]
- def specialize(self, **kwargs):
- return Node(parent=self, **kwargs)
- ######
- # Chutney uses these:
- def getBuilder(self):
- """Return a NodeBuilder instance to set up this node (that is, to
- write all the files that need to be in place so that this
- node can be run by a NodeController).
- """
- if self._builder is None:
- self._builder = LocalNodeBuilder(self._env)
- return self._builder
- def getController(self):
- """Return a NodeController instance to control this node (that is,
- to start it, stop it, see if it's running, etc.)
- """
- if self._controller is None:
- self._controller = LocalNodeController(self._env)
- return self._controller
- def setNodenum(self, num):
- """Assign a value to the 'nodenum' element of this node. Each node
- in a network gets its own nodenum.
- """
- self._env['nodenum'] = num
- #####
- # These are internal:
- def _createEnviron(self, parent, argdict):
- """Return an Environ that delegates to the parent node's Environ (if
- there is a parent node), or to the default environment.
- """
- if parent:
- parentenv = parent._env
- else:
- parentenv = self._getDefaultEnviron()
- return TorEnviron(parentenv, **argdict)
- def _getDefaultEnviron(self):
- """Return the default environment. Any variables that we can't find
- set for any particular node, we look for here.
- """
- return _BASE_ENVIRON
- class _NodeCommon(object):
- """Internal helper class for functionality shared by some NodeBuilders
- and some NodeControllers."""
- # XXXX maybe this should turn into a mixin.
- def __init__(self, env):
- self._env = env
- def expand(self, pat, includePath=(".",)):
- return chutney.Templating.Template(pat, includePath).format(self._env)
- def _getTorrcFname(self):
- """Return the name of the file where we'll be writing torrc"""
- return self.expand("${torrc_fname}")
- class NodeBuilder(_NodeCommon):
- """Abstract base class. A NodeBuilder is responsible for doing all the
- one-time prep needed to set up a node in a network.
- """
- def __init__(self, env):
- _NodeCommon.__init__(self, env)
- def checkConfig(self, net):
- """Try to format our torrc; raise an exception if we can't.
- """
- def preConfig(self, net):
- """Called on all nodes before any nodes configure: generates keys as
- needed.
- """
- def config(self, net):
- """Called to configure a node: creates a torrc file for it."""
- def postConfig(self, net):
- """Called on each nodes after all nodes configure."""
- class NodeController(_NodeCommon):
- """Abstract base class. A NodeController is responsible for running a
- node on the network.
- """
- def __init__(self, env):
- _NodeCommon.__init__(self, env)
- def check(self, listRunning=True, listNonRunning=False):
- """See if this node is running, stopped, or crashed. If it's running
- and listRunning is set, print a short statement. If it's
- stopped and listNonRunning is set, then print a short statement.
- If it's crashed, print a statement. Return True if the
- node is running, false otherwise.
- """
- def start(self):
- """Try to start this node; return True if we succeeded or it was
- already running, False if we failed."""
- def stop(self, sig=signal.SIGINT):
- """Try to stop this node by sending it the signal 'sig'."""
- class LocalNodeBuilder(NodeBuilder):
- # Environment members used:
- # torrc -- which torrc file to use
- # torrc_template_path -- path to search for torrc files and include files
- # authority -- bool -- are we an authority?
- # bridgeauthority -- bool -- are we a bridge authority?
- # relay -- bool -- are we a relay?
- # bridge -- bool -- are we a bridge?
- # hs -- bool -- are we a hidden service?
- # nodenum -- int -- set by chutney -- which unique node index is this?
- # dir -- path -- set by chutney -- data directory for this tor
- # tor_gencert -- path to tor_gencert binary
- # tor -- path to tor binary
- # auth_cert_lifetime -- lifetime of authority certs, in months.
- # ip -- IP to listen on (used only if authority or bridge)
- # ipv6_addr -- IPv6 address to listen on (used only if ipv6 bridge)
- # orport, dirport -- (used only if authority)
- # fingerprint -- used only if authority
- # dirserver_flags -- used only if authority
- # nick -- nickname of this router
- # Environment members set
- # fingerprint -- hex router key fingerprint
- # nodenum -- int -- set by chutney -- which unique node index is this?
- def __init__(self, env):
- NodeBuilder.__init__(self, env)
- self._env = env
- def _createTorrcFile(self, checkOnly=False):
- """Write the torrc file for this node, disabling any options
- that are not supported by env's tor binary using comments.
- If checkOnly, just make sure that the formatting is indeed
- possible.
- """
- fn_out = self._getTorrcFname()
- torrc_template = self._getTorrcTemplate()
- output = torrc_template.format(self._env)
- if checkOnly:
- # XXXX Is it time-consuming to format? If so, cache here.
- return
- # now filter the options we're about to write, commenting out
- # the options that the current tor binary doesn't support
- tor = self._env['tor']
- # find the options the current tor binary supports, and cache them
- if tor not in _TORRC_OPTIONS:
- # Note: some versions of tor (e.g. 0.2.4.23) require
- # --list-torrc-options to be the first argument
- cmdline = [
- tor,
- "--list-torrc-options",
- "--hush"]
- try:
- opts = subprocess.check_output(cmdline, bufsize=-1,
- universal_newlines=True)
- except OSError as e:
- # only catch file not found error
- if e.errno == errno.ENOENT:
- print("Cannot find tor binary %r. Use "
- "CHUTNEY_TOR environment variable to set the "
- "path, or put the binary into $PATH." % tor)
- sys.exit(0)
- else:
- raise
- # check we received a list of options, and nothing else
- assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
- torrc_opts = opts.split()
- # cache the options for this tor binary's path
- _TORRC_OPTIONS[tor] = torrc_opts
- else:
- torrc_opts = _TORRC_OPTIONS[tor]
- # check if each option is supported before writing it
- # TODO: what about unsupported values?
- # e.g. tor 0.2.4.23 doesn't support TestingV3AuthInitialVoteDelay 2
- # but later version do. I say throw this one to the user.
- with open(fn_out, 'w') as f:
- # we need to do case-insensitive option comparison
- # even if this is a static whitelist,
- # so we convert to lowercase as close to the loop as possible
- lower_opts = [opt.lower() for opt in torrc_opts]
- # keep ends when splitting lines, so we can write them out
- # using writelines() without messing around with "\n"s
- for line in output.splitlines(True):
- # check if the first word on the line is a supported option,
- # preserving empty lines and comment lines
- sline = line.strip()
- if (len(sline) == 0 or
- sline[0] == '#' or
- sline.split()[0].lower() in lower_opts):
- f.writelines([line])
- else:
- # well, this could get spammy
- # TODO: warn once per option per tor binary
- # TODO: print tor version?
- print(("The tor binary at %r does not support the "
- "option in the torrc line:\n"
- "%r") % (tor, line.strip()))
- # we could decide to skip these lines entirely
- # TODO: write tor version?
- f.writelines(["# " + tor + " unsupported: " + line])
- def _getTorrcTemplate(self):
- """Return the template used to write the torrc for this node."""
- template_path = self._env['torrc_template_path']
- return chutney.Templating.Template("$${include:$torrc}",
- includePath=template_path)
- def _getFreeVars(self):
- """Return a set of the free variables in the torrc template for this
- node.
- """
- template = self._getTorrcTemplate()
- return template.freevars(self._env)
- def checkConfig(self, net):
- """Try to format our torrc; raise an exception if we can't.
- """
- self._createTorrcFile(checkOnly=True)
- def preConfig(self, net):
- """Called on all nodes before any nodes configure: generates keys and
- hidden service directories as needed.
- """
- self._makeDataDir()
- if self._env['authority']:
- self._genAuthorityKey()
- if self._env['relay']:
- self._genRouterKey()
- if self._env['hs']:
- self._makeHiddenServiceDir()
- def config(self, net):
- """Called to configure a node: creates a torrc file for it."""
- self._createTorrcFile()
- # self._createScripts()
- def postConfig(self, net):
- """Called on each nodes after all nodes configure."""
- # self.net.addNode(self)
- pass
- def _makeDataDir(self):
- """Create the data directory (with keys subdirectory) for this node.
- """
- datadir = self._env['dir']
- mkdir_p(os.path.join(datadir, 'keys'))
- def _makeHiddenServiceDir(self):
- """Create the hidden service subdirectory for this node.
- The directory name is stored under the 'hs_directory' environment
- key. It is combined with the 'dir' data directory key to yield the
- path to the hidden service directory.
- 448 is the decimal representation of the octal number 0700. Since
- python2 only supports 0700 and python3 only supports 0o700, we can
- use neither.
- """
- datadir = self._env['dir']
- mkdir_p(os.path.join(datadir, self._env['hs_directory']), 448)
- def _genAuthorityKey(self):
- """Generate an authority identity and signing key for this authority,
- if they do not already exist."""
- datadir = self._env['dir']
- tor_gencert = self._env['tor_gencert']
- lifetime = self._env['auth_cert_lifetime']
- idfile = os.path.join(datadir, 'keys', "authority_identity_key")
- skfile = os.path.join(datadir, 'keys', "authority_signing_key")
- certfile = os.path.join(datadir, 'keys', "authority_certificate")
- addr = self.expand("${ip}:${dirport}")
- passphrase = self._env['auth_passphrase']
- if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
- return
- cmdline = [
- tor_gencert,
- '--create-identity-key',
- '--passphrase-fd', '0',
- '-i', idfile,
- '-s', skfile,
- '-c', certfile,
- '-m', str(lifetime),
- '-a', addr]
- print("Creating identity key %s for %s with %s" % (
- idfile, self._env['nick'], " ".join(cmdline)))
- try:
- p = subprocess.Popen(cmdline, stdin=subprocess.PIPE)
- except OSError as e:
- # only catch file not found error
- if e.errno == errno.ENOENT:
- print("Cannot find tor-gencert binary %r. Use "
- "CHUTNEY_TOR_GENCERT environment variable to set the "
- "path, or put the binary into $PATH." % tor_gencert)
- sys.exit(0)
- else:
- raise
- p.communicate(passphrase + "\n")
- assert p.returncode == 0 # XXXX BAD!
- def _genRouterKey(self):
- """Generate an identity key for this router, unless we already have,
- and set up the 'fingerprint' entry in the Environ.
- """
- datadir = self._env['dir']
- tor = self._env['tor']
- torrc = self._getTorrcFname()
- cmdline = [
- tor,
- "--quiet",
- "--ignore-missing-torrc",
- "-f", torrc,
- "--list-fingerprint",
- "--orport", "1",
- "--datadirectory", datadir]
- try:
- p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
- except OSError as e:
- # only catch file not found error
- if e.errno == errno.ENOENT:
- print("Cannot find tor binary %r. Use "
- "CHUTNEY_TOR environment variable to set the "
- "path, or put the binary into $PATH." % tor)
- sys.exit(0)
- else:
- raise
- stdout, stderr = p.communicate()
- fingerprint = "".join((stdout.rstrip().split('\n')[-1]).split()[1:])
- if not re.match(r'^[A-F0-9]{40}$', fingerprint):
- print(("Error when calling %r. It gave %r as a fingerprint "
- " and %r on stderr.") % (" ".join(cmdline), stdout, stderr))
- sys.exit(1)
- self._env['fingerprint'] = fingerprint
- def _getAltAuthLines(self, hasbridgeauth=False):
- """Return a combination of AlternateDirAuthority,
- AlternateHSAuthority and AlternateBridgeAuthority lines for
- this Node, appropriately. Non-authorities return ""."""
- if not self._env['authority']:
- return ""
- datadir = self._env['dir']
- certfile = os.path.join(datadir, 'keys', "authority_certificate")
- v3id = None
- with open(certfile, 'r') as f:
- for line in f:
- if line.startswith("fingerprint"):
- v3id = line.split()[1].strip()
- break
- assert v3id is not None
- if self._env['bridgeauthority']:
- # Bridge authorities return AlternateBridgeAuthority with
- # the 'bridge' flag set.
- options = ("AlternateBridgeAuthority",)
- self._env['dirserver_flags'] += " bridge"
- else:
- # Directory authorities return AlternateDirAuthority with
- # the 'hs' and 'v3ident' flags set.
- # XXXX This next line is needed for 'bridges' but breaks
- # 'basic'
- if hasbridgeauth:
- options = ("AlternateDirAuthority",)
- else:
- options = ("DirAuthority",)
- self._env['dirserver_flags'] += " hs v3ident=%s" % v3id
- authlines = ""
- for authopt in options:
- authlines += "%s %s orport=%s %s %s:%s %s\n" % (
- authopt, self._env['nick'], self._env['orport'],
- self._env['dirserver_flags'], self._env['ip'],
- self._env['dirport'], self._env['fingerprint'])
- return authlines
- def _getBridgeLines(self):
- """Return potential Bridge line for this Node. Non-bridge
- relays return "".
- """
- if not self._env['bridge']:
- return ""
- bridgelines = "Bridge %s:%s\n" % (self._env['ip'],
- self._env['orport'])
- if self._env['ipv6_addr'] is not None:
- bridgelines += "Bridge %s:%s\n" % (self._env['ipv6_addr'],
- self._env['orport'])
- return bridgelines
- class LocalNodeController(NodeController):
- def __init__(self, env):
- NodeController.__init__(self, env)
- self._env = env
- def getPid(self):
- """Assuming that this node has its pidfile in ${dir}/pid, return
- the pid of the running process, or None if there is no pid in the
- file.
- """
- pidfile = os.path.join(self._env['dir'], 'pid')
- if not os.path.exists(pidfile):
- return None
- with open(pidfile, 'r') as f:
- return int(f.read())
- def isRunning(self, pid=None):
- """Return true iff this node is running. (If 'pid' is provided, we
- assume that the pid provided is the one of this node. Otherwise
- we call getPid().
- """
- if pid is None:
- pid = self.getPid()
- if pid is None:
- return False
- try:
- os.kill(pid, 0) # "kill 0" == "are you there?"
- except OSError as e:
- if e.errno == errno.ESRCH:
- return False
- raise
- # okay, so the process exists. Say "True" for now.
- # XXXX check if this is really tor!
- return True
- def check(self, listRunning=True, listNonRunning=False):
- """See if this node is running, stopped, or crashed. If it's running
- and listRunning is set, print a short statement. If it's
- stopped and listNonRunning is set, then print a short statement.
- If it's crashed, print a statement. Return True if the
- node is running, false otherwise.
- """
- # XXX Split this into "check" and "print" parts.
- pid = self.getPid()
- nick = self._env['nick']
- datadir = self._env['dir']
- corefile = "core.%s" % pid
- if self.isRunning(pid):
- if listRunning:
- print("%s is running with PID %s" % (nick, pid))
- return True
- elif os.path.exists(os.path.join(datadir, corefile)):
- if listNonRunning:
- print("%s seems to have crashed, and left core file %s" % (
- nick, corefile))
- return False
- else:
- if listNonRunning:
- print("%s is stopped" % nick)
- return False
- def hup(self):
- """Send a SIGHUP to this node, if it's running."""
- pid = self.getPid()
- nick = self._env['nick']
- if self.isRunning(pid):
- print("Sending sighup to %s" % nick)
- os.kill(pid, signal.SIGHUP)
- return True
- else:
- print("%s is not running" % nick)
- return False
- def start(self):
- """Try to start this node; return True if we succeeded or it was
- already running, False if we failed."""
- if self.isRunning():
- print("%s is already running" % self._env['nick'])
- return True
- tor_path = self._env['tor']
- torrc = self._getTorrcFname()
- cmdline = [
- tor_path,
- "--quiet",
- "-f", torrc,
- ]
- try:
- p = subprocess.Popen(cmdline)
- except OSError as e:
- # only catch file not found error
- if e.errno == errno.ENOENT:
- print("Cannot find tor binary %r. Use CHUTNEY_TOR "
- "environment variable to set the path, or put the "
- "binary into $PATH." % tor_path)
- sys.exit(0)
- else:
- raise
- if self.waitOnLaunch():
- # this requires that RunAsDaemon is set
- p.wait()
- else:
- # this does not require RunAsDaemon to be set, but is slower.
- #
- # poll() only catches failures before the call itself
- # so let's sleep a little first
- # this does, of course, slow down process launch
- # which can require an adjustment to the voting interval
- #
- # avoid writing a newline or space when polling
- # so output comes out neatly
- sys.stdout.write('.')
- sys.stdout.flush()
- time.sleep(self._env['poll_launch_time'])
- p.poll()
- if p.returncode is not None and p.returncode != 0:
- if self._env['poll_launch_time'] is None:
- print("Couldn't launch %s (%s): %s" % (self._env['nick'],
- " ".join(cmdline),
- p.returncode))
- else:
- print("Couldn't poll %s (%s) "
- "after waiting %s seconds for launch"
- ": %s" % (self._env['nick'],
- " ".join(cmdline),
- self._env['poll_launch_time'],
- p.returncode))
- return False
- return True
- def stop(self, sig=signal.SIGINT):
- """Try to stop this node by sending it the signal 'sig'."""
- pid = self.getPid()
- if not self.isRunning(pid):
- print("%s is not running" % self._env['nick'])
- return
- os.kill(pid, sig)
- def cleanup_lockfile(self):
- lf = self._env['lockfile']
- if not self.isRunning() and os.path.exists(lf):
- print('Removing stale lock file for {0} ...'.format(
- self._env['nick']))
- os.remove(lf)
- def waitOnLaunch(self):
- """Check whether we can wait() for the tor process to launch"""
- # TODO: is this the best place for this code?
- # RunAsDaemon default is 0
- runAsDaemon = False
- with open(self._getTorrcFname(), 'r') as f:
- for line in f.readlines():
- stline = line.strip()
- # if the line isn't all whitespace or blank
- if len(stline) > 0:
- splline = stline.split()
- # if the line has at least two tokens on it
- if (len(splline) > 0 and
- splline[0].lower() == "RunAsDaemon".lower() and
- splline[1] == "1"):
- # use the RunAsDaemon value from the torrc
- # TODO: multiple values?
- runAsDaemon = True
- if runAsDaemon:
- # we must use wait() instead of poll()
- self._env['poll_launch_time'] = None
- return True
- else:
- # we must use poll() instead of wait()
- if self._env['poll_launch_time'] is None:
- self._env['poll_launch_time'] = \
- self._env['poll_launch_time_default']
- return False
- DEFAULTS = {
- 'authority': False,
- 'bridgeauthority': False,
- 'hasbridgeauth': False,
- 'relay': False,
- 'bridge': False,
- 'hs': False,
- 'hs_directory': 'hidden_service',
- 'hs-hostname': None,
- 'connlimit': 60,
- 'net_base_dir': os.environ.get('CHUTNEY_DATA_DIR', 'net'),
- 'tor': os.environ.get('CHUTNEY_TOR', 'tor'),
- 'tor-gencert': os.environ.get('CHUTNEY_TOR_GENCERT', None),
- 'auth_cert_lifetime': 12,
- 'ip': '127.0.0.1',
- 'ipv6_addr': None,
- 'dirserver_flags': 'no-v2',
- 'chutney_dir': '.',
- 'torrc_fname': '${dir}/torrc',
- 'orport_base': 5000,
- 'dirport_base': 7000,
- 'controlport_base': 8000,
- 'socksport_base': 9000,
- 'authorities': "AlternateDirAuthority bleargh bad torrc file!",
- 'bridges': "Bridge bleargh bad torrc file!",
- 'core': True,
- # poll_launch_time: None means wait on launch (requires RunAsDaemon),
- # otherwise, poll after that many seconds (can be fractional/decimal)
- 'poll_launch_time': None,
- # Used when poll_launch_time is None, but RunAsDaemon is not set
- # Set low so that we don't interfere with the voting interval
- 'poll_launch_time_default': 0.1,
- # the number of bytes of random data we send on each connection
- 'data_bytes': int(os.environ.get('CHUTNEY_DATA_BYTES', 10 * 1024)),
- # the number of times each client will connect
- 'connection_count': int(os.environ.get('CHUTNEY_CONNECTIONS', 1)),
- # Do we want every client to connect to every HS, or one client
- # to connect to each HS?
- # (Clients choose an exit at random, so this doesn't apply to exits.)
- 'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
- }
- class TorEnviron(chutney.Templating.Environ):
- """Subclass of chutney.Templating.Environ to implement commonly-used
- substitutions.
- Environment fields provided:
- orport, controlport, socksport, dirport:
- dir:
- nick:
- tor_gencert:
- auth_passphrase:
- torrc_template_path:
- hs_hostname:
- Environment fields used:
- nodenum
- tag
- orport_base, controlport_base, socksport_base, dirport_base
- tor-gencert (note hyphen)
- chutney_dir
- tor
- dir
- hs_directory
- nick (debugging only)
- hs-hostname (note hyphen)
- XXXX document the above. Or document all fields in one place?
- """
- def __init__(self, parent=None, **kwargs):
- chutney.Templating.Environ.__init__(self, parent=parent, **kwargs)
- def _get_orport(self, my):
- return my['orport_base'] + my['nodenum']
- def _get_controlport(self, my):
- return my['controlport_base'] + my['nodenum']
- def _get_socksport(self, my):
- return my['socksport_base'] + my['nodenum']
- def _get_dirport(self, my):
- return my['dirport_base'] + my['nodenum']
- def _get_dir(self, my):
- return os.path.abspath(os.path.join(my['net_base_dir'],
- "nodes",
- "%03d%s" % (
- my['nodenum'], my['tag'])))
- def _get_nick(self, my):
- return "test%03d%s" % (my['nodenum'], my['tag'])
- def _get_tor_gencert(self, my):
- return my['tor-gencert'] or '{0}-gencert'.format(my['tor'])
- def _get_auth_passphrase(self, my):
- return self['nick'] # OMG TEH SECURE!
- def _get_torrc_template_path(self, my):
- return [os.path.join(my['chutney_dir'], 'torrc_templates')]
- def _get_lockfile(self, my):
- return os.path.join(self['dir'], 'lock')
- # A hs generates its key on first run,
- # so check for it at the last possible moment,
- # but cache it in memory to avoid repeatedly reading the file
- # XXXX - this is not like the other functions in this class,
- # as it reads from a file created by the hidden service
- def _get_hs_hostname(self, my):
- if my['hs-hostname'] is None:
- datadir = my['dir']
- # a file containing a single line with the hs' .onion address
- hs_hostname_file = os.path.join(datadir, my['hs_directory'],
- 'hostname')
- try:
- with open(hs_hostname_file, 'r') as hostnamefp:
- hostname = hostnamefp.read()
- # the hostname file ends with a newline
- hostname = hostname.strip()
- my['hs-hostname'] = hostname
- except IOError as e:
- print("Error: hs %r error %d: %r opening hostname file '%r'" %
- (my['nick'], e.errno, e.strerror, hs_hostname_file))
- return my['hs-hostname']
- class Network(object):
- """A network of Tor nodes, plus functions to manipulate them
- """
- def __init__(self, defaultEnviron):
- self._nodes = []
- self._dfltEnv = defaultEnviron
- self._nextnodenum = 0
- def _addNode(self, n):
- n.setNodenum(self._nextnodenum)
- self._nextnodenum += 1
- self._nodes.append(n)
- def move_aside_nodes(self):
- net_base_dir = os.environ.get('CHUTNEY_DATA_DIR', 'net')
- nodesdir = os.path.join(net_base_dir, 'nodes')
- if not os.path.exists(nodesdir):
- return
- newdir = newdirbase = "%s.%d" % (nodesdir, time.time())
- i = 0
- while os.path.exists(newdir):
- i += 1
- newdir = "%s.%d" % (newdirbase, i)
- print("NOTE: renaming %r to %r" % (nodesdir, newdir))
- os.rename(nodesdir, newdir)
- def _checkConfig(self):
- for n in self._nodes:
- n.getBuilder().checkConfig(self)
- def configure(self):
- # shutil.rmtree(os.path.join(os.getcwd(),'net','nodes'),ignore_errors=True)
- self.move_aside_nodes()
- network = self
- altauthlines = []
- bridgelines = []
- builders = [n.getBuilder() for n in self._nodes]
- self._checkConfig()
- # XXX don't change node names or types or count if anything is
- # XXX running!
- for b in builders:
- b.preConfig(network)
- altauthlines.append(b._getAltAuthLines(
- self._dfltEnv['hasbridgeauth']))
- bridgelines.append(b._getBridgeLines())
- self._dfltEnv['authorities'] = "".join(altauthlines)
- self._dfltEnv['bridges'] = "".join(bridgelines)
- for b in builders:
- b.config(network)
- for b in builders:
- b.postConfig(network)
- def status(self):
- statuses = [n.getController().check() for n in self._nodes]
- n_ok = len([x for x in statuses if x])
- print("%d/%d nodes are running" % (n_ok, len(self._nodes)))
- return n_ok == len(self._nodes)
- def restart(self):
- self.stop()
- self.start()
- def start(self):
- if self._dfltEnv['poll_launch_time'] is not None:
- # format polling correctly - avoid printing a newline
- sys.stdout.write("Starting nodes")
- sys.stdout.flush()
- else:
- print("Starting nodes")
- rv = all([n.getController().start() for n in self._nodes])
- # now print a newline unconditionally - this stops poll()ing
- # output from being squashed together, at the cost of a blank
- # line in wait()ing output
- print("")
- return rv
- def hup(self):
- print("Sending SIGHUP to nodes")
- return all([n.getController().hup() for n in self._nodes])
- def stop(self):
- controllers = [n.getController() for n in self._nodes]
- for sig, desc in [(signal.SIGINT, "SIGINT"),
- (signal.SIGINT, "another SIGINT"),
- (signal.SIGKILL, "SIGKILL")]:
- print("Sending %s to nodes" % desc)
- for c in controllers:
- if c.isRunning():
- c.stop(sig=sig)
- print("Waiting for nodes to finish.")
- for n in range(15):
- time.sleep(1)
- if all(not c.isRunning() for c in controllers):
- # check for stale lock file when Tor crashes
- for c in controllers:
- c.cleanup_lockfile()
- return
- sys.stdout.write(".")
- sys.stdout.flush()
- for c in controllers:
- c.check(listNonRunning=False)
- def verify(self):
- print("Verifying data transmission:")
- status = self._verify_traffic()
- print("Transmission: %s" % ("Success" if status else "Failure"))
- if not status:
- # TODO: allow the debug flag to be passed as an argument to
- # src/test/test-network.sh and chutney
- print("Set 'debug_flag = True' in Traffic.py to diagnose.")
- return status
- def _verify_traffic(self):
- """Verify (parts of) the network by sending traffic through it
- and verify what is received."""
- LISTEN_PORT = 4747 # FIXME: Do better! Note the default exit policy.
- # HSs must have a HiddenServiceDir with
- # "HiddenServicePort <HS_PORT> 127.0.0.1:<LISTEN_PORT>"
- HS_PORT = 5858
- # The amount of data to send between each source-sink pair,
- # each time the source connects.
- # We create a source-sink pair for each (bridge) client to an exit,
- # and a source-sink pair for a (bridge) client to each hidden service
- DATALEN = self._dfltEnv['data_bytes']
- # Print a dot each time a sink verifies this much data
- DOTDATALEN = 5 * 1024 * 1024 # Octets.
- TIMEOUT = 3 # Seconds.
- # Calculate the amount of random data we should use
- randomlen = self._calculate_randomlen(DATALEN)
- reps = self._calculate_reps(DATALEN, randomlen)
- # sanity check
- if reps == 0:
- DATALEN = 0
- # Get the random data
- if randomlen > 0:
- # print a dot after every DOTDATALEN data is verified, rounding up
- dot_reps = self._calculate_reps(DOTDATALEN, randomlen)
- # make sure we get at least one dot per transmission
- dot_reps = min(reps, dot_reps)
- with open('/dev/urandom', 'r') as randfp:
- tmpdata = randfp.read(randomlen)
- else:
- dot_reps = 0
- tmpdata = {}
- # now make the connections
- bind_to = ('127.0.0.1', LISTEN_PORT)
- tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT, reps,
- dot_reps)
- client_list = filter(lambda n:
- n._env['tag'] == 'c' or n._env['tag'] == 'bc',
- self._nodes)
- exit_list = filter(lambda n:
- ('exit' in n._env.keys()) and n._env['exit'] == 1,
- self._nodes)
- hs_list = filter(lambda n:
- n._env['tag'] == 'h',
- self._nodes)
- if len(client_list) == 0:
- print(" Unable to verify network: no client nodes available")
- return False
- if len(exit_list) == 0 and len(hs_list) == 0:
- print(" Unable to verify network: no exit/hs nodes available")
- print(" Exit nodes must be declared 'relay=1, exit=1'")
- print(" HS nodes must be declared 'tag=\"hs\"'")
- return False
- print("Connecting:")
- # the number of tor nodes in paths which will send DATALEN data
- # if a node is used in two paths, we count it twice
- # this is a lower bound, as cannabilised circuits are one node longer
- total_path_node_count = 0
- total_path_node_count += self._configure_exits(tt, bind_to, tmpdata,
- reps, client_list,
- exit_list, LISTEN_PORT)
- total_path_node_count += self._configure_hs(tt, tmpdata, reps,
- client_list, hs_list,
- HS_PORT, LISTEN_PORT)
- print("Transmitting Data:")
- start_time = time.clock()
- status = tt.run()
- end_time = time.clock()
- # if we fail, don't report the bandwidth
- if not status:
- return status
- # otherwise, report bandwidth used, if sufficient data was transmitted
- self._report_bandwidth(DATALEN, total_path_node_count,
- start_time, end_time)
- return status
- # In order to performance test a tor network, we need to transmit
- # several hundred megabytes of data or more. Passing around this
- # much data in Python has its own performance impacts, so we provide
- # a smaller amount of random data instead, and repeat it to DATALEN
- def _calculate_randomlen(self, datalen):
- MAX_RANDOMLEN = 128 * 1024 # Octets.
- if datalen > MAX_RANDOMLEN:
- return MAX_RANDOMLEN
- else:
- return datalen
- def _calculate_reps(self, datalen, replen):
- # sanity checks
- if datalen == 0 or replen == 0:
- return 0
- # effectively rounds datalen up to the nearest replen
- if replen < datalen:
- return (datalen + replen - 1) / replen
- else:
- return 1
- # if there are any exits, each client / bridge client transmits
- # via 4 nodes (including the client) to an arbitrary exit
- # Each client binds directly to 127.0.0.1:LISTEN_PORT via an Exit relay
- def _configure_exits(self, tt, bind_to, tmpdata, reps, client_list,
- exit_list, LISTEN_PORT):
- CLIENT_EXIT_PATH_NODES = 4
- connection_count = self._dfltEnv['connection_count']
- exit_path_node_count = 0
- if len(exit_list) > 0:
- exit_path_node_count += (len(client_list) *
- CLIENT_EXIT_PATH_NODES *
- connection_count)
- for op in client_list:
- print(" Exit to %s:%d via client %s:%s"
- % ('127.0.0.1', LISTEN_PORT,
- 'localhost', op._env['socksport']))
- for i in range(connection_count):
- proxy = ('localhost', int(op._env['socksport']))
- tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, proxy,
- reps))
- return exit_path_node_count
- # The HS redirects .onion connections made to hs_hostname:HS_PORT
- # to the Traffic Tester's 127.0.0.1:LISTEN_PORT
- # an arbitrary client / bridge client transmits via 8 nodes
- # (including the client and hs) to each hidden service
- # Instead of binding directly to LISTEN_PORT via an Exit relay,
- # we bind to hs_hostname:HS_PORT via a hidden service connection
- def _configure_hs(self, tt, tmpdata, reps, client_list, hs_list, HS_PORT,
- LISTEN_PORT):
- CLIENT_HS_PATH_NODES = 8
- connection_count = self._dfltEnv['connection_count']
- hs_path_node_count = (len(hs_list) * CLIENT_HS_PATH_NODES *
- connection_count)
- # Each client in hs_client_list connects to each hs
- if self._dfltEnv['hs_multi_client']:
- hs_client_list = client_list
- hs_path_node_count *= len(client_list)
- else:
- # only use the first client in the list
- hs_client_list = client_list[:1]
- # Setup the connections from each client in hs_client_list to each hs
- for hs in hs_list:
- hs_bind_to = (hs._env['hs_hostname'], HS_PORT)
- for client in hs_client_list:
- print(" HS to %s:%d (%s:%d) via client %s:%s"
- % (hs._env['hs_hostname'], HS_PORT,
- '127.0.0.1', LISTEN_PORT,
- 'localhost', client._env['socksport']))
- for i in range(connection_count):
- proxy = ('localhost', int(client._env['socksport']))
- tt.add(chutney.Traffic.Source(tt, hs_bind_to, tmpdata,
- proxy, reps))
- return hs_path_node_count
- # calculate the single stream bandwidth and overall tor bandwidth
- # the single stream bandwidth is the bandwidth of the
- # slowest stream of all the simultaneously transmitted streams
- # the overall bandwidth estimates the simultaneous bandwidth between
- # all tor nodes over all simultaneous streams, assuming:
- # * minimum path lengths (no cannibalized circuits)
- # * unlimited network bandwidth (that is, localhost)
- # * tor performance is CPU-limited
- # This be used to estimate the bandwidth capacity of a CPU-bound
- # tor relay running on this machine
- def _report_bandwidth(self, data_length, total_path_node_count,
- start_time, end_time):
- # otherwise, if we sent at least 5 MB cumulative total, and
- # it took us at least a second to send, report bandwidth
- MIN_BWDATA = 5 * 1024 * 1024 # Octets.
- MIN_ELAPSED_TIME = 1.0 # Seconds.
- cumulative_data_sent = total_path_node_count * data_length
- elapsed_time = end_time - start_time
- if (cumulative_data_sent >= MIN_BWDATA and
- elapsed_time >= MIN_ELAPSED_TIME):
- # Report megabytes per second
- BWDIVISOR = 1024*1024
- single_stream_bandwidth = (data_length / elapsed_time / BWDIVISOR)
- overall_bandwidth = (cumulative_data_sent / elapsed_time /
- BWDIVISOR)
- print("Single Stream Bandwidth: %.2f MBytes/s"
- % single_stream_bandwidth)
- print("Overall tor Bandwidth: %.2f MBytes/s"
- % overall_bandwidth)
- def ConfigureNodes(nodelist):
- network = _THE_NETWORK
- for n in nodelist:
- network._addNode(n)
- if n._env['bridgeauthority']:
- network._dfltEnv['hasbridgeauth'] = True
- def usage(network):
- return "\n".join(["Usage: chutney {command} {networkfile}",
- "Known commands are: %s" % (
- " ".join(x for x in dir(network)
- if not x.startswith("_")))])
- def exit_on_error(err_msg):
- print("Error: {0}\n".format(err_msg))
- print(usage(_THE_NETWORK))
- sys.exit(1)
- def runConfigFile(verb, data):
- _GLOBALS = dict(_BASE_ENVIRON=_BASE_ENVIRON,
- Node=Node,
- ConfigureNodes=ConfigureNodes,
- _THE_NETWORK=_THE_NETWORK)
- exec(data, _GLOBALS)
- network = _GLOBALS['_THE_NETWORK']
- if not hasattr(network, verb):
- print(usage(network))
- print("Error: I don't know how to %s." % verb)
- return
- return getattr(network, verb)()
- def parseArgs():
- if len(sys.argv) < 3:
- exit_on_error("Not enough arguments given.")
- if not os.path.isfile(sys.argv[2]):
- exit_on_error("Cannot find networkfile: {0}.".format(sys.argv[2]))
- return {'network_cfg': sys.argv[2], 'action': sys.argv[1]}
- def main():
- global _BASE_ENVIRON
- global _TORRC_OPTIONS
- global _THE_NETWORK
- _BASE_ENVIRON = TorEnviron(chutney.Templating.Environ(**DEFAULTS))
- # _TORRC_OPTIONS gets initialised on demand as a map of
- # "/path/to/tor" => ["SupportedOption1", "SupportedOption2", ...]
- # Or it can be pre-populated as a static whitelist of options
- _TORRC_OPTIONS = dict()
- _THE_NETWORK = Network(_BASE_ENVIRON)
- args = parseArgs()
- f = open(args['network_cfg'])
- result = runConfigFile(args['action'], f)
- if result is False:
- return -1
- return 0
- if __name__ == '__main__':
- sys.exit(main())
|