Changes to support multiple servers

lib/chutney/TorNet.py

@@ -167,10 +167,15 @@ def _warnMissingTor(tor_path, cmdline, tor_name="tor"):
     """Log a warning that the binary tor_name can't be found at tor_path
        while running cmdline.
     """
-    print(("Cannot find the {} binary at '{}' for the command line '{}'. " +
-           "Set the TOR_DIR environment variable to the directory " +
-           "containing {}.")
-          .format(tor_name, tor_path, " ".join(cmdline), tor_name))
+    help_msg_fmt = "Set the '{}' environment variable to the directory containing '{}'."
+    if tor_name == "tor":
+        help_msg = help_msg_fmt.format("CHUTNEY_TOR", tor_name)
+    elif tor_name == "tor-gencert":
+        help_msg = help_msg_fmt.format("CHUTNEY_TOR_GENCERT", tor_name)
+    else:
+        help_msg = ""
+    print(("Cannot find the {} binary at '{}' for the command line '{}'. {}")
+          .format(tor_name, tor_path, " ".join(cmdline), help_msg))
 
 def run_tor(cmdline, exit_on_missing=True):
     """Run the tor command line cmdline, which must start with the path or
@@ -257,7 +262,7 @@ def run_tor_gencert(cmdline, passphrase):
                         tor_name="tor-gencert",
                         stdin=subprocess.PIPE)
     (stdouterr, empty_stderr) = p.communicate(passphrase + "\n")
-    debug(stdouterr)
+    print(stdouterr)
     assert p.returncode == 0  # XXXX BAD!
     assert empty_stderr is None
     return stdouterr
@@ -282,14 +287,19 @@ def tor_gencert_exists(gencert):
         return False
 
 @chutney.Util.memoized
-def get_tor_version(tor):
+def get_tor_version(tor, remote_hostname=None):
     """Return the version of the tor binary.
        Versions are cached for each unique tor path.
     """
-    cmdline = [
+    cmdline = []
+
+    if remote_hostname != None:
+        cmdline.extend(['ssh', remote_hostname])
+
+    cmdline.extend([
         tor,
         "--version",
-    ]
+    ])
     tor_version = run_tor(cmdline)
     # clean it up a bit
     tor_version = tor_version.strip()
@@ -301,14 +311,19 @@ def get_tor_version(tor):
     return tor_version
 
 @chutney.Util.memoized
-def get_torrc_options(tor):
+def get_torrc_options(tor, remote_hostname=None):
     """Return the torrc options supported by the tor binary.
        Options are cached for each unique tor path.
     """
-    cmdline = [
+    cmdline = []
+
+    if remote_hostname != None:
+        cmdline.extend(['ssh', remote_hostname])
+
+    cmdline.extend([
         tor,
         "--list-torrc-options",
-    ]
+    ])
     opts = run_tor(cmdline)
     # check we received a list of options, and nothing else
     assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
@@ -397,7 +412,10 @@ class Node(object):
            node can be run by a NodeController).
         """
         if self._builder is None:
-            self._builder = LocalNodeBuilder(self._env)
+            if self._env['remote_hostname'] != None:
+                self._builder = RemoteNodeBuilder(self._env)
+            else:
+                self._builder = LocalNodeBuilder(self._env)
         return self._builder
 
     def getController(self):
@@ -405,7 +423,10 @@ class Node(object):
            to start it, stop it, see if it's running, etc.)
         """
         if self._controller is None:
-            self._controller = LocalNodeController(self._env)
+            if self._env['remote_hostname'] != None:
+                self._controller = RemoteNodeController(self._env)
+            else:
+                self._controller = LocalNodeController(self._env)
         return self._controller
 
     def setNodenum(self, num):
@@ -502,11 +523,423 @@ class NodeController(_NodeCommon):
         """Try to start this node; return True if we succeeded or it was
            already running, False if we failed."""
 
-    def stop(self, sig=signal.SIGINT):
-        """Try to stop this node by sending it the signal 'sig'."""
+    def stop(self, sig=signal.SIGINT):
+        """Try to stop this node by sending it the signal 'sig'."""
+
+
+class LocalNodeBuilder(NodeBuilder):
+
+    # Environment members used:
+    # torrc -- which torrc file to use
+    # torrc_template_path -- path to search for torrc files and include files
+    # authority -- bool -- are we an authority?
+    # bridgeauthority -- bool -- are we a bridge authority?
+    # relay -- bool -- are we a relay?
+    # bridge -- bool -- are we a bridge?
+    # hs -- bool -- are we a hidden service?
+    # nodenum -- int -- set by chutney -- which unique node index is this?
+    # dir -- path -- set by chutney -- data directory for this tor
+    # tor_gencert -- path to tor_gencert binary
+    # tor -- path to tor binary
+    # auth_cert_lifetime -- lifetime of authority certs, in months.
+    # ip -- primary IP address (usually IPv4) to listen on
+    # ipv6_addr -- secondary IP address (usually IPv6) to listen on
+    # orport, dirport -- used on authorities, relays, and bridges. The orport
+    #                    is used for both IPv4 and IPv6, if present
+    # fingerprint -- used only if authority
+    # dirserver_flags -- used only if authority
+    # nick -- nickname of this router
+
+    # Environment members set
+    # fingerprint -- hex router key fingerprint
+    # nodenum -- int -- set by chutney -- which unique node index is this?
+
+    def __init__(self, env):
+        NodeBuilder.__init__(self, env)
+        self._env = env
+
+    def _createTorrcFile(self, checkOnly=False):
+        """Write the torrc file for this node, disabling any options
+           that are not supported by env's tor binary using comments.
+           If checkOnly, just make sure that the formatting is indeed
+           possible.
+        """
+        global torrc_option_warn_count
+
+        fn_out = self._getTorrcFname()
+        torrc_template = self._getTorrcTemplate()
+        output = torrc_template.format(self._env)
+        if checkOnly:
+            # XXXX Is it time-consuming to format? If so, cache here.
+            return
+        # now filter the options we're about to write, commenting out
+        # the options that the current tor binary doesn't support
+        tor = self._env['tor']
+        tor_version = get_tor_version(tor)
+        torrc_opts = get_torrc_options(tor)
+        # check if each option is supported before writing it
+        # Unsupported option values may need special handling.
+        with open(fn_out, 'w') as f:
+            # we need to do case-insensitive option comparison
+            lower_opts = [opt.lower() for opt in torrc_opts]
+            # keep ends when splitting lines, so we can write them out
+            # using writelines() without messing around with "\n"s
+            for line in output.splitlines(True):
+                # check if the first word on the line is a supported option,
+                # preserving empty lines and comment lines
+                sline = line.strip()
+                if (len(sline) == 0 or
+                        sline[0] == '#' or
+                        sline.split()[0].lower() in lower_opts):
+                    pass
+                else:
+                    warn_msg = (("The tor binary at {} does not support " +
+                                "the option in the torrc line:\n{}")
+                                .format(tor, line.strip()))
+                    if torrc_option_warn_count < TORRC_OPTION_WARN_LIMIT:
+                        print(warn_msg)
+                        torrc_option_warn_count += 1
+                    else:
+                        debug(warn_msg)
+                    # always dump the full output to the torrc file
+                    line = ("# {} version {} does not support: {}"
+                            .format(tor, tor_version, line))
+                f.writelines([line])
+
+    def _getTorrcTemplate(self):
+        """Return the template used to write the torrc for this node."""
+        template_path = self._env['torrc_template_path']
+        return chutney.Templating.Template("$${include:$torrc}",
+                                           includePath=template_path)
+
+    def _getFreeVars(self):
+        """Return a set of the free variables in the torrc template for this
+           node.
+        """
+        template = self._getTorrcTemplate()
+        return template.freevars(self._env)
+
+    def checkConfig(self, net):
+        """Try to format our torrc; raise an exception if we can't.
+        """
+        self._createTorrcFile(checkOnly=True)
+
+    def preConfig(self, net):
+        """Called on all nodes before any nodes configure: generates keys and
+           hidden service directories as needed.
+        """
+        self._makeDataDir()
+        if self._env['authority']:
+            self._genAuthorityKey()
+        if self._env['relay']:
+            self._genRouterKey()
+        if self._env['hs']:
+            self._makeHiddenServiceDir()
+
+    def config(self, net):
+        """Called to configure a node: creates a torrc file for it."""
+        self._createTorrcFile()
+        # self._createScripts()
+
+    def postConfig(self, net):
+        """Called on each nodes after all nodes configure."""
+        # self.net.addNode(self)
+        pass
+
+    def isSupported(self, net):
+        """Return true if this node appears to have everything it needs;
+           false otherwise."""
+
+        if not tor_exists(self._env['tor']):
+            print("No binary found for %r"%self._env['tor'])
+            return False
+
+        if self._env['authority']:
+            if not tor_has_module(self._env['tor'], "dirauth"):
+                print("No dirauth support in %r"%self._env['tor'])
+                return False
+            if not tor_gencert_exists(self._env['tor-gencert']):
+                print("No binary found for tor-gencert %r"%self._env['tor-gencrrt'])
+
+    def _makeDataDir(self):
+        """Create the data directory (with keys subdirectory) for this node.
+        """
+        datadir = self._env['dir']
+        make_datadir_subdirectory(datadir, "keys")
+
+    def _makeHiddenServiceDir(self):
+        """Create the hidden service subdirectory for this node.
+
+          The directory name is stored under the 'hs_directory' environment
+          key. It is combined with the 'dir' data directory key to yield the
+          path to the hidden service directory.
+        """
+        datadir = self._env['dir']
+        make_datadir_subdirectory(datadir, self._env['hs_directory'])
+
+    def _genAuthorityKey(self):
+        """Generate an authority identity and signing key for this authority,
+           if they do not already exist."""
+        datadir = self._env['dir']
+        tor_gencert = self._env['tor_gencert']
+        lifetime = self._env['auth_cert_lifetime']
+        idfile = os.path.join(datadir, 'keys', "authority_identity_key")
+        skfile = os.path.join(datadir, 'keys', "authority_signing_key")
+        certfile = os.path.join(datadir, 'keys', "authority_certificate")
+        addr = self.expand("${ip}:${dirport}")
+        passphrase = self._env['auth_passphrase']
+        if all(os.path.exists(f) for f in [idfile, skfile, certfile]):
+            return
+        cmdline = [
+            tor_gencert,
+            '--create-identity-key',
+            '--passphrase-fd', '0',
+            '-i', idfile,
+            '-s', skfile,
+            '-c', certfile,
+            '-m', str(lifetime),
+            '-a', addr,
+            ]
+        # nicknames are testNNNaa[OLD], but we want them to look tidy
+        print("Creating identity key for {:12} with {}"
+              .format(self._env['nick'], cmdline[0]))
+        debug("Identity key path '{}', command '{}'"
+              .format(idfile, " ".join(cmdline)))
+        run_tor_gencert(cmdline, passphrase)
+
+    def _genRouterKey(self):
+        """Generate an identity key for this router, unless we already have,
+           and set up the 'fingerprint' entry in the Environ.
+        """
+        datadir = self._env['dir']
+        tor = self._env['tor']
+        torrc = self._getTorrcFname()
+        cmdline = [
+            tor,
+            "--ignore-missing-torrc",
+            "-f", torrc,
+            "--list-fingerprint",
+            "--orport", "1",
+            "--datadirectory", datadir,
+            ]
+        stdouterr = run_tor(cmdline)
+        fingerprint = "".join((stdouterr.rstrip().split('\n')[-1]).split()[1:])
+        if not re.match(r'^[A-F0-9]{40}$', fingerprint):
+            print("Error when getting fingerprint using '%r'. It output '%r'."
+                  .format(" ".join(cmdline), stdouterr))
+            sys.exit(1)
+        self._env['fingerprint'] = fingerprint
+
+    def _getAltAuthLines(self, hasbridgeauth=False):
+        """Return a combination of AlternateDirAuthority,
+        and AlternateBridgeAuthority lines for
+        this Node, appropriately.  Non-authorities return ""."""
+        if not self._env['authority']:
+            return ""
+
+        datadir = self._env['dir']
+        certfile = os.path.join(datadir, 'keys', "authority_certificate")
+        v3id = None
+        with open(certfile, 'r') as f:
+            for line in f:
+                if line.startswith("fingerprint"):
+                    v3id = line.split()[1].strip()
+                    break
+
+        assert v3id is not None
+
+        if self._env['bridgeauthority']:
+            # Bridge authorities return AlternateBridgeAuthority with
+            # the 'bridge' flag set.
+            options = ("AlternateBridgeAuthority",)
+            self._env['dirserver_flags'] += " bridge"
+        else:
+            # Directory authorities return AlternateDirAuthority with
+            # the 'v3ident' flag set.
+            # XXXX This next line is needed for 'bridges' but breaks
+            # 'basic'
+            if hasbridgeauth:
+                options = ("AlternateDirAuthority",)
+            else:
+                options = ("DirAuthority",)
+            self._env['dirserver_flags'] += " v3ident=%s" % v3id
+
+        authlines = ""
+        for authopt in options:
+            authlines += "%s %s orport=%s" % (
+                authopt, self._env['nick'], self._env['orport'])
+            # It's ok to give an authority's IPv6 address to an IPv4-only
+            # client or relay: it will and must ignore it
+            # and yes, the orport is the same on IPv4 and IPv6
+            if self._env['ipv6_addr'] is not None:
+                authlines += " ipv6=%s:%s" % (self._env['ipv6_addr'],
+                                              self._env['orport'])
+            authlines += " %s %s:%s %s\n" % (
+                self._env['dirserver_flags'], self._env['ip'],
+                self._env['dirport'], self._env['fingerprint'])
+        return authlines
+
+    def _getBridgeLines(self):
+        """Return potential Bridge line for this Node. Non-bridge
+        relays return "".
+        """
+        if not self._env['bridge']:
+            return ""
+
+        if self._env['pt_bridge']:
+            port = self._env['ptport']
+            transport = self._env['pt_transport']
+            extra = self._env['pt_extra']
+        else:
+            # the orport is the same on IPv4 and IPv6
+            port = self._env['orport']
+            transport = ""
+            extra = ""
+
+        BRIDGE_LINE_TEMPLATE = "Bridge %s %s:%s %s %s\n"
+
+        bridgelines = BRIDGE_LINE_TEMPLATE % (transport,
+                                              self._env['ip'],
+                                              port,
+                                              self._env['fingerprint'],
+                                              extra)
+        if self._env['ipv6_addr'] is not None:
+            bridgelines += BRIDGE_LINE_TEMPLATE % (transport,
+                                                   self._env['ipv6_addr'],
+                                                   port,
+                                                   self._env['fingerprint'],
+                                                   extra)
+        return bridgelines
 
 
-class LocalNodeBuilder(NodeBuilder):
+def scp_file(abs_filepath, host):
+    if not os.path.isabs(abs_filepath) or abs_filepath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('SCP path must be absolute and must be in /tmp')
+    assert(':' not in host)
+    assert(':' not in abs_filepath)
+    remote_filepath = os.path.dirname(abs_filepath)
+    cmd = ['scp', abs_filepath, ':'.join([host, remote_filepath])]
+    print('Transferring file: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def scp_dir(abs_dirpath, host):
+    if not os.path.isabs(abs_dirpath) or abs_dirpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('SCP path must be absolute and must be in /tmp')
+    assert(':' not in host)
+    assert(':' not in abs_dirpath)
+    remote_dirpath = os.path.dirname(abs_dirpath)
+    cmd = ['scp', '-r', abs_dirpath, ':'.join([host, remote_dirpath])]
+    print('Transferring files: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def scp_dir_backwards(abs_dirpath, host):
+    if not os.path.isabs(abs_dirpath) or abs_dirpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('SCP path must be absolute and must be in /tmp')
+    assert(':' not in host)
+    assert(':' not in abs_dirpath)
+    remote_dirpath = os.path.dirname(abs_dirpath)
+    cmd = ['scp', '-r', ':'.join([host, abs_dirpath]), remote_dirpath]
+    print('Transferring files backwards: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def ssh_mkdir_p(abs_dirpath, remote_hostname):
+    if not os.path.isabs(abs_dirpath) or abs_dirpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    cmd = ['ssh', remote_hostname, 'mkdir', '-p', abs_dirpath]
+    print('Making directory: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def ssh_rm_if_exists(abs_dirpath, remote_hostname):
+    if not os.path.isabs(abs_dirpath) or abs_dirpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    cmd = ['ssh', remote_hostname, 'rm', '-f', abs_dirpath]
+    print('Removing: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def ssh_file_exists(abs_filepath, remote_hostname):
+    if not os.path.isabs(abs_filepath) or abs_filepath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    assert('"' not in abs_filepath)
+    assert('$' not in abs_filepath)
+    assert('\\' not in abs_filepath)
+    cmd = ['ssh', remote_hostname, '[ -f "{}" ] && exit 0 || exit 99'.format(abs_filepath)]
+    print('Checking file existence: {}'.format(cmd))
+    try:
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 99:
+            return False
+        raise
+    return True
+
+def ssh_symlink(abs_dirpath, abs_linkpath, remote_hostname):
+    if not os.path.isabs(abs_dirpath) or abs_dirpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    if not os.path.isabs(abs_linkpath) or abs_linkpath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Link must be absolute and must be in /tmp')
+    cmd = ['ssh', remote_hostname, 'ln', '-s', abs_dirpath, abs_linkpath]
+    print('Making link: {}'.format(cmd))
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+def ssh_read_file(abs_filepath, remote_hostname):
+    if not os.path.isabs(abs_filepath) or abs_filepath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    if not ssh_file_exists(abs_filepath, remote_hostname):
+        return None
+    cmd = ['ssh', remote_hostname, 'cat', abs_filepath]
+    print('Reading file: {}'.format(cmd))
+    try:
+        return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        # although something else may have gone wrong, we'll assume the file doesn't exist
+        print('File existed, but now returns an error; assuming it no longer exists')
+        return None
+
+def ssh_kill(pid, code, remote_hostname):
+    assert pid > 1
+    cmd = ['ssh', remote_hostname, 'kill', '-s', str(code), str(pid)]
+    print('Sending signal: {}'.format(cmd))
+    try:
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        if e.returncode != 1:
+            # the process might not exist
+            return False
+    return True
+
+def ssh_grep(pattern, abs_filepath, remote_hostname):
+    if not os.path.isabs(abs_filepath) or abs_filepath[0:5] != '/tmp/':
+        # this check for '/tmp' is in no way secure, but helps prevent me from shooting
+        # myself in the foot
+        raise Exception('Path must be absolute and must be in /tmp')
+    cmd = ['ssh', remote_hostname, 'egrep', '"{}"'.format(pattern), abs_filepath]
+    #print('Text search: {}'.format(cmd))
+    try:
+        return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('utf-8')
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            # egrep will exit with 1 if it doesn't find anything
+            return ''
+
+class RemoteNodeBuilder(NodeBuilder):
 
     # Environment members used:
     # torrc -- which torrc file to use
@@ -554,8 +987,8 @@ class LocalNodeBuilder(NodeBuilder):
         # now filter the options we're about to write, commenting out
         # the options that the current tor binary doesn't support
         tor = self._env['tor']
-        tor_version = get_tor_version(tor)
-        torrc_opts = get_torrc_options(tor)
+        tor_version = get_tor_version(tor, remote_hostname=self._env['remote_hostname'])
+        torrc_opts = get_torrc_options(tor, remote_hostname=self._env['remote_hostname'])
         # check if each option is supported before writing it
         # Unsupported option values may need special handling.
         with open(fn_out, 'w') as f:
@@ -623,7 +1056,8 @@ class LocalNodeBuilder(NodeBuilder):
     def postConfig(self, net):
         """Called on each nodes after all nodes configure."""
         # self.net.addNode(self)
-        pass
+        scp_dir(os.path.abspath(self._env['dir']), self._env['remote_hostname'])
+        shutil.rmtree(self._env['dir'])
 
     def isSupported(self, net):
         """Return true if this node appears to have everything it needs;
@@ -811,7 +1245,10 @@ class LocalNodeController(NodeController):
             return None
 
         with open(pidfile, 'r') as f:
-            return int(f.read())
+            try:
+                return int(f.read())
+            except ValueError:
+                return None
 
     def isRunning(self, pid=None):
         """Return true iff this node is running.  (If 'pid' is provided, we
@@ -904,6 +1341,8 @@ class LocalNodeController(NodeController):
             add_environ_vars = add_environ_vars.copy()
         #
         if self._env['google_cpu_profiler'] is True:
+            if add_environ_vars is None:
+                add_environ_vars = {}
             add_environ_vars['CPUPROFILE'] = os.path.join(self._env['dir'], 'cpu-prof.out')
         #
         cmdline.extend([
@@ -1025,6 +1464,303 @@ class LocalNodeController(NodeController):
         pct, _, _ = self.getLastBootstrapStatus()
         return pct == 100
 
+    def getRemoteFiles(self):
+        pass
+
+class RemoteNodeController(NodeController):
+
+    def __init__(self, env):
+        NodeController.__init__(self, env)
+        self._env = env
+
+    def getNick(self):
+        """Return the nickname for this node."""
+        return self._env['nick']
+
+    def getPid(self):
+        """Assuming that this node has its pidfile in ${dir}/pid, return
+           the pid of the running process, or None if there is no pid in the
+           file.
+        """
+        pidfile = os.path.join(self._env['dir'], 'pid')
+        if self._env['remote_hostname'] is None:
+            if not os.path.exists(pidfile):
+                return None
+            with open(pidfile, 'r') as f:
+                try:
+                    return int(f.read())
+                except ValueError:
+                    return None
+        else:
+            pid = ssh_read_file(pidfile, self._env['remote_hostname'])
+            if pid is None:
+                return None
+            try:
+                return int(pid)
+            except ValueError:
+                return None
+
+    def isRunning(self, pid=None):
+        """Return true iff this node is running.  (If 'pid' is provided, we
+           assume that the pid provided is the one of this node.  Otherwise
+           we call getPid().
+        """
+        if pid is None:
+            pid = self.getPid()
+        if pid is None:
+            return False
+
+        if self._env['remote_hostname'] is None:
+            try:
+                os.kill(pid, 0)  # "kill 0" == "are you there?"
+            except OSError as e:
+                if e.errno == errno.ESRCH:
+                    return False
+                raise
+            # okay, so the process exists.  Say "True" for now.
+            # XXXX check if this is really tor!
+            return True
+        else:
+            return ssh_kill(pid, 0, self._env['remote_hostname'])
+
+    def check(self, listRunning=True, listNonRunning=False):
+        """See if this node is running, stopped, or crashed.  If it's running
+           and listRunning is set, print a short statement.  If it's
+           stopped and listNonRunning is set, then print a short statement.
+           If it's crashed, print a statement.  Return True if the
+           node is running, false otherwise.
+        """
+        # XXX Split this into "check" and "print" parts.
+        pid = self.getPid()
+        nick = self._env['nick']
+        datadir = self._env['dir']
+        corefile = os.path.join(datadir, "core.%s" % pid)
+        tor_version = get_tor_version(self._env['tor'], remote_hostname=self._env['remote_hostname'])
+
+        hostname_help_str = ''
+        if self._env['remote_hostname'] is not None:
+            hostname_help_str = ' on \'{}\''.format(self._env['remote_hostname'])
+
+        if self._env['remote_hostname'] is None:
+            corefile_exists = os.path.exists(corefile)
+        else:
+            corefile_exists = ssh_file_exists(corefile, self._env['remote_hostname'])
+
+        if self.isRunning(pid):
+            if listRunning:
+                # PIDs are typically 65535 or less
+                print("{:12} is running with PID {:5}{}: {}"
+                      .format(nick, pid, hostname_help_str, tor_version))
+            return True
+        elif corefile_exists:
+            if listNonRunning:
+                print("{:12} seems to have crashed{}, and left core file {}: {}"
+                      .format(nick, hostname_help_str, corefile, tor_version))
+            return False
+        else:
+            if listNonRunning:
+                print("{:12} is stopped{}: {}"
+                      .format(nick, hostname_help_str, tor_version))
+            return False
+
+    def hup(self):
+        """Send a SIGHUP to this node, if it's running."""
+        pid = self.getPid()
+        nick = self._env['nick']
+        if self.isRunning(pid):
+            print("Sending sighup to {}".format(nick))
+            if self._env['remote_hostname'] is None:
+                os.kill(pid, signal.SIGHUP)
+            else:
+                ssh_kill(pid, int(signal.SIGHUP), self._env['remote_hostname'])
+            return True
+        else:
+            print("{:12} is not running".format(nick))
+            return False
+
+    def start(self):
+        """Try to start this node; return True if we succeeded or it was
+           already running, False if we failed."""
+
+        if self.isRunning():
+            print("{:12} is already running".format(self._env['nick']))
+            return True
+        tor_path = self._env['tor']
+        torrc = self._getTorrcFname()
+        #
+        add_environ_vars = self._env['add_environ_vars']
+        if add_environ_vars is not None:
+            add_environ_vars = add_environ_vars.copy()
+        #
+        if self._env['google_cpu_profiler'] is True:
+            if add_environ_vars is None:
+                add_environ_vars = {}
+            add_environ_vars['CPUPROFILE'] = os.path.join(self._env['dir'], 'cpu-prof.out')
+        #
+        cmdline = []
+        if self._env['remote_hostname'] is not None:
+            cmdline.extend(['ssh', self._env['remote_hostname']])
+            if add_environ_vars is not None:
+                for x in add_environ_vars:
+                    cmdline.extend(['{}={}'.format(x, add_environ_vars[x])])
+                #
+            #
+        #
+        if self._env['numa_settings'] is not None:
+            (numa_node, processors) = self._env['numa_settings']
+            cmdline.extend([
+                'numactl',
+                '--membind={}'.format(numa_node),
+                '--physcpubind={}'.format(','.join(map(str, processors))),
+                ])
+        #
+        if self._env['valgrind_settings'] is not None:
+            cmdline.append('valgrind')
+            cmdline.extend(self._env['valgrind_settings'])
+            cmdline.append('--log-file={}'.format(self._env['valgrind_log']))
+        #
+        cmdline.extend([
+            tor_path,
+            "-f", torrc,
+            ])
+        if self._env['remote_hostname'] is not None:
+            print('Starting tor with: {}'.format(cmdline))
+        p = launch_process(cmdline, add_environ_vars=add_environ_vars)
+        if self.waitOnLaunch():
+            # this requires that RunAsDaemon is set
+            (stdouterr, empty_stderr) = p.communicate()
+            debug(stdouterr)
+            assert empty_stderr is None
+        else:
+            # this does not require RunAsDaemon to be set, but is slower.
+            #
+            # poll() only catches failures before the call itself
+            # so let's sleep a little first
+            # this does, of course, slow down process launch
+            # which can require an adjustment to the voting interval
+            #
+            # avoid writing a newline or space when polling
+            # so output comes out neatly
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            time.sleep(self._env['poll_launch_time'])
+            p.poll()
+        if p.returncode is not None and p.returncode != 0:
+            if self._env['poll_launch_time'] is None:
+                print(("Couldn't launch {:12} command '{}': " +
+                       "exit {}, output '{}'")
+                      .format(self._env['nick'],
+                              " ".join(cmdline),
+                              p.returncode,
+                              stdouterr))
+            else:
+                print(("Couldn't poll {:12} command '{}' " +
+                       "after waiting {} seconds for launch: " +
+                       "exit {}").format(self._env['nick'],
+                                         " ".join(cmdline),
+                                         self._env['poll_launch_time'],
+                                         p.returncode))
+            return False
+        return True
+
+    def stop(self, sig=signal.SIGINT):
+        """Try to stop this node by sending it the signal 'sig'."""
+        pid = self.getPid()
+        if not self.isRunning(pid):
+            print("{:12} is not running".format(self._env['nick']))
+            return
+        if self._env['remote_hostname'] is None:
+            os.kill(pid, sig)
+        else:
+            ssh_kill(pid, int(sig), self._env['remote_hostname'])
+
+    def cleanup_lockfile(self):
+        lf = self._env['lockfile']
+        if not self.isRunning() and os.path.exists(lf):
+            debug("Removing stale lock file for {} ..."
+                  .format(self._env['nick']))
+            os.remove(lf)
+
+    def waitOnLaunch(self):
+        """Check whether we can wait() for the tor process to launch"""
+        # TODO: is this the best place for this code?
+        # RunAsDaemon default is 0
+        runAsDaemon = self._env['daemon']
+        '''
+        with open(self._getTorrcFname(), 'r') as f:
+            for line in f.readlines():
+                stline = line.strip()
+                # if the line isn't all whitespace or blank
+                if len(stline) > 0:
+                    splline = stline.split()
+                    # if the line has at least two tokens on it
+                    if (len(splline) > 0 and
+                            splline[0].lower() == "RunAsDaemon".lower() and
+                            splline[1] == "1"):
+                        # use the RunAsDaemon value from the torrc
+                        # TODO: multiple values?
+                        runAsDaemon = True
+        '''
+        if runAsDaemon:
+            # we must use wait() instead of poll()
+            self._env['poll_launch_time'] = None
+            return True
+        else:
+            # we must use poll() instead of wait()
+            if self._env['poll_launch_time'] is None:
+                self._env['poll_launch_time'] = \
+                    self._env['poll_launch_time_default']
+            return False
+
+    def getLogfile(self, info=False):
+        """Return the expected path to the logfile for this instance."""
+        datadir = self._env['dir']
+        if info:
+            logname = "info.log"
+        else:
+            logname = "notice.log"
+        return os.path.join(datadir, logname)
+
+    def getLastBootstrapStatus(self):
+        """Look through the logs and return the last bootstrap message
+           received as a 3-tuple of percentage complete, keyword
+           (optional), and message.
+        """
+        logfname = self.getLogfile()
+
+        def find_bootstrap_messages(lines):
+            percent, keyword, message = -100, "no_message", "No bootstrap messages yet."
+            for line in lines:
+                m = re.search(r'Bootstrapped (\d+)%(?: \(([^\)]*)\))?: (.*)',
+                              line)
+                if m:
+                    percent, keyword, message = m.groups()
+                    percent = int(percent)
+            return (percent, keyword, message)
+
+        if self._env['remote_hostname'] is None:
+            if not os.path.exists(logfname):
+                return (-200, "no_logfile", "There is no logfile yet.")
+            with open(logfname, 'r') as f:
+                return find_bootstrap_messages(f)
+        else:
+            messages = ssh_grep("Bootstrapped [0-9]+%", logfname, self._env['remote_hostname']).split('\n')
+            if messages is None:
+                return (-200, "no_logfile", "There is no logfile yet.")
+            return find_bootstrap_messages(messages)
+
+    def isBootstrapped(self):
+        """Return true iff the logfile says that this instance is
+           bootstrapped."""
+        pct, _, _ = self.getLastBootstrapStatus()
+        return pct == 100
+
+    def getRemoteFiles(self):
+        if self._env['remote_hostname'] is not None:
+            path = os.path.abspath(self._env['dir'])
+            ssh_rm_if_exists(os.path.join(path, 'control'), self._env['remote_hostname'])
+            scp_dir_backwards(path, self._env['remote_hostname'])
+
 # XXX: document these options
 DEFAULTS = {
     'authority': False,
@@ -1109,6 +1845,9 @@ DEFAULTS = {
     'add_environ_vars': None,
     'log_files': ['notice', 'info', 'debug'],
     'google_cpu_profiler': False,
+    'remote_hostname': None,
+    'num_additional_eventloops': None,
+    'log_throughput': False,
 }
 
 
@@ -1242,6 +1981,20 @@ class TorEnviron(chutney.Templating.Environ):
             num_cpus_line = '#' + num_cpus_line
         return num_cpus_line
 
+    def _get_num_additional_eventloops_line(self, my):
+        num = my['num_additional_eventloops']
+        line = 'NumAdditionalEventloops {}'.format(num)
+        if num is None:
+            line = '#' + line
+        return line
+
+    def _get_throughput_log_file_line(self, my):
+        line = 'ThroughputLogFile {}'.format(os.path.join(self['dir'],
+                                                          'relay_throughput.log'))
+        if not my['log_throughput']:
+            line = '#' + line
+        return line
+
     def _get_owning_controller_process(self, my):
         cpid = my['controlling_pid']
         ocp_line = ('__OwningControllerProcess %d' % (cpid))
@@ -1351,6 +2104,10 @@ class Network(object):
         print("NOTE: creating %r, linking to %r" % (newnodesdir, nodeslink))
         # this gets created with mode 0700, that's probably ok
         mkdir_p(newnodesdir)
+        remote_hostnames = list(set([x._env['remote_hostname'] for x in self._nodes if x._env['remote_hostname'] is not None]))
+        if len(remote_hostnames) != 0:
+            for x in remote_hostnames:
+                ssh_mkdir_p(newnodesdir, x)
         try:
             os.unlink(nodeslink)
         except OSError as e:
@@ -1359,7 +2116,13 @@ class Network(object):
                 pass
             else:
                 raise
+        if len(remote_hostnames) != 0:
+            for x in remote_hostnames:
+                ssh_rm_if_exists(nodeslink, x)
         os.symlink(newnodesdir, nodeslink)
+        if len(remote_hostnames) != 0:
+            for x in remote_hostnames:
+                ssh_symlink(newnodesdir, nodeslink, x)
 
     def _checkConfig(self):
         for n in self._nodes:
@@ -1502,6 +2265,11 @@ class Network(object):
                 sys.stdout.write("\n")
                 sys.stdout.flush()
 
+    def get_remote_files(self):
+        controllers = [n.getController() for n in self._nodes]
+        for c in controllers:
+            c.getRemoteFiles()
+
 
 def Require(feature):
     network = _THE_NETWORK

tools/test-network-impl.sh

@@ -86,5 +86,7 @@ else
     exit 0
 fi
 
+"$CHUTNEY" get_remote_files "$CHUTNEY_NETWORK"
+
 "$WARNINGS"
 exit 0

torrc_templates/common.i

@@ -17,6 +17,9 @@ V3AuthNIntervalsValid 2
 # This line will be commented out if 'num_cpus' is None
 ${num_cpus_line}
 
+# This line will be commented out if 'num_additional_eventloops_line' is None
+${num_additional_eventloops_line}
+
 # measureme
 MeasuremeLogFile ${measureme_log_dir}/measureme-${nick}
 
@@ -45,6 +48,8 @@ PidFile ${dir}/pid
 
 ${log_file_lines}
 
+${throughput_log_file_line}
+
 ProtocolWarnings 1
 SafeLogging 0
 LogTimeGranularity 1