Merge remote-tracking branch 'tor-github/pr/32'

lib/chutney/Host.py

@@ -0,0 +1,20 @@
+
+import socket
+import chutney.Util
+
+@chutney.Util.memoized
+def is_ipv6_supported():
+    """Return true iff ipv6 is supported on this host."""
+    try:
+        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s.bind(("::1", 0))
+        s.listen(128)
+        a = s.getsockname()
+        s2 = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        s2.settimeout(1)
+        s2.connect(a)
+        return True
+    except socket.error:
+        return False
+
+

lib/chutney/TorNet.py

@@ -23,8 +23,10 @@ import importlib
 
 from chutney.Debug import debug_flag, debug
 
+import chutney.Host
 import chutney.Templating
 import chutney.Traffic
+import chutney.Util
 
 _BASE_ENVIRON = None
 _TOR_VERSIONS = None
@@ -37,6 +39,9 @@ torrc_option_warn_count =  0
 # Get verbose tracebacks, so we can diagnose better.
 cgitb.enable(format="plain")
 
+class MissingBinaryException(Exception):
+    pass
+
 def getenv_int(envvar, default):
     """
        Return the value of the environment variable 'envar' as an integer,
@@ -131,11 +136,14 @@ def _warnMissingTor(tor_path, cmdline, tor_name="tor"):
            "containing {}.")
           .format(tor_name, tor_path, " ".join(cmdline), tor_name))
 
-def run_tor(cmdline):
+def run_tor(cmdline, exit_on_missing=True):
     """Run the tor command line cmdline, which must start with the path or
        name of a tor binary.
 
        Returns the combined stdout and stderr of the process.
+
+       If exit_on_missing is true, warn and exit if the tor binary is missing.
+       Otherwise, raise a MissingBinaryException.
     """
     if not debug_flag:
         cmdline.append("--quiet")
@@ -148,20 +156,26 @@ def run_tor(cmdline):
     except OSError as e:
         # only catch file not found error
         if e.errno == errno.ENOENT:
-            _warnMissingTor(cmdline[0], cmdline)
-            sys.exit(1)
+            if exit_on_missing:
+                _warnMissingTor(cmdline[0], cmdline)
+                sys.exit(1)
+            else:
+                raise MissingBinaryException()
         else:
             raise
     except subprocess.CalledProcessError as e:
         # only catch file not found error
         if e.returncode == 127:
-            _warnMissingTor(cmdline[0], cmdline)
-            sys.exit(1)
+            if exit_on_missing:
+                _warnMissingTor(cmdline[0], cmdline)
+                sys.exit(1)
+            else:
+                raise MissingBinaryException()
         else:
             raise
     return stdouterr
 
-def launch_process(cmdline, tor_name="tor", stdin=None):
+def launch_process(cmdline, tor_name="tor", stdin=None, exit_on_missing=True):
     """Launch the command line cmdline, which must start with the path or
        name of a binary. Use tor_name as the canonical name of the binary.
        Pass stdin to the Popen constructor.
@@ -182,8 +196,11 @@ def launch_process(cmdline, tor_name="tor", stdin=None):
     except OSError as e:
         # only catch file not found error
         if e.errno == errno.ENOENT:
-            _warnMissingTor(cmdline[0], cmdline, tor_name=tor_name)
-            sys.exit(1)
+            if exit_on_missing:
+                _warnMissingTor(cmdline[0], cmdline, tor_name=tor_name)
+                sys.exit(1)
+            else:
+                raise MissingBinaryException()
         else:
             raise
     return p
@@ -204,49 +221,94 @@ def run_tor_gencert(cmdline, passphrase):
     assert empty_stderr is None
     return stdouterr
 
+@chutney.Util.memoized
+def tor_exists(tor):
+    """Return true iff this tor binary exists."""
+    try:
+        run_tor([tor, "--quiet", "--version"], exit_on_missing=False)
+        return True
+    except MissingBinaryException:
+        return False
+
+@chutney.Util.memoized
+def tor_gencert_exists(gencert):
+    """Return true iff this tor-gencert binary exists."""
+    try:
+        p = launch_process([gencert, "--help"], exit_on_missing=False)
+        p.wait()
+        return True
+    except MissingBinaryException:
+        return False
+
+@chutney.Util.memoized
 def get_tor_version(tor):
     """Return the version of the tor binary.
        Versions are cached for each unique tor path.
     """
-    # find the version of the current tor binary, and cache it
-    if tor not in _TOR_VERSIONS:
-        cmdline = [
-            tor,
-            "--version",
-        ]
-        tor_version = run_tor(cmdline)
-        # clean it up a bit
-        tor_version = tor_version.strip()
-        tor_version = tor_version.replace("version ", "")
-        tor_version = tor_version.replace(").", ")")
-        # check we received a tor version, and nothing else
-        assert re.match(r'^[-+.() A-Za-z0-9]+$', tor_version)
-        # cache the version for this tor binary's path
-        _TOR_VERSIONS[tor] = tor_version
-    else:
-        tor_version = _TOR_VERSIONS[tor]
+    cmdline = [
+        tor,
+        "--version",
+    ]
+    tor_version = run_tor(cmdline)
+    # clean it up a bit
+    tor_version = tor_version.strip()
+    tor_version = tor_version.replace("version ", "")
+    tor_version = tor_version.replace(").", ")")
+    # check we received a tor version, and nothing else
+    assert re.match(r'^[-+.() A-Za-z0-9]+$', tor_version)
+
     return tor_version
 
+@chutney.Util.memoized
 def get_torrc_options(tor):
     """Return the torrc options supported by the tor binary.
        Options are cached for each unique tor path.
     """
-    # find the options the current tor binary supports, and cache them
-    if tor not in _TORRC_OPTIONS:
-        cmdline = [
-            tor,
-            "--list-torrc-options",
-        ]
-        opts = run_tor(cmdline)
-        # check we received a list of options, and nothing else
-        assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
-        torrc_opts = opts.split()
-        # cache the options for this tor binary's path
-        _TORRC_OPTIONS[tor] = torrc_opts
-    else:
-        torrc_opts = _TORRC_OPTIONS[tor]
+    cmdline = [
+        tor,
+        "--list-torrc-options",
+    ]
+    opts = run_tor(cmdline)
+    # check we received a list of options, and nothing else
+    assert re.match(r'(^\w+$)+', opts, flags=re.MULTILINE)
+    torrc_opts = opts.split()
+
     return torrc_opts
 
+@chutney.Util.memoized
+def get_tor_modules(tor):
+    """Check the list of compile-time modules advertised by the given
+       'tor' binary, and return a map from module name to a boolean
+       describing whether it is supported.
+
+       Unlisted modules are ones that Tor did not treat as compile-time
+       optional modules.
+    """
+    cmdline = [
+        tor,
+        "--list-modules",
+        "--quiet"
+        ]
+    try:
+        mods = run_tor(cmdline)
+    except subprocess.CalledProcessError as e:
+        # Tor doesn't support --list-modules; act as if it said nothing.
+        mods = ""
+
+    supported = {}
+    for line in mods.split("\n"):
+        m = re.match(r'^(\S+): (yes|no)', line)
+        if not m:
+            continue
+        supported[m.group(1)] = (m.group(2) == "yes")
+
+    return supported
+
+def tor_has_module(tor, modname, default=True):
+    """Return true iff the given tor binary supports a given compile-time
+       module.  If the module is not listed, return 'default'.
+    """
+    return get_tor_modules(tor).get(modname, default)
 
 class Node(object):
 
@@ -366,6 +428,11 @@ class NodeBuilder(_NodeCommon):
         """Called on each nodes after all nodes configure."""
 
 
+    def isSupported(self, net):
+        """Return true if this node appears to have everything it needs;
+           false otherwise."""
+
+
 class NodeController(_NodeCommon):
 
     """Abstract base class.  A NodeController is responsible for running a
@@ -509,6 +576,21 @@ class LocalNodeBuilder(NodeBuilder):
         # self.net.addNode(self)
         pass
 
+    def isSupported(self, net):
+        """Return true if this node appears to have everything it needs;
+           false otherwise."""
+
+        if not tor_exists(self._env['tor']):
+            print("No binary found for %r"%self._env['tor'])
+            return False
+
+        if self._env['authority']:
+            if not tor_has_module(self._env['tor'], "dirauth"):
+                print("No dirauth support in %r"%self._env['tor'])
+                return False
+            if not tor_gencert_exists(self._env['tor-gencert']):
+                print("No binary found for tor-gencert %r"%self._env['tor-gencrrt'])
+
     def _makeDataDir(self):
         """Create the data directory (with keys subdirectory) for this node.
         """
@@ -1052,14 +1134,17 @@ class TorEnviron(chutney.Templating.Environ):
             dns_conf = TorEnviron.OFFLINE_DNS_RESOLV_CONF
         return "ServerDNSResolvConfFile %s" % (dns_conf)
 
+KNOWN_REQUIREMENTS = {
+    "IPV6": chutney.Host.is_ipv6_supported
+}
 
 class Network(object):
-
     """A network of Tor nodes, plus functions to manipulate them
     """
 
     def __init__(self, defaultEnviron):
         self._nodes = []
+        self._requirements = []
         self._dfltEnv = defaultEnviron
         self._nextnodenum = 0
 
@@ -1068,6 +1153,12 @@ class Network(object):
         self._nextnodenum += 1
         self._nodes.append(n)
 
+    def _addRequirement(self, requirement):
+        requirement = requirement.upper()
+        if requirement not in KNOWN_REQUIREMENTS:
+            raise RuntimemeError(("Unrecognized requirement %r"%requirement))
+        self._requirements.append(requirement)
+
     def move_aside_nodes_dir(self):
         """Move aside the nodes directory, if it exists and is not a link.
         Used for backwards-compatibility only: nodes is created as a link to
@@ -1128,6 +1219,22 @@ class Network(object):
         for n in self._nodes:
             n.getBuilder().checkConfig(self)
 
+    def supported(self):
+        """Check whether this network is supported by the set of binaries
+           and host information we have.
+        """
+        missing_any = False
+        for r in self._requirements:
+            if not KNOWN_REQUIREMENTS[r]():
+                print(("Can't run this network: %s is missing."))
+                missing_any = True
+        for n in self._nodes:
+            if not n.getBuilder().isSupported(self):
+                missing_any = False
+
+        if missing_any:
+            sys.exit(1)
+
     def configure(self):
         self.create_new_nodes_dir()
         network = self
@@ -1244,6 +1351,10 @@ class Network(object):
                 sys.stdout.flush()
 
 
+def Require(feature):
+    network = _THE_NETWORK
+    network._addRequirement(feature)
+
 def ConfigureNodes(nodelist):
     network = _THE_NETWORK
 
@@ -1252,7 +1363,6 @@ def ConfigureNodes(nodelist):
         if n._env['bridgeauthority']:
             network._dfltEnv['hasbridgeauth'] = True
 
-
 def getTests():
     tests = []
     chutney_path = get_absolute_chutney_path()
@@ -1283,6 +1393,7 @@ def exit_on_error(err_msg):
 def runConfigFile(verb, data):
     _GLOBALS = dict(_BASE_ENVIRON=_BASE_ENVIRON,
                     Node=Node,
+                    Require=Require,
                     ConfigureNodes=ConfigureNodes,
                     _THE_NETWORK=_THE_NETWORK,
                     torrc_option_warn_count=0,
@@ -1319,17 +1430,8 @@ def parseArgs():
 
 def main():
     global _BASE_ENVIRON
-    global _TOR_VERSIONS
-    global _TORRC_OPTIONS
     global _THE_NETWORK
     _BASE_ENVIRON = TorEnviron(chutney.Templating.Environ(**DEFAULTS))
-    # _TOR_VERSIONS gets initialised on demand as a map of
-    # "/path/to/tor" => "Tor version ..."
-    _TOR_VERSIONS = dict()
-    # _TORRC_OPTIONS gets initialised on demand as a map of
-    # "/path/to/tor" => ["SupportedOption1", "SupportedOption2", ...]
-    # Or it can be pre-populated as a static whitelist of options
-    _TORRC_OPTIONS = dict()
     _THE_NETWORK = Network(_BASE_ENVIRON)
 
     args = parseArgs()

lib/chutney/Util.py

@@ -0,0 +1,13 @@
+
+
+def memoized(fn):
+    """Decorator: memoize a function."""
+    memory = {}
+    def memoized_fn(*args, **kwargs):
+        key = (args, tuple(sorted(kwargs.items())))
+        try:
+            result = memory[key]
+        except KeyError:
+            result = memory[key] = fn(*args, **kwargs)
+        return result
+    return memoized_fn

networks/bridges+ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl")

networks/bridges+ipv6+hs

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl")

networks/bridges+ipv6+hs-v2

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl")

networks/bridges+ipv6+hs-v23

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl")

networks/bridges+ipv6-min

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay.tmpl")

networks/client-ipv6-only

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/client-ipv6-only-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-client-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-client-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-v23-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-v23-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-v3-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/hs-v3-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/ipv6-exit-min

@@ -1,3 +1,4 @@
+Require("IPV6")
 # By default, Authorities are not configured as exits
 Authority = Node(tag="a", authority=1, relay=1, torrc="authority.tmpl")
 IPv6ExitRelay = Node(tag="r", relay=1, exit=1, torrc="relay-exit-v6-only.tmpl")

networks/single-onion-client-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-client-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-v23-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-v23-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-v3-ipv6

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

networks/single-onion-v3-ipv6-md

@@ -1,3 +1,4 @@
+Require("IPV6")
 import os
 # By default, Authorities are not configured as exits
 Authority6 = Node(tag="a", authority=1, relay=1,

tools/bootstrap-network.sh

@@ -53,6 +53,11 @@ export CHUTNEY_NETWORK="$CHUTNEY_PATH/networks/$NETWORK_FLAVOUR"
 
 "$CHUTNEY" stop "$CHUTNEY_NETWORK"
 
+if ! "$CHUTNEY" supported "$CHUTNEY_NETWORK"; then
+    echo "%myname: network not supported."
+    exit 77
+fi
+
 echo "$myname: bootstrapping network: $NETWORK_FLAVOUR"
 "$CHUTNEY" configure "$CHUTNEY_NETWORK"
 

tools/test-network-impl.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 
 if ! "$CHUTNEY_PATH/tools/bootstrap-network.sh" "$NETWORK_FLAVOUR"; then
+    if test "$?" = 77; then
+	$ECHO "SKIP: $NETWORK_FLAVOR not supported."
+	exit 77
+    fi
     CHUTNEY_WARNINGS_IGNORE_EXPECTED=false CHUTNEY_WARNINGS_SUMMARY=false \
         "$WARNING_COMMAND"
     "$WARNINGS"

tools/test-network.sh

@@ -351,6 +351,9 @@ while [ "$n_attempts" -lt "$max_attempts" ]; do
 	$ECHO "==== Chutney succeeded after $n_attempts attempt(s)."
 	exit 0
     fi
+    if test "$?" = 77; then
+	exit 77
+    fi
 done
 
 $ECHO "Chutney failed $n_attempts times; we may have a problem here."