Move network tests to separate Python files

Based on a patch from chobe.

Closes ticket 9087.

README

@@ -69,3 +69,8 @@ The working files:
 
   You can override the directory "./net" with the CHUTNEY_DATA_DIR
   environment variable.
+
+Test scripts:
+  The test scripts are stored in the "scripts/chutney_tests" directory. These
+  Python files must define a "run_test(network)" function. Files starting with
+  an underscore ("_") are ignored.

chutney

@@ -3,7 +3,7 @@
 set -o errexit
 set -o nounset
 
-export PYTHONPATH="$(dirname "${0}")/lib:${PYTHONPATH-}"
+export PYTHONPATH="$(dirname "${0}")/lib:$(dirname "${0}")/scripts:${PYTHONPATH-}"
 
 binaries="python2 python"
 

lib/chutney/TorNet.py

@@ -19,6 +19,7 @@ import re
 import errno
 import time
 import shutil
+import importlib
 
 import chutney.Templating
 import chutney.Traffic
@@ -903,199 +904,6 @@ class Network(object):
             for c in controllers:
                 c.check(listNonRunning=False)
 
-    def verify(self):
-        print("Verifying data transmission:")
-        status = self._verify_traffic()
-        print("Transmission: %s" % ("Success" if status else "Failure"))
-        if not status:
-            # TODO: allow the debug flag to be passed as an argument to
-            # src/test/test-network.sh and chutney
-            print("Set 'debug_flag = True' in Traffic.py to diagnose.")
-        return status
-
-    def _verify_traffic(self):
-        """Verify (parts of) the network by sending traffic through it
-        and verify what is received."""
-        LISTEN_PORT = 4747  # FIXME: Do better! Note the default exit policy.
-        # HSs must have a HiddenServiceDir with
-        # "HiddenServicePort <HS_PORT> <CHUTNEY_LISTEN_ADDRESS>:<LISTEN_PORT>"
-        HS_PORT = 5858
-        # The amount of data to send between each source-sink pair,
-        # each time the source connects.
-        # We create a source-sink pair for each (bridge) client to an exit,
-        # and a source-sink pair for a (bridge) client to each hidden service
-        DATALEN = self._dfltEnv['data_bytes']
-        # Print a dot each time a sink verifies this much data
-        DOTDATALEN = 5 * 1024 * 1024  # Octets.
-        TIMEOUT = 3                   # Seconds.
-        # Calculate the amount of random data we should use
-        randomlen = self._calculate_randomlen(DATALEN)
-        reps = self._calculate_reps(DATALEN, randomlen)
-        # sanity check
-        if reps == 0:
-            DATALEN = 0
-        # Get the random data
-        if randomlen > 0:
-            # print a dot after every DOTDATALEN data is verified, rounding up
-            dot_reps = self._calculate_reps(DOTDATALEN, randomlen)
-            # make sure we get at least one dot per transmission
-            dot_reps = min(reps, dot_reps)
-            with open('/dev/urandom', 'r') as randfp:
-                tmpdata = randfp.read(randomlen)
-        else:
-            dot_reps = 0
-            tmpdata = {}
-        # now make the connections
-        bind_to = (DEFAULTS['ip'], LISTEN_PORT)
-        tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT, reps,
-                                           dot_reps)
-        client_list = filter(lambda n:
-                             n._env['tag'] == 'c' or n._env['tag'] == 'bc',
-                             self._nodes)
-        exit_list = filter(lambda n:
-                           ('exit' in n._env.keys()) and n._env['exit'] == 1,
-                           self._nodes)
-        hs_list = filter(lambda n:
-                         n._env['tag'] == 'h',
-                         self._nodes)
-        if len(client_list) == 0:
-            print("  Unable to verify network: no client nodes available")
-            return False
-        if len(exit_list) == 0 and len(hs_list) == 0:
-            print("  Unable to verify network: no exit/hs nodes available")
-            print("  Exit nodes must be declared 'relay=1, exit=1'")
-            print("  HS nodes must be declared 'tag=\"hs\"'")
-            return False
-        print("Connecting:")
-        # the number of tor nodes in paths which will send DATALEN data
-        # if a node is used in two paths, we count it twice
-        # this is a lower bound, as cannabilised circuits are one node longer
-        total_path_node_count = 0
-        total_path_node_count += self._configure_exits(tt, bind_to, tmpdata,
-                                                       reps, client_list,
-                                                       exit_list, LISTEN_PORT)
-        total_path_node_count += self._configure_hs(tt, tmpdata, reps,
-                                                    client_list, hs_list,
-                                                    HS_PORT, LISTEN_PORT)
-        print("Transmitting Data:")
-        start_time = time.clock()
-        status = tt.run()
-        end_time = time.clock()
-        # if we fail, don't report the bandwidth
-        if not status:
-            return status
-        # otherwise, report bandwidth used, if sufficient data was transmitted
-        self._report_bandwidth(DATALEN, total_path_node_count,
-                               start_time, end_time)
-        return status
-
-    # In order to performance test a tor network, we need to transmit
-    # several hundred megabytes of data or more. Passing around this
-    # much data in Python has its own performance impacts, so we provide
-    # a smaller amount of random data instead, and repeat it to DATALEN
-    def _calculate_randomlen(self, datalen):
-        MAX_RANDOMLEN = 128 * 1024   # Octets.
-        if datalen > MAX_RANDOMLEN:
-            return MAX_RANDOMLEN
-        else:
-            return datalen
-
-    def _calculate_reps(self, datalen, replen):
-        # sanity checks
-        if datalen == 0 or replen == 0:
-            return 0
-        # effectively rounds datalen up to the nearest replen
-        if replen < datalen:
-            return (datalen + replen - 1) / replen
-        else:
-            return 1
-
-    # if there are any exits, each client / bridge client transmits
-    # via 4 nodes (including the client) to an arbitrary exit
-    # Each client binds directly to <CHUTNEY_LISTEN_ADDRESS>:LISTEN_PORT
-    # via an Exit relay
-    def _configure_exits(self, tt, bind_to, tmpdata, reps, client_list,
-                         exit_list, LISTEN_PORT):
-        CLIENT_EXIT_PATH_NODES = 4
-        connection_count = self._dfltEnv['connection_count']
-        exit_path_node_count = 0
-        if len(exit_list) > 0:
-            exit_path_node_count += (len(client_list) *
-                                     CLIENT_EXIT_PATH_NODES *
-                                     connection_count)
-            for op in client_list:
-                print("  Exit to %s:%d via client %s:%s"
-                      % (DEFAULTS['ip'], LISTEN_PORT,
-                         'localhost', op._env['socksport']))
-                for i in range(connection_count):
-                    proxy = ('localhost', int(op._env['socksport']))
-                    tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, proxy,
-                                                  reps))
-        return exit_path_node_count
-
-    # The HS redirects .onion connections made to hs_hostname:HS_PORT
-    # to the Traffic Tester's CHUTNEY_LISTEN_ADDRESS:LISTEN_PORT
-    # an arbitrary client / bridge client transmits via 8 nodes
-    # (including the client and hs) to each hidden service
-    # Instead of binding directly to LISTEN_PORT via an Exit relay,
-    # we bind to hs_hostname:HS_PORT via a hidden service connection
-    def _configure_hs(self, tt, tmpdata, reps, client_list, hs_list, HS_PORT,
-                      LISTEN_PORT):
-        CLIENT_HS_PATH_NODES = 8
-        connection_count = self._dfltEnv['connection_count']
-        hs_path_node_count = (len(hs_list) * CLIENT_HS_PATH_NODES *
-                              connection_count)
-        # Each client in hs_client_list connects to each hs
-        if self._dfltEnv['hs_multi_client']:
-            hs_client_list = client_list
-            hs_path_node_count *= len(client_list)
-        else:
-            # only use the first client in the list
-            hs_client_list = client_list[:1]
-        # Setup the connections from each client in hs_client_list to each hs
-        for hs in hs_list:
-            hs_bind_to = (hs._env['hs_hostname'], HS_PORT)
-            for client in hs_client_list:
-                print("  HS to %s:%d (%s:%d) via client %s:%s"
-                      % (hs._env['hs_hostname'], HS_PORT,
-                         DEFAULTS['ip'], LISTEN_PORT,
-                         'localhost', client._env['socksport']))
-                for i in range(connection_count):
-                    proxy = ('localhost', int(client._env['socksport']))
-                    tt.add(chutney.Traffic.Source(tt, hs_bind_to, tmpdata,
-                                                  proxy, reps))
-        return hs_path_node_count
-
-    # calculate the single stream bandwidth and overall tor bandwidth
-    # the single stream bandwidth is the bandwidth of the
-    # slowest stream of all the simultaneously transmitted streams
-    # the overall bandwidth estimates the simultaneous bandwidth between
-    # all tor nodes over all simultaneous streams, assuming:
-    # * minimum path lengths (no cannibalized circuits)
-    # * unlimited network bandwidth (that is, localhost)
-    # * tor performance is CPU-limited
-    # This be used to estimate the bandwidth capacity of a CPU-bound
-    # tor relay running on this machine
-    def _report_bandwidth(self, data_length, total_path_node_count,
-                          start_time, end_time):
-        # otherwise, if we sent at least 5 MB cumulative total, and
-        # it took us at least a second to send, report bandwidth
-        MIN_BWDATA = 5 * 1024 * 1024  # Octets.
-        MIN_ELAPSED_TIME = 1.0        # Seconds.
-        cumulative_data_sent = total_path_node_count * data_length
-        elapsed_time = end_time - start_time
-        if (cumulative_data_sent >= MIN_BWDATA and
-                elapsed_time >= MIN_ELAPSED_TIME):
-            # Report megabytes per second
-            BWDIVISOR = 1024*1024
-            single_stream_bandwidth = (data_length / elapsed_time / BWDIVISOR)
-            overall_bandwidth = (cumulative_data_sent / elapsed_time /
-                                 BWDIVISOR)
-            print("Single Stream Bandwidth: %.2f MBytes/s"
-                  % single_stream_bandwidth)
-            print("Overall tor Bandwidth: %.2f MBytes/s"
-                  % overall_bandwidth)
-
 
 def ConfigureNodes(nodelist):
     network = _THE_NETWORK
@@ -1106,11 +914,22 @@ def ConfigureNodes(nodelist):
             network._dfltEnv['hasbridgeauth'] = True
 
 
+def getTests():
+    tests = []
+    for x in os.listdir("scripts/chutney_tests/"):
+        if not x.startswith("_") and os.path.splitext(x)[1] == ".py":
+            tests.append(os.path.splitext(x)[0])
+    return tests
+
+
 def usage(network):
-    return "\n".join(["Usage: chutney {command} {networkfile}",
+    return "\n".join(["Usage: chutney {command/test} {networkfile}",
                       "Known commands are: %s" % (
                           " ".join(x for x in dir(network)
-                                   if not x.startswith("_")))])
+                                   if not x.startswith("_"))),
+                      "Known tests are: %s" % (
+                          " ".join(getTests()))
+                      ])
 
 
 def exit_on_error(err_msg):
@@ -1128,6 +947,16 @@ def runConfigFile(verb, data):
     exec(data, _GLOBALS)
     network = _GLOBALS['_THE_NETWORK']
 
+    # let's check if the verb is a valid test and run it
+    if verb in getTests():
+        test_module = importlib.import_module("chutney_tests.{}".format(verb))
+        try:
+            return test_module.run_test(network)
+        except AttributeError:
+            print("Test {!r} has no 'run_test(network)' function".format(verb))
+            return False
+
+    # tell the user we don't know what their verb meant
     if not hasattr(network, verb):
         print(usage(network))
         print("Error: I don't know how to %s." % verb)

scripts/chutney_tests/verify.py

@@ -0,0 +1,203 @@
+import time
+import chutney
+
+
+def run_test(network):
+    print("Verifying data transmission:")
+    status = _verify_traffic(network)
+    print("Transmission: %s" % ("Success" if status else "Failure"))
+    if not status:
+        # TODO: allow the debug flag to be passed as an argument to
+        # src/test/test-network.sh and chutney
+        print("Set 'debug_flag = True' in Traffic.py to diagnose.")
+    return status
+
+
+def _verify_traffic(network):
+    """Verify (parts of) the network by sending traffic through it
+    and verify what is received."""
+    LISTEN_ADDR = network._dfltEnv['ip']
+    LISTEN_PORT = 4747  # FIXME: Do better! Note the default exit policy.
+    # HSs must have a HiddenServiceDir with
+    # "HiddenServicePort <HS_PORT> <CHUTNEY_LISTEN_ADDRESS>:<LISTEN_PORT>"
+    HS_PORT = 5858
+    # The amount of data to send between each source-sink pair,
+    # each time the source connects.
+    # We create a source-sink pair for each (bridge) client to an exit,
+    # and a source-sink pair for a (bridge) client to each hidden service
+    DATALEN = network._dfltEnv['data_bytes']
+    # Print a dot each time a sink verifies this much data
+    DOTDATALEN = 5 * 1024 * 1024  # Octets.
+    TIMEOUT = 3                   # Seconds.
+    # Calculate the amount of random data we should use
+    randomlen = _calculate_randomlen(DATALEN)
+    reps = _calculate_reps(DATALEN, randomlen)
+    connection_count = network._dfltEnv['connection_count']
+    # sanity check
+    if reps == 0:
+        DATALEN = 0
+    # Get the random data
+    if randomlen > 0:
+        # print a dot after every DOTDATALEN data is verified, rounding up
+        dot_reps = _calculate_reps(DOTDATALEN, randomlen)
+        # make sure we get at least one dot per transmission
+        dot_reps = min(reps, dot_reps)
+        with open('/dev/urandom', 'r') as randfp:
+            tmpdata = randfp.read(randomlen)
+    else:
+        dot_reps = 0
+        tmpdata = {}
+    # now make the connections
+    bind_to = (LISTEN_ADDR, LISTEN_PORT)
+    tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT, reps,
+                                       dot_reps)
+    client_list = filter(lambda n:
+                         n._env['tag'] == 'c' or n._env['tag'] == 'bc',
+                         network._nodes)
+    exit_list = filter(lambda n:
+                       ('exit' in n._env.keys()) and n._env['exit'] == 1,
+                       network._nodes)
+    hs_list = filter(lambda n:
+                     n._env['tag'] == 'h',
+                     network._nodes)
+    if len(client_list) == 0:
+        print("  Unable to verify network: no client nodes available")
+        return False
+    if len(exit_list) == 0 and len(hs_list) == 0:
+        print("  Unable to verify network: no exit/hs nodes available")
+        print("  Exit nodes must be declared 'relay=1, exit=1'")
+        print("  HS nodes must be declared 'tag=\"hs\"'")
+        return False
+    print("Connecting:")
+    # the number of tor nodes in paths which will send DATALEN data
+    # if a node is used in two paths, we count it twice
+    # this is a lower bound, as cannabilised circuits are one node longer
+    total_path_node_count = 0
+    total_path_node_count += _configure_exits(tt, bind_to, tmpdata, reps,
+                                              client_list, exit_list,
+                                              LISTEN_ADDR, LISTEN_PORT,
+                                              connection_count)
+    total_path_node_count += _configure_hs(tt, tmpdata, reps, client_list,
+                                           hs_list, HS_PORT, LISTEN_ADDR,
+                                           LISTEN_PORT, connection_count,
+                                           network._dfltEnv['hs_multi_client'])
+    print("Transmitting Data:")
+    start_time = time.clock()
+    status = tt.run()
+    end_time = time.clock()
+    # if we fail, don't report the bandwidth
+    if not status:
+        return status
+    # otherwise, report bandwidth used, if sufficient data was transmitted
+    _report_bandwidth(DATALEN, total_path_node_count, start_time, end_time)
+    return status
+
+
+# In order to performance test a tor network, we need to transmit
+# several hundred megabytes of data or more. Passing around this
+# much data in Python has its own performance impacts, so we provide
+# a smaller amount of random data instead, and repeat it to DATALEN
+def _calculate_randomlen(datalen):
+    MAX_RANDOMLEN = 128 * 1024   # Octets.
+    if datalen > MAX_RANDOMLEN:
+        return MAX_RANDOMLEN
+    else:
+        return datalen
+
+
+def _calculate_reps(datalen, replen):
+    # sanity checks
+    if datalen == 0 or replen == 0:
+        return 0
+    # effectively rounds datalen up to the nearest replen
+    if replen < datalen:
+        return (datalen + replen - 1) / replen
+    else:
+        return 1
+
+
+# if there are any exits, each client / bridge client transmits
+# via 4 nodes (including the client) to an arbitrary exit
+# Each client binds directly to <CHUTNEY_LISTEN_ADDRESS>:LISTEN_PORT
+# via an Exit relay
+def _configure_exits(tt, bind_to, tmpdata, reps, client_list, exit_list,
+                     LISTEN_ADDR, LISTEN_PORT, connection_count):
+    CLIENT_EXIT_PATH_NODES = 4
+    exit_path_node_count = 0
+    if len(exit_list) > 0:
+        exit_path_node_count += (len(client_list) *
+                                 CLIENT_EXIT_PATH_NODES *
+                                 connection_count)
+        for op in client_list:
+            print("  Exit to %s:%d via client %s:%s"
+                  % (LISTEN_ADDR, LISTEN_PORT,
+                     'localhost', op._env['socksport']))
+            for _ in range(connection_count):
+                proxy = ('localhost', int(op._env['socksport']))
+                tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata, proxy,
+                                              reps))
+    return exit_path_node_count
+
+
+# The HS redirects .onion connections made to hs_hostname:HS_PORT
+# to the Traffic Tester's CHUTNEY_LISTEN_ADDRESS:LISTEN_PORT
+# an arbitrary client / bridge client transmits via 8 nodes
+# (including the client and hs) to each hidden service
+# Instead of binding directly to LISTEN_PORT via an Exit relay,
+# we bind to hs_hostname:HS_PORT via a hidden service connection
+def _configure_hs(tt, tmpdata, reps, client_list, hs_list, HS_PORT,
+                  LISTEN_ADDR, LISTEN_PORT, connection_count, hs_multi_client):
+    CLIENT_HS_PATH_NODES = 8
+    hs_path_node_count = (len(hs_list) * CLIENT_HS_PATH_NODES *
+                          connection_count)
+    # Each client in hs_client_list connects to each hs
+    if hs_multi_client:
+        hs_client_list = client_list
+        hs_path_node_count *= len(client_list)
+    else:
+        # only use the first client in the list
+        hs_client_list = client_list[:1]
+    # Setup the connections from each client in hs_client_list to each hs
+    for hs in hs_list:
+        hs_bind_to = (hs._env['hs_hostname'], HS_PORT)
+        for client in hs_client_list:
+            print("  HS to %s:%d (%s:%d) via client %s:%s"
+                  % (hs._env['hs_hostname'], HS_PORT,
+                     LISTEN_ADDR, LISTEN_PORT,
+                     'localhost', client._env['socksport']))
+            for _ in range(connection_count):
+                proxy = ('localhost', int(client._env['socksport']))
+                tt.add(chutney.Traffic.Source(tt, hs_bind_to, tmpdata,
+                                              proxy, reps))
+    return hs_path_node_count
+
+
+# calculate the single stream bandwidth and overall tor bandwidth
+# the single stream bandwidth is the bandwidth of the
+# slowest stream of all the simultaneously transmitted streams
+# the overall bandwidth estimates the simultaneous bandwidth between
+# all tor nodes over all simultaneous streams, assuming:
+# * minimum path lengths (no cannibalized circuits)
+# * unlimited network bandwidth (that is, localhost)
+# * tor performance is CPU-limited
+# This be used to estimate the bandwidth capacity of a CPU-bound
+# tor relay running on this machine
+def _report_bandwidth(data_length, total_path_node_count, start_time,
+                      end_time):
+    # otherwise, if we sent at least 5 MB cumulative total, and
+    # it took us at least a second to send, report bandwidth
+    MIN_BWDATA = 5 * 1024 * 1024  # Octets.
+    MIN_ELAPSED_TIME = 1.0        # Seconds.
+    cumulative_data_sent = total_path_node_count * data_length
+    elapsed_time = end_time - start_time
+    if (cumulative_data_sent >= MIN_BWDATA and
+            elapsed_time >= MIN_ELAPSED_TIME):
+        # Report megabytes per second
+        BWDIVISOR = 1024*1024
+        single_stream_bandwidth = (data_length / elapsed_time / BWDIVISOR)
+        overall_bandwidth = (cumulative_data_sent / elapsed_time /
+                             BWDIVISOR)
+        print("Single Stream Bandwidth: %.2f MBytes/s"
+              % single_stream_bandwidth)
+        print("Overall tor Bandwidth: %.2f MBytes/s"
+              % overall_bandwidth)