Make chutney verify retry until success

Return immediately on success.
Only return failure after 60 seconds with no successes.

This makes networks that bootstrap quickly verify much faster,
and networks that bootstrap slowly are much more reliable.

README

@@ -64,6 +64,16 @@ HS Connection Tests:
   # Default behavior is one client connects to each HS
   ./chutney stop networks/hs-025
 
+Waiting for the network to bootstrap:
+
+  Commands like "chutney verify" start immediately, and keep trying for
+  CHUTNEY_BOOTSTRAP_TIME seconds. If they haven't been successful after that
+  time, they fail.
+
+  The tools/test-network.sh script waits 15 seconds before calling chutney
+  verify, because that's the minimum amount of time it takes to bootstrap a
+  consensus containing relays.
+
 Changing the network address:
 
    Chutney defaults to binding to localhost. To change the bind address,

lib/chutney/TorNet.py

@@ -700,6 +700,13 @@ DEFAULTS = {
     # to connect to each HS?
     # (Clients choose an exit at random, so this doesn't apply to exits.)
     'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
+    # How long should verify (and similar commands) wait for a successful
+    # outcome? (seconds)
+    # We check BOOTSTRAP_TIME for compatibility with old versions of
+    # test-network.sh
+    'bootstrap_time': int(os.environ.get('CHUTNEY_BOOTSTRAP_TIME',
+                                         os.environ.get('BOOTSTRAP_TIME',
+                                                        60))),
 }
 
 

scripts/chutney_tests/verify.py

@@ -3,8 +3,20 @@ import chutney
 
 
 def run_test(network):
-    print("Verifying data transmission:")
-    status = _verify_traffic(network)
+    wait_time = network._dfltEnv['bootstrap_time']
+    start_time = time.time()
+    end_time = start_time + wait_time
+    print("Verifying data transmission: (retrying for up to %d seconds)"
+          % wait_time)
+    status = False
+    # Keep on retrying the verify until it succeeds or times out
+    while not status and time.time() < end_time:
+        # TrafficTester connections time out after ~3 seconds
+        # a TrafficTester times out after ~10 seconds if no data is being sent
+        status = _verify_traffic(network)
+        # Avoid madly spewing output if we fail immediately each time
+        if not status:
+            time.sleep(2)
     print("Transmission: %s" % ("Success" if status else "Failure"))
     if not status:
         # TODO: allow the debug flag to be passed as an argument to

tools/test-network.sh

@@ -21,7 +21,7 @@ do
       shift
     ;;
     --delay|--sleep|--bootstrap-time|--time)
-      export BOOTSTRAP_TIME="$2"
+      export CHUTNEY_BOOTSTRAP_TIME="$2"
       shift
     ;;
     # Environmental variables used by chutney verify performance tests
@@ -137,13 +137,12 @@ fi
 cd "$CHUTNEY_PATH"
 ./tools/bootstrap-network.sh $NETWORK_FLAVOUR || exit 2
 
-# Sleep some, waiting for the network to bootstrap.
-# TODO: Add chutney command 'bootstrap-status' and use that instead.
-BOOTSTRAP_TIME=${BOOTSTRAP_TIME:-35}
-$ECHO_N "$myname: sleeping for $BOOTSTRAP_TIME seconds"
-n=$BOOTSTRAP_TIME; while [ $n -gt 0 ]; do
-    sleep 1; n=$(expr $n - 1); $ECHO_N .
-done; echo ""
+# chutney verify starts immediately, and keeps on trying for 60 seconds
+CHUTNEY_BOOTSTRAP_TIME=${CHUTNEY_BOOTSTRAP_TIME:-60}
+# but even the fastest tor networks take 5 seconds for their first consensus
+# and then 10 seconds after that for relays to bootstrap and upload descriptors
+echo "Waiting 15 seconds for a consensus containing relaysto be generated..."
+sleep 15
 ./chutney verify $CHUTNEY_NETWORK
 VERIFY_EXIT_STATUS=$?
 # work around a bug/feature in make -j2 (or more)