Add "make test-network-all" to verify multiple test networks

make test-network-all is Makefile target which verifies a series
of test networks generated using test-network.sh and chutney.

It runs IPv6 and mixed version test networks if the prerequisites are
available.

Each test network reports PASS, FAIL, or SKIP.
Closes ticket 16953. Patch by "teor".

Also adds "--hs-multi-client 1" option to TEST_NETWORK_FLAGS.
This resolves #17012.

Larger networks, such as bridges+hs, may fail until #16952 is merged.

Makefile.am

@@ -36,13 +36,16 @@ EXTRA_DIST+= \
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
-TEST_NETWORK_FLAGS="--coverage"
+TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS
-TEST_NETWORK_FLAGS=
+TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
 
+TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
+TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
+
 #install-data-local:
 #	$(INSTALL) -m 755 -d $(LOCALSTATEDIR)/lib/tor
 
@@ -88,6 +91,36 @@ need-chutney-path:
 test-network: need-chutney-path all
 	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
 
+# Run all available tests using automake's test-driver
+# only run IPv6 tests if we can ping6 ::1 (localhost)
+# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
+# only run mixed tests if we have a tor-stable binary
+# see #17015 for autodetection of different tor versions
+test-network-all: need-chutney-path all test-driver
+	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
+	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
+	if ping6 -q -o ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+	else \
+		echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+	fi; \
+	if command -v tor-stable >/dev/null 2>&1; then \
+		echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+	else \
+		echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+	fi; \
+	for f in $$skip_flavors; do \
+		echo "SKIP: $$f"; \
+	done; \
+	for f in $$flavors; do \
+		./test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+	done
+	@echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."
+
 need-stem-path:
 	@if test ! -d "$$STEM_SOURCE_DIR"; then \
 		echo '$$STEM_SOURCE_DIR was not set.'; echo; \
@@ -173,3 +206,4 @@ mostlyclean-local:
 	rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
 	rm -rf $(HTML_COVER_DIR)
 	rm -rf $(top_builddir)/doc/doxygen
+	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)

src/test/include.am

@@ -8,6 +8,13 @@ endif
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue src/test/test_keygen.sh $(TESTSCRIPTS)
 
+# These flavors are run using automake's test-driver and test-network.sh
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-min bridges+hs
+# only run if we can ping6 ::1 (localhost)
+TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min
+# only run if we can find a stable (or simply another) version of tor
+TEST_CHUTNEY_FLAVORS_MIXED = mixed
+
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.  Below is a kludge to work around.
 ###