Merge branch 'travis-osx-034' into travis-osx-master

Replace master .travis.yml with 034 .travis.yml.
All the changes in master have been backported to the
034 .travis.yml already.

Replace master src/test/test_rust.sh with 034
src/test/test_rust.sh, which was backported from
master. One 033/034-specific commit needs to be
reverted.

.travis.yml

@@ -1,40 +1,97 @@
 language: c
 
-## Comment out the compiler list for now to allow an explicit build
-## matrix.
-# compiler:
-#   - gcc
-#   - clang
+cache:
+  ccache: true
+  ## cargo: true
+  directories:
+    - $HOME/.cargo
+    ## where we point CARGO_TARGET_DIR in all our cargo invocations
+    - $TRAVIS_BUILD_DIR/src/rust/target
 
-notifications:
-  irc:
-    channels:
-      - "irc.oftc.net#tor-ci"
-    template:
-      - "%{repository} %{branch} %{commit} - %{author}: %{commit_subject}"
-      - "Build #%{build_number} %{result}. Details: %{build_url}"
-    on_success: change
-    on_failure: change
-  email:
-    on_success: never
-    on_failure: change
+compiler:
+  - gcc
+  - clang
 
 os:
   - linux
-  ## Uncomment the following line to also run the entire build matrix on OSX.
-  ## This will make your CI builds take roughly ten times longer to finish.
-  # - osx
+  - osx
 
-## Use the Ubuntu Trusty images.
-dist: trusty
+## The build matrix in the following stanza expands into builds for each
+## OS and compiler.
+env:
+  global:
+    ## The Travis CI environment allows us two cores, so let's use both.
+    - MAKEFLAGS="-j 2"
+    ## We turn on hardening by default
+    ## Also known as --enable-fragile-hardening in 0.3.0.3-alpha and later
+    - HARDENING_OPTIONS="--enable-expensive-hardening"
+    ## We turn off asciidoc by default, because it's slow
+    - ASCIIDOC_OPTIONS="--disable-asciidoc"
+  matrix:
+    ## We want to use each build option at least once
+    ##
+    ## We don't list default variable values, because we set the defaults
+    ## in global (or the default is unset)
+    -
+    ## We turn off hardening for Rust builds, because they are incompatible,
+    ## and it's going to take a while for them to be fixed. See:
+    ##   https:/trac.torproject.org/projects/tor/ticket/25386
+    ##   https:/trac.torproject.org/projects/tor/ticket/26398
+    - RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" HARDENING_OPTIONS=""
+
+matrix:
+  ## include creates builds with gcc, linux, sudo: false
+  include:
+    ## We include a single coverage build with the best options for coverage
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    ## We only want to check these build option combinations once
+    ## (they shouldn't vary by compiler or OS)
+    ## We run rust and coverage with hardening off, which seems like enough
+    # - env: HARDENING_OPTIONS=""
+    ## We check asciidoc with distcheck, to make sure we remove doc products
+    - env: DISTCHECK="yes" ASCIIDOC_OPTIONS=""
+    ## Check rust offline with distcheck, to make sure we remove rust products
+    ## But without hardening (see above)
+    - env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
+    ## Check disable module dirauth with rust, and without rust but with distcheck
+    - env: MODULES_OPTIONS="--disable-module-dirauth" RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
+    - env: DISTCHECK="yes" MODULES_OPTIONS="--disable-module-dirauth"
+
+  ## Uncomment to allow the build to report success (with non-required
+  ## sub-builds continuing to run) if all required sub-builds have
+  ## succeeded.  This is somewhat buggy currently: it can cause
+  ## duplicate notifications and prematurely report success if a
+  ## single sub-build has succeeded.  See
+  ## https://github.com/travis-ci/travis-ci/issues/1696
+  # fast_finish: true
 
-## We don't need sudo. (The "apt:" stanza after this allows us to not need sudo;
-## otherwise, we would need it for getting dependencies.)
+  ## Careful! We use global envs, which makes it hard to exclude or
+  ## allow failures by env:
+  ## https://docs.travis-ci.com/user/customizing-the-build#matching-jobs-with-allow_failures
+  exclude:
+    ## Clang doesn't work in containerized builds, see below.
+    - compiler: clang
+      sudo: false
+    ## We also exclude non-containerized gcc, because they're slow and redundant.
+    - compiler: gcc
+      sudo: required
+
+## We don't need sudo. (The "apt:" stanza after this allows us to not need
+## sudo; otherwise, we would need it for getting dependencies.)
 ##
-## We override this in the explicit build matrix to work around a
-## Travis CI environment regression
+## But we use "sudo: required" to force non-containerized builds, working
+## around a Travis CI environment issue: clang LeakAnalyzer fails
+## because it requires ptrace and the containerized environment no
+## longer allows ptrace.
 ## https://github.com/travis-ci/travis-ci/issues/9033
-sudo: false
+##
+## In the matrix above, we exclude redundant combinations.
+sudo:
+  - false
+  - required
+
+## (Linux only) Use the latest Linux image (Ubuntu Trusty)
+dist: trusty
 
 ## (Linux only) Download our dependencies
 addons:
@@ -50,122 +107,79 @@ addons:
       - libseccomp-dev
       ## zstd doesn't exist in Ubuntu Trusty
       #- libzstd
+      ## Conditional dependencies
+      ## Always installed, so we don't need sudo
+      - asciidoc
+      - docbook-xsl
+      - docbook-xml
+      - xmlto
 
-## The build matrix in the following two stanzas expands into four builds (per OS):
-##
-##  * with GCC, with Rust
-##  * with GCC, without Rust
-##  * with Clang, with Rust
-##  * with Clang, without Rust
-env:
-  global:
-    ## The Travis CI environment allows us two cores, so let's use both.
-    - MAKEFLAGS="-j 2"
-    - HARDENING_OPTIONS="--enable-fragile-hardening"
-  matrix:
-    ## Leave at least one entry here or Travis seems to generate a
-    ## matrix entry with empty matrix environment variables.  Leaving
-    ## more than one entry causes unwanted matrix entries with
-    ## unspecified compilers.
-    - RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    # - RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    # - RUST_OPTIONS=""
-
-matrix:
-  ## Uncomment to allow the build to report success (with non-required
-  ## sub-builds continuing to run) if all required sub-builds have
-  ## succeeded.  This is somewhat buggy currently: it can cause
-  ## duplicate notifications and prematurely report success if a
-  ## single sub-build has succeeded.  See
-  ## https://github.com/travis-ci/travis-ci/issues/1696
-  # fast_finish: true
-
-  ## Uncomment the appropriate lines below to allow the build to
-  ## report success even if some less-critical sub-builds fail and it
-  ## seems likely to take a while for someone to fix it.  Currently
-  ## Travis CI doesn't distinguish "all builds succeeded" from "some
-  ## non-required sub-builds failed" except on the individual build's
-  ## page, which makes it somewhat annoying to detect from the
-  ## branches and build history pages.  See
-  ## https://github.com/travis-ci/travis-ci/issues/8716
-  allow_failures:
-    - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    - env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    # - compiler: clang
-
-  ## Create explicit matrix entries to work around a Travis CI
-  ## environment issue.  Missing keys inherit from the first list
-  ## entry under that key outside the "include" clause.
-  include:
-    - compiler: gcc
-    - compiler: gcc
-      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - compiler: gcc
-      env: RUST_OPTIONS=""
-    - compiler: gcc
-      env: COVERAGE_OPTIONS="--enable-coverage"
-    - compiler: gcc
-      env: DISTCHECK="yes" RUST_OPTIONS=""
-    - compiler: gcc
-      env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    - compiler: gcc
-      env: MODULES_OPTIONS="--disable-module-dirauth"
-    - compiler: gcc
-      env: HARDENING_OPTIONS="" RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - compiler: gcc
-      env: DISTCHECK="yes" HARDENING_OPTIONS="" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
-    ## The "sudo: required" forces non-containerized builds, working
-    ## around a Travis CI environment issue: clang LeakAnalyzer fails
-    ## because it requires ptrace and the containerized environment no
-    ## longer allows ptrace.
-    - compiler: clang
-      sudo: required
-    - compiler: clang
-      sudo: required
-      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true
-    - compiler: clang
-      sudo: required
-      env: RUST_OPTIONS=""
-    - compiler: clang
-      sudo: required
-      env: MODULES_OPTIONS="--disable-module-dirauth"
+## (OSX only) Use the default OSX image
+## See https://docs.travis-ci.com/user/reference/osx#os-x-version
+## Default is Xcode 9.4 on macOS 10.13 as of August 2018
+#osx_image: xcode9.4
 
 before_install:
-  ## If we're on OSX, homebrew usually needs to updated first
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
-  ## Download rustup
-  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
-  - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
+  ## If we're on OSX, homebrew usually needs to be updated first
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
+  ## We might be upgrading some useless packages, but that's better than missing an upgrade
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew upgrade; fi
+  ## Create empty rust directories for non-Rust builds, so caching succeeds
+  - if [[ "$RUST_OPTIONS" == "" ]]; then mkdir -p $HOME/.cargo $TRAVIS_BUILD_DIR/src/rust/target; fi
 
 install:
+  ## If we're on OSX use brew to install ccache (ccache is automatically installed on Linux)
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install ccache; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/usr/local/opt/ccache/libexec:$PATH"; fi
   ## If we're on OSX use brew to install required dependencies (for Linux, see the "apt:" section above)
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated openssl    || brew upgrade openssl;    }; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated libevent   || brew upgrade libevent;   }; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated pkg-config || brew upgrade pkg-config; }; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libevent; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install pkg-config; fi
+  ## macOS comes with zlib by default, so the homebrew install is keg-only
+  # - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install zlib; fi
   ## If we're on OSX also install the optional dependencies
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated xz         || brew upgrade xz;         }; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated libscrypt  || brew upgrade libscrypt;  }; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated zstd       || brew upgrade zstd;       }; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libscrypt; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install xz; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install zstd; fi
+  ## If we're on OSX, OpenSSL is keg-only, so tor 0.2.9 and later need to be configured --with-openssl-dir= to build
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then OPENSSL_OPTIONS=--with-openssl-dir=`brew --prefix openssl`; fi
+  ## Install conditional features
+  ## Install coveralls
+  - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
+  ## If we're on OSX, and using asciidoc, install asciidoc
+  - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install asciidoc; fi
+  ## If we're using Rust, download rustup
+  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
   ## Install the stable channels of rustc and cargo and setup our toolchain environment
   - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain stable; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
-  ## Get some info about rustc and cargo
+  ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
+  - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
+  ##
+  ## Finally, list installed package versions
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then dpkg-query --show; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi
+  ## Get some info about rustup, rustc and cargo
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustup; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
-  ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
-  - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
 
 script:
   - ./autogen.sh
-  - ./configure $RUST_OPTIONS $COVERAGE_OPTIONS $MODULES_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules $HARDENING_OPTIONS
+  - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules"
+  - echo $CONFIGURE_FLAGS
+  - ./configure $CONFIGURE_FLAGS
   ## We run `make check` because that's what https://jenkins.torproject.org does.
   - if [[ "$DISTCHECK" == "" ]]; then make check; fi
-  - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$RUST_OPTIONS $COVERAGE_OPTIONS --disable-asciidoc --enable-fatal-warnings --disable-silent-rules $HARDENING_OPTIONS"; fi
+  - if [[ "$DISTCHECK" != "" ]]; then make distcheck DISTCHECK_CONFIGURE_FLAGS="$CONFIGURE_FLAGS"; fi
 
 after_failure:
+  ## configure will leave a log file with more details of config failures.
+  ## But the log is too long for travis' rendered view, so tail it.
+  - tail -1000 config.log
   ## `make check` will leave a log file with more details of test failures.
   - if [[ "$DISTCHECK" == "" ]]; then cat test-suite.log; fi
   ## `make distcheck` puts it somewhere different.
@@ -174,3 +188,16 @@ after_failure:
 after_success:
   ## If this build was one that produced coverage, upload it.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then coveralls -b . --exclude src/test --exclude src/trunnel --gcov-options '\-p'; fi
+
+notifications:
+  irc:
+    channels:
+      - "irc.oftc.net#tor-ci"
+    template:
+      - "%{repository} %{branch} %{commit} - %{author}: %{commit_subject}"
+      - "Build #%{build_number} %{result}. Details: %{build_url}"
+    on_success: change
+    on_failure: change
+  email:
+    on_success: never
+    on_failure: change

changes/bug26497

@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.

changes/bug26497-backport

@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Backport test_rust.sh from master.
+      Fixes bug 26497; bugfix on 0.3.1.5-alpha.

changes/bug26497-cd

@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.

changes/bug27088

@@ -0,0 +1,5 @@
+  o Minor bugfixes (continuous integration):
+    - Pass the module flags to distcheck configure, and
+      log the flags before running configure. (Backported
+      to 0.2.9 and later as a precaution.)
+      Fixes bug 27088; bugfix on 0.3.4.1-alpha.

changes/bug27090

@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Build with zstd on macOS.
+      Fixes bug 27090; bugfix on 0.3.1.5-alpha.

changes/bug27093

@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.

changes/ticket24629

@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Enable macOS builds in our Travis CI configuration.
+      Closes ticket 24629.

changes/ticket24629-backport

@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Backport Travis rust distcheck to 0.3.3.
+      Closes ticket 24629.

changes/ticket26952-cargo

@@ -0,0 +1,3 @@
+  o Minor features (continuous integration, rust):
+    - Use cargo cache in our Travis CI configuration.
+      Closes ticket 26952.

changes/ticket26952-ccache

@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Use ccache in our Travis CI configuration.
+      Closes ticket 26952.

changes/ticket27087

@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Run asciidoc during Travis CI.
+      Implements ticket 27087.

src/rust/tor_rust/include.am

@@ -7,7 +7,6 @@ EXTRA_CARGO_OPTIONS=
 @TOR_RUST_LIB_PATH@: FORCE
 	( cd "$(abs_top_builddir)/src/rust" ; \
 		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
-		CARGO_HOME="$(abs_top_builddir)/src/rust" \
 		$(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \
 		$(CARGO_ONLINE) \
 		--manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )
@@ -15,7 +14,6 @@ EXTRA_CARGO_OPTIONS=
 distclean-rust:
 	( cd "$(abs_top_builddir)/src/rust" ; \
 		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
-		CARGO_HOME="$(abs_top_builddir)/src/rust" \
 		$(CARGO) clean $(EXTRA_CARGO_OPTIONS) \
 		$(CARGO_ONLINE) \
 		--manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )

src/test/test_rust.sh

@@ -3,14 +3,13 @@
 
 set -e
 
-export LSAN_OPTIONS=suppressions=${abs_top_srcdir}/src/test/rust_supp.txt
+export LSAN_OPTIONS=suppressions=${abs_top_srcdir:-../../..}/src/test/rust_supp.txt
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
-	cd "${cargo_toml_dir}" && \
+	cd "${abs_top_builddir:-../../..}/src/rust" && \
 	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
-	    CARGO_HOME="${abs_top_builddir:-../../..}/src/rust/.cargo" \
-	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+	    "${CARGO:-cargo}" test --all-features ${CARGO_ONLINE-"--frozen"} \
 	    ${EXTRA_CARGO_OPTIONS} \
 	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
     fi