Merge branch 'additional_rust_test_fixes' into maint-0.3.4

.gitignore

@@ -45,6 +45,7 @@ uptime-*.json
 /autom4te.cache
 /build-stamp
 /compile
+/config.rust
 /configure
 /Doxyfile
 /orconfig.h
@@ -55,6 +56,7 @@ uptime-*.json
 /config.guess
 /config.sub
 /conftest*
+/link_rust.sh
 /micro-revision.*
 /patch-stamp
 /stamp-h

config.rust.in

@@ -0,0 +1,22 @@
+# Used by our cargo build.rs script to get variables from autoconf.
+#
+# The "configure" script will generate "config.rust" from "config.rust.in",
+# and then build.rs will read "config.rust".
+
+BUILDDIR=@BUILDDIR@
+TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@
+TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@
+TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@
+TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@
+TOR_LIB_MATH=@TOR_LIB_MATH@
+TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@
+TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@
+TOR_LIB_WS32=@TOR_LIB_WS32@
+TOR_LIB_GDI=@TOR_LIB_GDI@
+TOR_LIB_USERENV=@TOR_LIB_USERENV@
+CURVE25519_LIBS=@CURVE25519_LIBS@
+TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@
+TOR_LZMA_LIBS=@TOR_LZMA_LIBS@
+TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@
+LIBS=@LIBS@
+LDFLAGS=@LDFLAGS@

configure.ac

@@ -1118,6 +1118,33 @@ if test "$fragile_hardening" = "yes"; then
 TOR_CHECK_CFLAGS([-fno-omit-frame-pointer])
 fi
 
+dnl Find the correct libraries to add in order to use the sanitizers.
+dnl
+dnl When building Rust, Cargo will run the linker with the -nodefaultlibs
+dnl option, which will prevent the compiler from linking the sanitizer
+dnl libraries it needs.  We need to specify them manually.
+dnl
+dnl What's more, we need to specify them in a linker script rather than
+dnl from build.rs: these options aren't allowed in the cargo:rustc-flags
+dnl variable.
+RUST_LINKER_OPTIONS=""
+if test "x$have_clang" = "xyes"; then
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_ASAN"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_UBSAN"
+	fi
+else
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lasan"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lubsan"
+	fi
+fi
+AC_SUBST(RUST_LINKER_OPTIONS)
+
 CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN"
 CFLAGS_CONSTTIME="$CFLAGS_FWRAPV"
 
@@ -2267,6 +2294,8 @@ CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_z
 AC_CONFIG_FILES([
         Doxyfile
         Makefile
+        config.rust
+        link_rust.sh
         contrib/dist/suse/tor.sh
         contrib/operator-tools/tor.logrotate
         contrib/dist/tor.sh

link_rust.sh.in

@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# A linker script used when building Rust tests.  Autoconf makes link_rust.sh
+# from link_rust_sh.in, and uses it to pass extra options to the linker
+# when linking Rust stuff.
+#
+# We'd like to remove the need for this, but build.rs doesn't let us pass
+# -static-libasan and -static-libubsan to the linker.
+
+$CCLD @RUST_LINKER_OPTIONS@ "$@"

src/common/util.c

@@ -5375,3 +5375,4 @@ tor_ntohll(uint64_t a)
 {
   return tor_htonll(a);
 }
+

src/or/control.c

@@ -7736,3 +7736,4 @@ control_testing_set_global_event_mask(uint64_t mask)
   global_event_mask = mask;
 }
 #endif /* defined(TOR_UNIT_TESTS) */
+

src/or/geoip.c

@@ -1884,3 +1884,4 @@ geoip_free_all(void)
   memset(geoip_digest, 0, sizeof(geoip_digest));
   memset(geoip6_digest, 0, sizeof(geoip6_digest));
 }
+

src/rust/build.rs

@@ -0,0 +1,179 @@
+//! Build script for Rust modules in Tor.
+//!
+//! We need to use this because some of our Rust tests need to use some
+//! of our C modules, which need to link some external libraries.
+//!
+//! This script works by looking at a "config.rust" file generated by our
+//! configure script, and then building a set of options for cargo to pass to
+//! the compiler.
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io::prelude::*;
+use std::io;
+use std::path::PathBuf;
+
+/// Wrapper around a key-value map.
+struct Config(
+    HashMap<String,String>
+);
+
+/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR
+/// location provided by cargo and recursing up the directory tree.  Note that
+/// we need to look in the OUT_DIR, since autoconf will place generated files
+/// in the build directory.
+fn find_cfg() -> io::Result<String> {
+    let mut path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    loop {
+        path.push("config.rust");
+        if path.exists() {
+            return Ok(path.to_str().unwrap().to_owned());
+        }
+        path.pop(); // remove config.rust
+        if ! path.pop() { // can't remove last part of directory
+            return Err(io::Error::new(io::ErrorKind::NotFound,
+                                      "No config.rust"));
+        }
+    }
+}
+
+impl Config {
+    /// Find the config.rust file and try to parse it.
+    ///
+    /// The file format is a series of lines of the form KEY=VAL, with
+    /// any blank lines and lines starting with # ignored.
+    fn load() -> io::Result<Config> {
+        let path = find_cfg()?;
+        let f = File::open(&path)?;
+        let reader = io::BufReader::new(f);
+        let mut map = HashMap::new();
+        for line in reader.lines() {
+            let s = line?;
+            if s.trim().starts_with("#") || s.trim() == "" {
+                continue;
+            }
+            let idx = match s.find("=") {
+                None => {
+                    return Err(io::Error::new(io::ErrorKind::InvalidData,
+                                              "missing ="));
+                },
+                Some(x) => x
+            };
+            let (var,eq_val) = s.split_at(idx);
+            let val = &eq_val[1..];
+            map.insert(var.to_owned(), val.to_owned());
+        }
+        Ok(Config(map))
+    }
+
+    /// Return a reference to the value whose key is 'key'.
+    ///
+    /// Panics if 'key' is not found in the configuration.
+    fn get(&self, key : &str) -> &str {
+        self.0.get(key).unwrap()
+    }
+
+    /// Add a dependency on a static C library that is part of Tor, by name.
+    fn component(&self, s : &str) {
+        println!("cargo:rustc-link-lib=static={}", s);
+    }
+
+    /// Add a dependency on a native library that is not part of Tor, by name.
+    fn dependency(&self, s : &str) {
+        println!("cargo:rustc-link-lib={}", s);
+    }
+
+    /// Add a link path, relative to Tor's build directory.
+    fn link_relpath(&self, s : &str) {
+        let builddir = self.get("BUILDDIR");
+        println!("cargo:rustc-link-search=native={}/{}", builddir, s);
+    }
+
+    /// Add an absolute link path.
+    fn link_path(&self, s : &str) {
+        println!("cargo:rustc-link-search=native={}", s);
+    }
+
+    /// Parse the CFLAGS in s, looking for -l and -L items, and adding
+    /// rust configuration as appropriate.
+    fn from_cflags(&self, s : &str) {
+        let mut next_is_lib = false;
+        let mut next_is_path = false;
+        for ent in self.get(s).split_whitespace() {
+            if next_is_lib {
+                self.dependency(ent);
+                next_is_lib = false;
+            } else if next_is_path {
+                self.link_path(ent);
+                next_is_path = false;
+            } else if ent == "-l" {
+                next_is_lib = true;
+            } else if ent == "-L" {
+                next_is_path = true;
+            } else if ent.starts_with("-L") {
+                self.link_path(&ent[2..]);
+            } else if ent.starts_with("-l") {
+                self.dependency(&ent[2..]);
+            }
+        }
+    }
+}
+
+pub fn main() {
+    let cfg = Config::load().unwrap();
+    let package = env::var("CARGO_PKG_NAME").unwrap();
+
+    match package.as_ref() {
+        "crypto" => {
+            // Right now, I'm having a separate configuration for each Rust
+            // package, since I'm hoping we can trim them down.  Once we have a
+            // second Rust package that needs to use this build script, let's
+            // extract some of this stuff into a module.
+            //
+            // This is a ridiculous amount of code to be pulling in just
+            // to test our crypto library: modularity would be our
+            // friend here.
+            cfg.from_cflags("TOR_LDFLAGS_zlib");
+            cfg.from_cflags("TOR_LDFLAGS_openssl");
+            cfg.from_cflags("TOR_LDFLAGS_libevent");
+
+            cfg.link_relpath("src/common");
+            cfg.link_relpath("src/ext/keccak-tiny");
+            cfg.link_relpath("src/ext/keccak-tiny");
+            cfg.link_relpath("src/ext/ed25519/ref10");
+            cfg.link_relpath("src/ext/ed25519/donna");
+            cfg.link_relpath("src/trunnel");
+
+            // Note that we can't pull in "libtor-testing", or else we
+            // will have dependencies on all the other rust packages that
+            // tor uses.  We must be careful with factoring and dependencies
+            // moving forward!
+            cfg.component("or-crypto-testing");
+            cfg.component("or-ctime-testing");
+            cfg.component("or-testing");
+            cfg.component("or-event-testing");
+            cfg.component("or-ctime-testing");
+            cfg.component("curve25519_donna");
+            cfg.component("keccak-tiny");
+            cfg.component("ed25519_ref10");
+            cfg.component("ed25519_donna");
+            cfg.component("or-trunnel-testing");
+
+            cfg.from_cflags("TOR_ZLIB_LIBS");
+            cfg.from_cflags("TOR_LIB_MATH");
+            cfg.from_cflags("TOR_OPENSSL_LIBS");
+            cfg.from_cflags("TOR_LIBEVENT_LIBS");
+            cfg.from_cflags("TOR_LIB_WS32");
+            cfg.from_cflags("TOR_LIB_GDI");
+            cfg.from_cflags("TOR_LIB_USERENV");
+            cfg.from_cflags("CURVE25519_LIBS");
+            cfg.from_cflags("TOR_LZMA_LIBS");
+            cfg.from_cflags("TOR_ZSTD_LIBS");
+            cfg.from_cflags("LIBS");
+        },
+        _ => {
+            panic!("No configuration in build.rs for package {}", package);
+        }
+    }
+}

src/rust/crypto/Cargo.toml

@@ -4,6 +4,7 @@ authors = ["The Tor Project",
 name = "crypto"
 version = "0.0.1"
 publish = false
+build = "../build.rs"
 
 [lib]
 name = "crypto"
@@ -25,4 +26,3 @@ rand = { version = "=0.5.0-pre.2", default-features = false }
 rand_core = { version = "=0.2.0-pre.0", default-features = false }
 
 [features]
-

src/rust/crypto/digests/sha2.rs

@@ -43,7 +43,7 @@ pub struct Sha256 {
 ///
 /// # Examples
 ///
-/// ```
+/// ```rust,no_run
 /// use crypto::digests::sha2::{Sha256, Digest};
 ///
 /// let mut hasher: Sha256 = Sha256::default();
@@ -66,7 +66,7 @@ impl BlockInput for Sha256 {
 ///
 /// # Examples
 ///
-/// ```
+/// ```rust,no_run
 /// use crypto::digests::sha2::{Sha256, Digest};
 ///
 /// let mut hasher: Sha256 = Sha256::default();
@@ -110,7 +110,7 @@ pub struct Sha512 {
 ///
 /// # Examples
 ///
-/// ```
+/// ```rust,no_run
 /// use crypto::digests::sha2::{Sha512, Digest};
 ///
 /// let mut hasher: Sha512 = Sha512::default();
@@ -133,7 +133,7 @@ impl BlockInput for Sha512 {
 ///
 /// # Examples
 ///
-/// ```
+/// ```rust,no_run
 /// use crypto::digests::sha2::{Sha512, Digest};
 ///
 /// let mut hasher: Sha512 = Sha512::default();
@@ -154,7 +154,7 @@ impl Input for Sha512 {
 // FIXME: Once const generics land in Rust, we should genericise calling
 // crypto_digest_get_digest in external::crypto_digest.
 impl FixedOutput for Sha512 {
-    type OutputSize = U32;
+    type OutputSize = U64;
 
     fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> {
         let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine);
@@ -178,6 +178,9 @@ mod test {
     fn sha256_digest() {
         let mut h: Sha256 = Sha256::new();
         let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN];
+        let expected = [151, 223, 53, 136, 181, 163, 242, 75, 171, 195,
+                        133, 27, 55, 47, 11, 167, 26, 157, 205, 222, 212,
+                        59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157];
 
         h.input(b"foo");
         h.input(b"bar");
@@ -187,7 +190,7 @@ mod test {
 
         println!("{:?}", &result[..]);
 
-        assert_eq!(&result[..], &b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"[..]);
+        assert_eq!(result, expected);
     }
 
     #[test]
@@ -200,6 +203,12 @@ mod test {
         let mut h: Sha512 = Sha512::new();
         let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN];
 
+        let expected = [203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167,
+                153, 217, 233, 8, 190, 69, 231, 103, 245, 209, 71, 212, 116,
+                73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, 122,
+                221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38,
+                179, 237, 61, 126, 246, 240, 103, 202, 153, 24, 90];
+
         h.input(b"foo");
         h.input(b"bar");
         h.input(b"baz");
@@ -208,6 +217,6 @@ mod test {
 
         println!("{:?}", &result[..]);
 
-        assert_eq!(&result[..], &b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"[..]);
+        assert_eq!(&result[..], &expected[..]);
     }
 }

src/rust/crypto/lib.rs

@@ -9,7 +9,7 @@
 //! The `digests` module contains submodules for specific hash digests
 //! and extendable output functions.
 //!
-//! ```
+//! ```rust,no_run
 //! use crypto::digests::sha2::*;
 //!
 //! let mut hasher: Sha256 = Sha256::default();
@@ -43,4 +43,3 @@ extern crate tor_log;
 
 pub mod digests;  // Unfortunately named "digests" plural to avoid name conflict with the digest crate
 pub mod rand;
-

src/rust/external/crypto_digest.rs

@@ -140,7 +140,7 @@ extern "C" {
     fn crypto_digest_new() -> *mut crypto_digest_t;
     fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
     fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
-    fn crypto_digest_free(digest: *mut crypto_digest_t);
+    fn crypto_digest_free_(digest: *mut crypto_digest_t);
     fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t);
     fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t);
     fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t;
@@ -292,6 +292,14 @@ impl CryptoDigest {
     }
 }
 
+impl Drop for CryptoDigest {
+    fn drop(&mut self) {
+        unsafe {
+            crypto_digest_free_(self.0 as *mut crypto_digest_t);
+        }
+    }
+}
+
 /// Get the 256-bit digest output of a `crypto_digest_t`.
 ///
 /// # Inputs

src/rust/include.am

@@ -1,6 +1,7 @@
 include src/rust/tor_rust/include.am
 
 EXTRA_DIST +=\
+	src/rust/build.rs \
 	src/rust/Cargo.toml \
 	src/rust/Cargo.lock \
 	src/rust/.cargo/config.in \

src/test/include.am

@@ -10,7 +10,10 @@ TESTS_ENVIRONMENT = \
 	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \
 	export CARGO="$(CARGO)"; \
 	export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \
-	export CARGO_ONLINE="$(CARGO_ONLINE)";
+	export CARGO_ONLINE="$(CARGO_ONLINE)"; \
+        export CCLD="$(CCLD)"; \
+        chmod +x "$(abs_top_builddir)/link_rust.sh"; \
+        export RUSTFLAGS="-C linker=$(abs_top_builddir)/link_rust.sh";
 
 TESTSCRIPTS = \
 	src/test/fuzz_static_testcases.sh \
@@ -347,7 +350,7 @@ src_test_test_bt_cl_LDADD = src/common/libor-testing.a \
 	src/trace/libor-trace.a \
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ 
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@
 src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
 
@@ -361,6 +364,7 @@ EXTRA_DIST += \
 	src/test/fuzz_static_testcases.sh \
 	src/test/slownacl_curve25519.py \
 	src/test/zero_length_keys.sh \
+        src/test/rust_supp.txt \
 	src/test/test_keygen.sh \
 	src/test/test_key_expiration.sh \
 	src/test/test_zero_length_keys.sh \

src/test/rust_supp.txt

@@ -0,0 +1 @@
+leak:backtrace_alloc

src/test/test_rust.sh

@@ -3,6 +3,7 @@
 
 set -e
 
+export LSAN_OPTIONS=suppressions=${abs_top_srcdir}/src/test/rust_supp.txt
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
@@ -16,5 +17,3 @@ for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
 done
 
 exit $exitcode
-
-