Merge branch 'log_dependencies'

.gitignore

@@ -165,15 +165,31 @@ uptime-*.json
 /src/lib/libcurve25519_donna.a
 /src/lib/libtor-compress.a
 /src/lib/libtor-compress-testing.a
+/src/lib/libtor-container.a
+/src/lib/libtor-container-testing.a
 /src/lib/libtor-crypt-ops.a
 /src/lib/libtor-crypt-ops-testing.a
 /src/lib/libtor-ctime.a
 /src/lib/libtor-ctime-testing.a
 /src/lib/libtor-err.a
 /src/lib/libtor-err-testing.a
+/src/lib/libtor-fdio.a
+/src/lib/libtor-fdio-testing.a
+/src/lib/libtor-intmath.a
+/src/lib/libtor-intmath-testing.a
+/src/lib/libtor-lock.a
+/src/lib/libtor-lock-testing.a
+/src/lib/libtor-log.a
+/src/lib/libtor-log-testing.a
+/src/lib/libtor-malloc.a
+/src/lib/libtor-malloc-testing.a
+/src/lib/libtor-string.a
+/src/lib/libtor-string-testing.a
 /src/lib/libtor-tls.a
 /src/lib/libtor-tls-testing.a
 /src/lib/libtor-trace.a
+/src/lib/libtor-wallclock.a
+/src/lib/libtor-wallclock-testing.a
 
 # /src/or/
 /src/or/Makefile

Makefile.am

@@ -40,14 +40,30 @@ endif
 # "Common" libraries used to link tor's utility code.
 TOR_UTIL_LIBS = \
 	src/common/libor.a \
+        src/lib/libtor-log.a \
+        src/lib/libtor-lock.a \
+        src/lib/libtor-fdio.a \
+	src/lib/libtor-container.a \
+        src/lib/libtor-string.a \
+	src/lib/libtor-malloc.a \
+        src/lib/libtor-wallclock.a \
         src/lib/libtor-err.a \
+        src/lib/libtor-intmath.a \
 	src/lib/libtor-ctime.a
 
 # Variants of the above for linking the testing variant of tor (for coverage
 # and tests)
 TOR_UTIL_TESTING_LIBS = \
 	src/common/libor-testing.a \
+        src/lib/libtor-log-testing.a \
+        src/lib/libtor-lock-testing.a \
+        src/lib/libtor-fdio-testing.a \
+	src/lib/libtor-container-testing.a \
+        src/lib/libtor-string-testing.a \
+	src/lib/libtor-malloc-testing.a \
+        src/lib/libtor-wallclock-testing.a \
         src/lib/libtor-err-testing.a \
+        src/lib/libtor-intmath.a \
         src/lib/libtor-ctime-testing.a
 
 # Internal crypto libraries used in Tor

scripts/maint/checkIncludes.py

@@ -58,12 +58,21 @@ def load_include_rules(fname):
             result.addPattern(line)
     return result
 
+MDD = "--mdd" in sys.argv[1:]
+
+if MDD:
+    print(
+
 for dirpath, dirnames, fnames in os.walk("src"):
     if ".may_include" in fnames:
         rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
-        for fname in fnames:
-            if fname_is_c(fname):
-                rules.applyToFile(os.path.join(dirpath,fname))
+        if MDD:
+            dp = dirpath.replace("src/","")
+            rules.dump_mdd(dp)
+        else:
+            for fname in fnames:
+                if fname_is_c(fname):
+                    rules.applyToFile(os.path.join(dirpath,fname))
 
 if trouble:
     err(

src/common/.may_include

@@ -5,10 +5,7 @@ lib/*/*.h
 # XXXX These all belong somewhere else
 ht.h
 linux_syscalls.inc
-micro-revision.i
 siphash.h
 src/ext/timeouts/timeout.c
-strlcat.c
-strlcpy.c
 tor_queue.h
 tor_readpassphrase.h

src/common/address.c

@@ -39,9 +39,10 @@
 #include "common/util.h"
 #include "common/util_format.h"
 #include "common/address.h"
-#include "common/torlog.h"
-#include "common/container.h"
+#include "lib/log/torlog.h"
+#include "lib/container/smartlist.h"
 #include "common/sandbox.h"
+#include "siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -2169,4 +2170,3 @@ tor_addr_port_eq(const tor_addr_port_t *a,
 {
   return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port;
 }
-

src/common/address.h

@@ -15,7 +15,7 @@
 #include "orconfig.h"
 #include "lib/cc/torint.h"
 #include "common/compat.h"
-#include "common/container.h"
+#include "lib/log/util_bug.h"
 
 #ifdef ADDRESS_PRIVATE
 
@@ -206,10 +206,11 @@ const char * fmt_addr32(uint32_t addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
-void interface_address6_list_free_(smartlist_t * addrs);// XXXX
+struct smartlist_t;
+void interface_address6_list_free_(struct smartlist_t * addrs);// XXXX
 #define interface_address6_list_free(addrs) \
-  FREE_AND_NULL(smartlist_t, interface_address6_list_free_, (addrs))
-MOCK_DECL(smartlist_t *,get_interface_address6_list,(int severity,
+  FREE_AND_NULL(struct smartlist_t, interface_address6_list_free_, (addrs))
+MOCK_DECL(struct smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
 
@@ -335,7 +336,7 @@ MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
  * Returns NULL on failure.
  * Use free_interface_address_list to free the returned list.
  */
-static inline smartlist_t *
+static inline struct smartlist_t *
 get_interface_address_list(int severity, int include_internal)
 {
   return get_interface_address6_list(severity, AF_INET, include_internal);
@@ -346,34 +347,33 @@ int tor_addr_port_eq(const tor_addr_port_t *a,
                      const tor_addr_port_t *b);
 
 #ifdef ADDRESS_PRIVATE
-MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity,
+MOCK_DECL(struct smartlist_t *,get_interface_addresses_raw,(int severity,
                                                      sa_family_t family));
 MOCK_DECL(int,get_interface_address6_via_udp_socket_hack,(int severity,
                                                           sa_family_t family,
                                                           tor_addr_t *addr));
 
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
-STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
+STATIC struct smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
                                          sa_family_t family);
-STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity,
+STATIC struct smartlist_t *get_interface_addresses_ifaddrs(int severity,
                                                     sa_family_t family);
 #endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
-STATIC smartlist_t *ip_adapter_addresses_to_smartlist(
+STATIC struct smartlist_t *ip_adapter_addresses_to_smartlist(
                                         const IP_ADAPTER_ADDRESSES *addresses);
-STATIC smartlist_t *get_interface_addresses_win32(int severity,
+STATIC struct smartlist_t *get_interface_addresses_win32(int severity,
                                                   sa_family_t family);
 #endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
-STATIC smartlist_t *ifreq_to_smartlist(char *ifr,
+STATIC struct smartlist_t *ifreq_to_smartlist(char *ifr,
                                        size_t buflen);
-STATIC smartlist_t *get_interface_addresses_ioctl(int severity,
+STATIC struct smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #endif /* defined(ADDRESS_PRIVATE) */
 
 #endif /* !defined(TOR_ADDRESS_H) */
-

src/common/address_set.c

@@ -14,7 +14,7 @@
 #include "common/address_set.h"
 #include "common/address.h"
 #include "common/compat.h"
-#include "common/container.h"
+#include "lib/container/bitarray.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "common/util.h"
 #include "siphash.h"
@@ -126,4 +126,3 @@ address_set_probably_contains(address_set_t *set,
   }
   return matches == N_BITS_PER_ITEM;
 }
-

src/common/buffers.c

@@ -26,7 +26,7 @@
 #include "lib/compress/compress.h"
 #include "common/util.h"
 #include "lib/cc/torint.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif

src/common/compat.c

@@ -124,20 +124,13 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #include <sys/file.h>
 #endif
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
-#include "common/container.h"
+#include "lib/container/smartlist.h"
+#include "lib/wallclock/tm_cvt.h"
 #include "common/address.h"
 #include "common/sandbox.h"
 
-/* Inline the strl functions if the platform doesn't have them. */
-#ifndef HAVE_STRLCPY
-#include "strlcpy.c"
-#endif
-#ifndef HAVE_STRLCAT
-#include "strlcat.c"
-#endif
-
 /* When set_max_file_descriptors() is called, update this with the max file
  * descriptor value so we can use it to check the limit when opening a new
  * socket. Default value is what Debian sets as the default hard limit. */
@@ -404,147 +397,6 @@ tor_munmap_file(tor_mmap_t *handle)
 #error "cannot implement tor_mmap_file"
 #endif /* defined(HAVE_MMAP) || ... || ... */
 
-/** Replacement for snprintf.  Differs from platform snprintf in two
- * ways: First, always NUL-terminates its output.  Second, always
- * returns -1 if the result is truncated.  (Note that this return
- * behavior does <i>not</i> conform to C99; it just happens to be
- * easier to emulate "return -1" with conformant implementations than
- * it is to emulate "return number that would be written" with
- * non-conformant implementations.) */
-int
-tor_snprintf(char *str, size_t size, const char *format, ...)
-{
-  va_list ap;
-  int r;
-  va_start(ap,format);
-  r = tor_vsnprintf(str,size,format,ap);
-  va_end(ap);
-  return r;
-}
-
-/** Replacement for vsnprintf; behavior differs as tor_snprintf differs from
- * snprintf.
- */
-int
-tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
-{
-  int r;
-  if (size == 0)
-    return -1; /* no place for the NUL */
-  if (size > SIZE_T_CEILING)
-    return -1;
-#ifdef _WIN32
-  r = _vsnprintf(str, size, format, args);
-#else
-  r = vsnprintf(str, size, format, args);
-#endif
-  str[size-1] = '\0';
-  if (r < 0 || r >= (ssize_t)size)
-    return -1;
-  return r;
-}
-
-/**
- * Portable asprintf implementation.  Does a printf() into a newly malloc'd
- * string.  Sets *<b>strp</b> to this string, and returns its length (not
- * including the terminating NUL character).
- *
- * You can treat this function as if its implementation were something like
-   <pre>
-     char buf[_INFINITY_];
-     tor_snprintf(buf, sizeof(buf), fmt, args);
-     *strp = tor_strdup(buf);
-     return strlen(*strp):
-   </pre>
- * Where _INFINITY_ is an imaginary constant so big that any string can fit
- * into it.
- */
-int
-tor_asprintf(char **strp, const char *fmt, ...)
-{
-  int r;
-  va_list args;
-  va_start(args, fmt);
-  r = tor_vasprintf(strp, fmt, args);
-  va_end(args);
-  if (!*strp || r < 0) {
-    /* LCOV_EXCL_START */
-    log_err(LD_BUG, "Internal error in asprintf");
-    tor_assert(0);
-    /* LCOV_EXCL_STOP */
-  }
-  return r;
-}
-
-/**
- * Portable vasprintf implementation.  Does a printf() into a newly malloc'd
- * string.  Differs from regular vasprintf in the same ways that
- * tor_asprintf() differs from regular asprintf.
- */
-int
-tor_vasprintf(char **strp, const char *fmt, va_list args)
-{
-  /* use a temporary variable in case *strp is in args. */
-  char *strp_tmp=NULL;
-#ifdef HAVE_VASPRINTF
-  /* If the platform gives us one, use it. */
-  int r = vasprintf(&strp_tmp, fmt, args);
-  if (r < 0)
-    *strp = NULL;
-  else
-    *strp = strp_tmp;
-  return r;
-#elif defined(HAVE__VSCPRINTF)
-  /* On Windows, _vsnprintf won't tell us the length of the string if it
-   * overflows, so we need to use _vcsprintf to tell how much to allocate */
-  int len, r;
-  va_list tmp_args;
-  va_copy(tmp_args, args);
-  len = _vscprintf(fmt, tmp_args);
-  va_end(tmp_args);
-  if (len < 0) {
-    *strp = NULL;
-    return -1;
-  }
-  strp_tmp = tor_malloc(len + 1);
-  r = _vsnprintf(strp_tmp, len+1, fmt, args);
-  if (r != len) {
-    tor_free(strp_tmp);
-    *strp = NULL;
-    return -1;
-  }
-  *strp = strp_tmp;
-  return len;
-#else
-  /* Everywhere else, we have a decent vsnprintf that tells us how many
-   * characters we need.  We give it a try on a short buffer first, since
-   * it might be nice to avoid the second vsnprintf call.
-   */
-  char buf[128];
-  int len, r;
-  va_list tmp_args;
-  va_copy(tmp_args, args);
-  /* vsnprintf() was properly checked but tor_vsnprintf() available so
-   * why not use it? */
-  len = tor_vsnprintf(buf, sizeof(buf), fmt, tmp_args);
-  va_end(tmp_args);
-  if (len < (int)sizeof(buf)) {
-    *strp = tor_strdup(buf);
-    return len;
-  }
-  strp_tmp = tor_malloc(len+1);
-  /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
-  if (r != len) {
-    tor_free(strp_tmp);
-    *strp = NULL;
-    return -1;
-  }
-  *strp = strp_tmp;
-  return len;
-#endif /* defined(HAVE_VASPRINTF) || ... */
-}
-
 /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at
  * <b>needle</b>, return a pointer to the first occurrence of the needle
  * within the haystack, or NULL if there is no such occurrence.
@@ -591,67 +443,6 @@ tor_memmem(const void *_haystack, size_t hlen,
 #endif /* defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2) */
 }
 
-/**
- * Tables to implement ctypes-replacement TOR_IS*() functions.  Each table
- * has 256 bits to look up whether a character is in some set or not.  This
- * fails on non-ASCII platforms, but it is hard to find a platform whose
- * character set is not a superset of ASCII nowadays. */
-
-/**@{*/
-const uint32_t TOR_ISALPHA_TABLE[8] =
-  { 0, 0, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
-const uint32_t TOR_ISALNUM_TABLE[8] =
-  { 0, 0x3ff0000, 0x7fffffe, 0x7fffffe, 0, 0, 0, 0 };
-const uint32_t TOR_ISSPACE_TABLE[8] = { 0x3e00, 0x1, 0, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISXDIGIT_TABLE[8] =
-  { 0, 0x3ff0000, 0x7e, 0x7e, 0, 0, 0, 0 };
-const uint32_t TOR_ISDIGIT_TABLE[8] = { 0, 0x3ff0000, 0, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISPRINT_TABLE[8] =
-  { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 };
-const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 };
-const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 };
-
-/** Upper-casing and lowercasing tables to map characters to upper/lowercase
- * equivalents.  Used by tor_toupper() and tor_tolower(). */
-/**@{*/
-const uint8_t TOR_TOUPPER_TABLE[256] = {
-  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
-  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
-  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
-  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
-  64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
-  80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,
-  96,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,
-  80,81,82,83,84,85,86,87,88,89,90,123,124,125,126,127,
-  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
-  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
-  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
-  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
-  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
-  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
-  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
-  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
-};
-const uint8_t TOR_TOLOWER_TABLE[256] = {
-  0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,
-  16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,
-  32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,
-  48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,
-  64,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
-  112,113,114,115,116,117,118,119,120,121,122,91,92,93,94,95,
-  96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,
-  112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,
-  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
-  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,
-  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,
-  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,
-  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,
-  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,
-  224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,
-  240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,
-};
-/**@}*/
-
 /** Helper for tor_strtok_r_impl: Advances cp past all characters in
  * <b>sep</b>, and returns its new value. */
 static char *
@@ -702,32 +493,6 @@ tor_strtok_r_impl(char *str, const char *sep, char **lasts)
   return start;
 }
 
-#ifdef _WIN32
-/** Take a filename and return a pointer to its final element.  This
- * function is called on __FILE__ to fix a MSVC nit where __FILE__
- * contains the full path to the file.  This is bad, because it
- * confuses users to find the home directory of the person who
- * compiled the binary in their warning messages.
- */
-const char *
-tor_fix_source_file(const char *fname)
-{
-  const char *cp1, *cp2, *r;
-  cp1 = strrchr(fname, '/');
-  cp2 = strrchr(fname, '\\');
-  if (cp1 && cp2) {
-    r = (cp1<cp2)?(cp2+1):(cp1+1);
-  } else if (cp1) {
-    r = cp1+1;
-  } else if (cp2) {
-    r = cp2+1;
-  } else {
-    r = fname;
-  }
-  return r;
-}
-#endif /* defined(_WIN32) */
-
 /**
  * Read a 16-bit value beginning at <b>cp</b>.  Equivalent to
  * *(uint16_t*)(cp), but will not cause segfaults on platforms that forbid
@@ -942,80 +707,6 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile)
   tor_free(lockfile);
 }
 
-/** @{ */
-/** Some old versions of Unix didn't define constants for these values,
- * and instead expect you to say 0, 1, or 2. */
-#ifndef SEEK_SET
-#define SEEK_SET 0
-#endif
-#ifndef SEEK_CUR
-#define SEEK_CUR 1
-#endif
-#ifndef SEEK_END
-#define SEEK_END 2
-#endif
-/** @} */
-
-/** Return the position of <b>fd</b> with respect to the start of the file. */
-off_t
-tor_fd_getpos(int fd)
-{
-#ifdef _WIN32
-  return (off_t) _lseek(fd, 0, SEEK_CUR);
-#else
-  return (off_t) lseek(fd, 0, SEEK_CUR);
-#endif
-}
-
-/** Move <b>fd</b> to the end of the file. Return -1 on error, 0 on success.
- * If the file is a pipe, do nothing and succeed.
- **/
-int
-tor_fd_seekend(int fd)
-{
-#ifdef _WIN32
-  return _lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
-#else
-  off_t rc = lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
-#ifdef ESPIPE
-  /* If we get an error and ESPIPE, then it's a pipe or a socket of a fifo:
-   * no need to worry. */
-  if (rc < 0 && errno == ESPIPE)
-    rc = 0;
-#endif /* defined(ESPIPE) */
-  return (rc < 0) ? -1 : 0;
-#endif /* defined(_WIN32) */
-}
-
-/** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0
- * on success. */
-int
-tor_fd_setpos(int fd, off_t pos)
-{
-#ifdef _WIN32
-  return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
-#else
-  return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
-#endif
-}
-
-/** Replacement for ftruncate(fd, 0): move to the front of the file and remove
- * all the rest of the file. Return -1 on error, 0 on success. */
-int
-tor_ftruncate(int fd)
-{
-  /* Rumor has it that some versions of ftruncate do not move the file pointer.
-   */
-  if (tor_fd_setpos(fd, 0) < 0)
-    return -1;
-
-#ifdef _WIN32
-  return _chsize(fd, 0);
-#else
-  return ftruncate(fd, 0);
-#endif
-}
-
 #undef DEBUG_SOCKET_COUNTING
 #ifdef DEBUG_SOCKET_COUNTING
 /** A bitarray of all fds that should be passed to tor_socket_close(). Only
@@ -2850,183 +2541,39 @@ compute_num_cpus(void)
   return num_cpus;
 }
 
-#if !defined(_WIN32)
-/** Defined iff we need to add locks when defining fake versions of reentrant
- * versions of time-related functions. */
-#define TIME_FNS_NEED_LOCKS
-#endif
-
-/** Helper: Deal with confused or out-of-bounds values from localtime_r and
- * friends.  (On some platforms, they can give out-of-bounds values or can
- * return NULL.)  If <b>islocal</b>, this is a localtime result; otherwise
- * it's from gmtime.  The function returns <b>r</b>, when given <b>timep</b>
- * as its input. If we need to store new results, store them in
- * <b>resultbuf</b>. */
-static struct tm *
-correct_tm(int islocal, const time_t *timep, struct tm *resultbuf,
-           struct tm *r)
-{
-  const char *outcome;
-
-  if (PREDICT_LIKELY(r)) {
-    /* We can't strftime dates after 9999 CE, and we want to avoid dates
-     * before 1 CE (avoiding the year 0 issue and negative years). */
-    if (r->tm_year > 8099) {
-      r->tm_year = 8099;
-      r->tm_mon = 11;
-      r->tm_mday = 31;
-      r->tm_yday = 364;
-      r->tm_wday = 6;
-      r->tm_hour = 23;
-      r->tm_min = 59;
-      r->tm_sec = 59;
-    } else if (r->tm_year < (1-1900)) {
-      r->tm_year = (1-1900);
-      r->tm_mon = 0;
-      r->tm_mday = 1;
-      r->tm_yday = 0;
-      r->tm_wday = 0;
-      r->tm_hour = 0;
-      r->tm_min = 0;
-      r->tm_sec = 0;
-    }
-    return r;
-  }
-
-  /* If we get here, gmtime or localtime returned NULL. It might have done
-   * this because of overrun or underrun, or it might have done it because of
-   * some other weird issue. */
-  if (timep) {
-    if (*timep < 0) {
-      r = resultbuf;
-      r->tm_year = 70; /* 1970 CE */
-      r->tm_mon = 0;
-      r->tm_mday = 1;
-      r->tm_yday = 0;
-      r->tm_wday = 0;
-      r->tm_hour = 0;
-      r->tm_min = 0 ;
-      r->tm_sec = 0;
-      outcome = "Rounding up to 1970";
-      goto done;
-    } else if (*timep >= INT32_MAX) {
-      /* Rounding down to INT32_MAX isn't so great, but keep in mind that we
-       * only do it if gmtime/localtime tells us NULL. */
-      r = resultbuf;
-      r->tm_year = 137; /* 2037 CE */
-      r->tm_mon = 11;
-      r->tm_mday = 31;
-      r->tm_yday = 364;
-      r->tm_wday = 6;
-      r->tm_hour = 23;
-      r->tm_min = 59;
-      r->tm_sec = 59;
-      outcome = "Rounding down to 2037";
-      goto done;
-    }
-  }
-
-  /* If we get here, then gmtime/localtime failed without getting an extreme
-   * value for *timep */
-  /* LCOV_EXCL_START */
-  tor_fragile_assert();
-  r = resultbuf;
-  memset(resultbuf, 0, sizeof(struct tm));
-  outcome="can't recover";
-  /* LCOV_EXCL_STOP */
- done:
-  log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s",
-           islocal?"localtime":"gmtime",
-           timep?I64_PRINTF_ARG(*timep):0,
-           strerror(errno),
-           outcome);
-  return r;
-}
-
-/** @{ */
 /** As localtime_r, but defined for platforms that don't have it:
  *
  * Convert *<b>timep</b> to a struct tm in local time, and store the value in
  * *<b>result</b>.  Return the result on success, or NULL on failure.
  */
-#ifdef HAVE_LOCALTIME_R
-struct tm *
-tor_localtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  r = localtime_r(timep, result);
-  return correct_tm(1, timep, result, r);
-}
-#elif defined(TIME_FNS_NEED_LOCKS)
-struct tm *
-tor_localtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  static tor_mutex_t *m=NULL;
-  if (!m) { m=tor_mutex_new(); }
-  tor_assert(result);
-  tor_mutex_acquire(m);
-  r = localtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  tor_mutex_release(m);
-  return correct_tm(1, timep, result, r);
-}
-#else
 struct tm *
 tor_localtime_r(const time_t *timep, struct tm *result)
 {
-  struct tm *r;
-  tor_assert(result);
-  r = localtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  return correct_tm(1, timep, result, r);
+  char *err = NULL;
+  struct tm *r = tor_localtime_r_msg(timep, result, &err);
+  if (err) {
+    log_warn(LD_BUG, "%s", err);
+    tor_free(err);
+  }
+  return r;
 }
-#endif /* defined(HAVE_LOCALTIME_R) || ... */
-/** @} */
 
-/** @{ */
 /** As gmtime_r, but defined for platforms that don't have it:
  *
  * Convert *<b>timep</b> to a struct tm in UTC, and store the value in
  * *<b>result</b>.  Return the result on success, or NULL on failure.
  */
-#ifdef HAVE_GMTIME_R
-struct tm *
-tor_gmtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  r = gmtime_r(timep, result);
-  return correct_tm(0, timep, result, r);
-}
-#elif defined(TIME_FNS_NEED_LOCKS)
 struct tm *
 tor_gmtime_r(const time_t *timep, struct tm *result)
 {
-  struct tm *r;
-  static tor_mutex_t *m=NULL;
-  if (!m) { m=tor_mutex_new(); }
-  tor_assert(result);
-  tor_mutex_acquire(m);
-  r = gmtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  tor_mutex_release(m);
-  return correct_tm(0, timep, result, r);
-}
-#else
-struct tm *
-tor_gmtime_r(const time_t *timep, struct tm *result)
-{
-  struct tm *r;
-  tor_assert(result);
-  r = gmtime(timep);
-  if (r)
-    memcpy(result, r, sizeof(struct tm));
-  return correct_tm(0, timep, result, r);
+  char *err = NULL;
+  struct tm *r = tor_gmtime_r_msg(timep, result, &err);
+  if (err) {
+    log_warn(LD_BUG, "%s", err);
+    tor_free(err);
+  }
+  return r;
 }
-#endif /* defined(HAVE_GMTIME_R) || ... */
 
 #if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
 #define HAVE_UNIX_MLOCKALL
@@ -3527,4 +3074,3 @@ tor_get_avail_disk_space(const char *path)
   return -1;
 #endif /* defined(HAVE_STATVFS) || ... */
 }
-

src/common/compat.h

@@ -47,37 +47,15 @@
 
 #include "lib/cc/compat_compiler.h"
 #include "common/compat_time.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
 
 #include <stdio.h>
 #include <errno.h>
 
 /* ===== Compiler compatibility */
 
-/* ===== String compatibility */
-#ifdef _WIN32
-/* Windows names string functions differently from most other platforms. */
-#define strncasecmp _strnicmp
-#define strcasecmp _stricmp
-#endif
-
-#if defined __APPLE__
-/* On OSX 10.9 and later, the overlap-checking code for strlcat would
- * appear to have a severe bug that can sometimes cause aborts in Tor.
- * Instead, use the non-checking variants.  This is sad.
- *
- * See https://trac.torproject.org/projects/tor/ticket/15205
- */
-#undef strlcat
-#undef strlcpy
-#endif /* defined __APPLE__ */
-
-#ifndef HAVE_STRLCAT
-size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
-#endif
-#ifndef HAVE_STRLCPY
-size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
-#endif
-
 /** Represents an mmaped file. Allocated via tor_mmap_file; freed with
  * tor_munmap_file. */
 typedef struct tor_mmap_t {
@@ -97,16 +75,6 @@ typedef struct tor_mmap_t {
 tor_mmap_t *tor_mmap_file(const char *filename) ATTR_NONNULL((1));
 int tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1));
 
-int tor_snprintf(char *str, size_t size, const char *format, ...)
-  CHECK_PRINTF(3,4) ATTR_NONNULL((1,3));
-int tor_vsnprintf(char *str, size_t size, const char *format, va_list args)
-  CHECK_PRINTF(3,0) ATTR_NONNULL((1,3));
-
-int tor_asprintf(char **strp, const char *fmt, ...)
-  CHECK_PRINTF(2,3);
-int tor_vasprintf(char **strp, const char *fmt, va_list args)
-  CHECK_PRINTF(2,0);
-
 const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen) ATTR_NONNULL((1,3));
 static const void *tor_memstr(const void *haystack, size_t hlen,
@@ -117,28 +85,6 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle)
   return tor_memmem(haystack, hlen, needle, strlen(needle));
 }
 
-/* Much of the time when we're checking ctypes, we're doing spec compliance,
- * which all assumes we're doing ASCII. */
-#define DECLARE_CTYPE_FN(name)                                          \
-  static int TOR_##name(char c);                                        \
-  extern const uint32_t TOR_##name##_TABLE[];                           \
-  static inline int TOR_##name(char c) {                                \
-    uint8_t u = c;                                                      \
-    return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1u << (u & 31)));     \
-  }
-DECLARE_CTYPE_FN(ISALPHA)
-DECLARE_CTYPE_FN(ISALNUM)
-DECLARE_CTYPE_FN(ISSPACE)
-DECLARE_CTYPE_FN(ISDIGIT)
-DECLARE_CTYPE_FN(ISXDIGIT)
-DECLARE_CTYPE_FN(ISPRINT)
-DECLARE_CTYPE_FN(ISLOWER)
-DECLARE_CTYPE_FN(ISUPPER)
-extern const uint8_t TOR_TOUPPER_TABLE[];
-extern const uint8_t TOR_TOLOWER_TABLE[];
-#define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c])
-#define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c])
-
 char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #ifdef HAVE_STRTOK_R
 #define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts)
@@ -146,14 +92,6 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
 #endif
 
-#ifdef _WIN32
-#define SHORT_FILE__ (tor_fix_source_file(__FILE__))
-const char *tor_fix_source_file(const char *fname);
-#else
-#define SHORT_FILE__ (__FILE__)
-#define tor_fix_source_file(s) (s)
-#endif /* defined(_WIN32) */
-
 /* ===== Time compatibility */
 
 struct tm *tor_localtime_r(const time_t *timep, struct tm *result);
@@ -214,11 +152,6 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-off_t tor_fd_getpos(int fd);
-int tor_fd_setpos(int fd, off_t pos);
-int tor_fd_seekend(int fd);
-int tor_ftruncate(int fd);
-
 int64_t tor_get_avail_disk_space(const char *path);
 
 #ifdef _WIN32

src/common/compat_libevent.c

@@ -14,7 +14,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 #include <event2/event.h>
 #include <event2/thread.h>

src/common/compat_pthreads.c

@@ -16,7 +16,7 @@
 #include <time.h>
 
 #include "common/compat.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 
 /** Wraps a void (*)(void*) function and its argument so we can
@@ -91,83 +91,6 @@ spawn_exit(void)
   pthread_exit(NULL);
 }
 
-/** A mutex attribute that we're going to use to tell pthreads that we want
- * "recursive" mutexes (i.e., once we can re-lock if we're already holding
- * them.) */
-static pthread_mutexattr_t attr_recursive;
-
-/** Initialize <b>mutex</b> so it can be locked.  Every mutex must be set
- * up with tor_mutex_init() or tor_mutex_new(); not both. */
-void
-tor_mutex_init(tor_mutex_t *mutex)
-{
-  if (PREDICT_UNLIKELY(!threads_initialized))
-    tor_threads_init(); // LCOV_EXCL_LINE
-  const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    raw_assert_unreached_msg("Error creating a mutex.");
-    // LCOV_EXCL_STOP
-  }
-}
-
-/** As tor_mutex_init, but initialize a mutex suitable that may be
- * non-recursive, if the OS supports that. */
-void
-tor_mutex_init_nonrecursive(tor_mutex_t *mutex)
-{
-  int err;
-  if (!threads_initialized)
-    tor_threads_init(); // LCOV_EXCL_LINE
-  err = pthread_mutex_init(&mutex->mutex, NULL);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    raw_assert_unreached_msg("Error creating a mutex.");
-    // LCOV_EXCL_STOP
-  }
-}
-
-/** Wait until <b>m</b> is free, then acquire it. */
-void
-tor_mutex_acquire(tor_mutex_t *m)
-{
-  int err;
-  raw_assert(m);
-  err = pthread_mutex_lock(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    raw_assert_unreached_msg("Error locking a mutex.");
-    // LCOV_EXCL_STOP
-  }
-}
-/** Release the lock <b>m</b> so another thread can have it. */
-void
-tor_mutex_release(tor_mutex_t *m)
-{
-  int err;
-  raw_assert(m);
-  err = pthread_mutex_unlock(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    raw_assert_unreached_msg("Error unlocking a mutex.");
-    // LCOV_EXCL_STOP
-  }
-}
-/** Clean up the mutex <b>m</b> so that it no longer uses any system
- * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
-void
-tor_mutex_uninit(tor_mutex_t *m)
-{
-  int err;
-  raw_assert(m);
-  err = pthread_mutex_destroy(&m->mutex);
-  if (PREDICT_UNLIKELY(err)) {
-    // LCOV_EXCL_START
-    raw_assert_unreached_msg("Error destroying a mutex.");
-    // LCOV_EXCL_STOP
-  }
-}
 /** Return an integer representing this thread. */
 unsigned long
 tor_get_thread_id(void)
@@ -328,8 +251,7 @@ void
 tor_threads_init(void)
 {
   if (!threads_initialized) {
-    pthread_mutexattr_init(&attr_recursive);
-    pthread_mutexattr_settype(&attr_recursive, PTHREAD_MUTEX_RECURSIVE);
+    tor_locking_init();
     const int ret1 = pthread_attr_init(&attr_detached);
     tor_assert(ret1 == 0);
 #ifndef PTHREAD_CREATE_DETACHED

src/common/compat_threads.c

@@ -17,7 +17,7 @@
 #include "common/compat_threads.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 #ifdef HAVE_SYS_EVENTFD_H
 #include <sys/eventfd.h>
@@ -29,33 +29,6 @@
 #include <unistd.h>
 #endif
 
-/** Return a newly allocated, ready-for-use mutex. */
-tor_mutex_t *
-tor_mutex_new(void)
-{
-  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
-  tor_mutex_init(m);
-  return m;
-}
-/** Return a newly allocated, ready-for-use mutex.  This one might be
- * non-recursive, if that's faster. */
-tor_mutex_t *
-tor_mutex_new_nonrecursive(void)
-{
-  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
-  tor_mutex_init_nonrecursive(m);
-  return m;
-}
-/** Release all storage and system resources held by <b>m</b>. */
-void
-tor_mutex_free_(tor_mutex_t *m)
-{
-  if (!m)
-    return;
-  tor_mutex_uninit(m);
-  tor_free(m);
-}
-
 /** Allocate and return a new condition variable. */
 tor_cond_t *
 tor_cond_new(void)
@@ -404,4 +377,3 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return oldval;
 }
 #endif /* !defined(HAVE_STDATOMIC_H) */
-

src/common/compat_threads.h

@@ -9,54 +9,15 @@
 #include "orconfig.h"
 #include "lib/cc/torint.h"
 #include "lib/testsupport/testsupport.h"
-
-#if defined(HAVE_PTHREAD_H) && !defined(_WIN32)
-#include <pthread.h>
-#endif
+#include "lib/lock/compat_mutex.h"
 
 #ifdef HAVE_STDATOMIC_H
 #include <stdatomic.h>
 #endif
 
-#if defined(_WIN32)
-#define USE_WIN32_THREADS
-#elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE)
-#define USE_PTHREADS
-#else
-#error "No threading system was found"
-#endif /* defined(_WIN32) || ... */
-
 int spawn_func(void (*func)(void *), void *data);
 void spawn_exit(void) ATTR_NORETURN;
 
-/* Because we use threads instead of processes on most platforms (Windows,
- * Linux, etc), we need locking for them.  On platforms with poor thread
- * support or broken gethostbyname_r, these functions are no-ops. */
-
-/** A generic lock structure for multithreaded builds. */
-typedef struct tor_mutex_t {
-#if defined(USE_WIN32_THREADS)
-  /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */
-  CRITICAL_SECTION mutex;
-#elif defined(USE_PTHREADS)
-  /** Pthreads-only: with pthreads, we implement locks with
-   * pthread_mutex_t. */
-  pthread_mutex_t mutex;
-#else
-  /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
-  int _unused;
-#endif /* defined(USE_WIN32_THREADS) || ... */
-} tor_mutex_t;
-
-tor_mutex_t *tor_mutex_new(void);
-tor_mutex_t *tor_mutex_new_nonrecursive(void);
-void tor_mutex_init(tor_mutex_t *m);
-void tor_mutex_init_nonrecursive(tor_mutex_t *m);
-void tor_mutex_acquire(tor_mutex_t *m);
-void tor_mutex_release(tor_mutex_t *m);
-void tor_mutex_free_(tor_mutex_t *m);
-#define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
-void tor_mutex_uninit(tor_mutex_t *m);
 unsigned long tor_get_thread_id(void);
 void tor_threads_init(void);
 
@@ -220,4 +181,3 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
 #endif /* defined(HAVE_STDATOMIC_H) */
 
 #endif /* !defined(TOR_COMPAT_THREADS_H) */
-

src/common/compat_time.c

@@ -35,15 +35,8 @@
 #endif
 
 #include "lib/err/torerr.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
-#include "common/container.h"
-
-#ifndef HAVE_GETTIMEOFDAY
-#ifdef HAVE_FTIME
-#include <sys/timeb.h>
-#endif
-#endif
 
 #ifdef _WIN32
 #undef HAVE_CLOCK_GETTIME
@@ -69,53 +62,6 @@ tor_sleep_msec(int msec)
 }
 #endif /* defined(TOR_UNIT_TESTS) */
 
-/** Set *timeval to the current time of day.  On error, log and terminate.
- * (Same as gettimeofday(timeval,NULL), but never returns -1.)
- */
-MOCK_IMPL(void,
-tor_gettimeofday, (struct timeval *timeval))
-{
-#ifdef _WIN32
-  /* Epoch bias copied from perl: number of units between windows epoch and
-   * Unix epoch. */
-#define EPOCH_BIAS U64_LITERAL(116444736000000000)
-#define UNITS_PER_SEC U64_LITERAL(10000000)
-#define USEC_PER_SEC U64_LITERAL(1000000)
-#define UNITS_PER_USEC U64_LITERAL(10)
-  union {
-    uint64_t ft_64;
-    FILETIME ft_ft;
-  } ft;
-  /* number of 100-nsec units since Jan 1, 1601 */
-  GetSystemTimeAsFileTime(&ft.ft_ft);
-  if (ft.ft_64 < EPOCH_BIAS) {
-    /* LCOV_EXCL_START */
-    log_err(LD_GENERAL,"System time is before 1970; failing.");
-    exit(1); // exit ok: system clock is broken.
-    /* LCOV_EXCL_STOP */
-  }
-  ft.ft_64 -= EPOCH_BIAS;
-  timeval->tv_sec = (unsigned) (ft.ft_64 / UNITS_PER_SEC);
-  timeval->tv_usec = (unsigned) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC);
-#elif defined(HAVE_GETTIMEOFDAY)
-  if (gettimeofday(timeval, NULL)) {
-    /* LCOV_EXCL_START */
-    /* If gettimeofday dies, we have either given a bad timezone (we didn't),
-       or segfaulted.*/
-    raw_assert_unreached_msg("gettimeofday failed");
-    /* LCOV_EXCL_STOP */
-  }
-#elif defined(HAVE_FTIME)
-  struct timeb tb;
-  ftime(&tb);
-  timeval->tv_sec = tb.time;
-  timeval->tv_usec = tb.millitm * 1000;
-#else
-#error "No way to get time."
-#endif /* defined(_WIN32) || ... */
-  return;
-}
-
 #define ONE_MILLION ((int64_t) (1000 * 1000))
 #define ONE_BILLION ((int64_t) (1000 * 1000 * 1000))
 

src/common/compat_time.h

@@ -19,6 +19,8 @@
 #define TOR_COMPAT_TIME_H
 
 #include "orconfig.h"
+#include "lib/wallclock/tor_gettimeofday.h"
+
 #ifdef _WIN32
 #undef HAVE_CLOCK_GETTIME
 #endif
@@ -200,8 +202,6 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start,
 #endif
 }
 
-MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
-
 #ifdef TOR_UNIT_TESTS
 void tor_sleep_msec(int msec);
 
@@ -230,4 +230,3 @@ void monotime_reset_ratchets_for_testing(void);
 #endif /* defined(COMPAT_TIME_PRIVATE) */
 
 #endif /* !defined(TOR_COMPAT_TIME_H) */
-

src/common/compat_winthreads.c

@@ -16,8 +16,7 @@
 #include <windows.h>
 #include <process.h>
 #include "common/util.h"
-#include "common/container.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 /* This value is more or less total cargo-cult */
 #define SPIN_COUNT 2000
@@ -55,33 +54,6 @@ spawn_exit(void)
   // LCOV_EXCL_STOP
 }
 
-void
-tor_mutex_init(tor_mutex_t *m)
-{
-  InitializeCriticalSection(&m->mutex);
-}
-void
-tor_mutex_init_nonrecursive(tor_mutex_t *m)
-{
-  InitializeCriticalSection(&m->mutex);
-}
-
-void
-tor_mutex_uninit(tor_mutex_t *m)
-{
-  DeleteCriticalSection(&m->mutex);
-}
-void
-tor_mutex_acquire(tor_mutex_t *m)
-{
-  raw_assert(m);
-  EnterCriticalSection(&m->mutex);
-}
-void
-tor_mutex_release(tor_mutex_t *m)
-{
-  LeaveCriticalSection(&m->mutex);
-}
 unsigned long
 tor_get_thread_id(void)
 {

src/common/confline.c

@@ -6,9 +6,9 @@
 
 #include "common/compat.h"
 #include "common/confline.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
-#include "common/container.h"
+#include "lib/container/smartlist.h"
 
 static int config_get_lines_aux(const char *string, config_line_t **result,
                                 int extended, int allow_include,
@@ -535,4 +535,3 @@ parse_config_line_from_str_verbose(const char *line, char **key_out,
 
   return line;
 }
-

src/common/confline.h

@@ -7,7 +7,7 @@
 #ifndef TOR_CONFLINE_H
 #define TOR_CONFLINE_H
 
-#include "common/container.h"
+struct smartlist_t;
 
 /** Ordinary configuration line. */
 #define CONFIG_LINE_NORMAL 0
@@ -47,7 +47,7 @@ int config_count_key(const config_line_t *a, const char *key);
 int config_get_lines(const char *string, config_line_t **result, int extended);
 int config_get_lines_include(const char *string, config_line_t **result,
                              int extended, int *has_include,
-                             smartlist_t *opened_lst);
+                             struct smartlist_t *opened_lst);
 void config_free_lines_(config_line_t *front);
 #define config_free_lines(front) \
   do {                           \
@@ -58,4 +58,3 @@ const char *parse_config_line_from_str_verbose(const char *line,
                                        char **key_out, char **value_out,
                                        const char **err_out);
 #endif /* !defined(TOR_CONFLINE_H) */
-

src/common/container.h

@@ -1,742 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#ifndef TOR_CONTAINER_H
-#define TOR_CONTAINER_H
-
-#include "common/util.h"
-#include "siphash.h"
-
-/** A resizeable list of pointers, with associated helpful functionality.
- *
- * The members of this struct are exposed only so that macros and inlines can
- * use them; all access to smartlist internals should go through the functions
- * and macros defined here.
- **/
-typedef struct smartlist_t {
-  /** @{ */
-  /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements
-   * before it needs to be resized.  Only the first <b>num_used</b> (\<=
-   * capacity) elements point to valid data.
-   */
-  void **list;
-  int num_used;
-  int capacity;
-  /** @} */
-} smartlist_t;
-
-MOCK_DECL(smartlist_t *, smartlist_new, (void));
-MOCK_DECL(void, smartlist_free_, (smartlist_t *sl));
-#define smartlist_free(sl) FREE_AND_NULL(smartlist_t, smartlist_free_, (sl))
-
-void smartlist_clear(smartlist_t *sl);
-void smartlist_add(smartlist_t *sl, void *element);
-void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
-void smartlist_remove(smartlist_t *sl, const void *element);
-void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
-void *smartlist_pop_last(smartlist_t *sl);
-void smartlist_reverse(smartlist_t *sl);
-void smartlist_string_remove(smartlist_t *sl, const char *element);
-int smartlist_contains(const smartlist_t *sl, const void *element);
-int smartlist_contains_string(const smartlist_t *sl, const char *element);
-int smartlist_pos(const smartlist_t *sl, const void *element);
-int smartlist_string_pos(const smartlist_t *, const char *elt);
-int smartlist_contains_string_case(const smartlist_t *sl, const char *element);
-int smartlist_contains_int_as_string(const smartlist_t *sl, int num);
-int smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2);
-int smartlist_contains_digest(const smartlist_t *sl, const char *element);
-int smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2);
-int smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2);
-void smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2);
-void smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2);
-
-/* smartlist_choose() is defined in crypto.[ch] */
-#ifdef DEBUG_SMARTLIST
-/** Return the number of items in sl.
- */
-static inline int smartlist_len(const smartlist_t *sl);
-static inline int smartlist_len(const smartlist_t *sl) {
-  tor_assert(sl);
-  return (sl)->num_used;
-}
-/** Return the <b>idx</b>th element of sl.
- */
-static inline void *smartlist_get(const smartlist_t *sl, int idx);
-static inline void *smartlist_get(const smartlist_t *sl, int idx) {
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(sl->num_used > idx);
-  return sl->list[idx];
-}
-static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
-  tor_assert(sl);
-  tor_assert(idx>=0);
-  tor_assert(sl->num_used > idx);
-  sl->list[idx] = val;
-}
-#else /* !(defined(DEBUG_SMARTLIST)) */
-#define smartlist_len(sl) ((sl)->num_used)
-#define smartlist_get(sl, idx) ((sl)->list[idx])
-#define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
-#endif /* defined(DEBUG_SMARTLIST) */
-
-/** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the
- * smartlist <b>sl</b>. */
-static inline void smartlist_swap(smartlist_t *sl, int idx1, int idx2)
-{
-  if (idx1 != idx2) {
-    void *elt = smartlist_get(sl, idx1);
-    smartlist_set(sl, idx1, smartlist_get(sl, idx2));
-    smartlist_set(sl, idx2, elt);
-  }
-}
-
-void smartlist_del(smartlist_t *sl, int idx);
-void smartlist_del_keeporder(smartlist_t *sl, int idx);
-void smartlist_insert(smartlist_t *sl, int idx, void *val);
-void smartlist_sort(smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b));
-void *smartlist_get_most_frequent_(const smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b),
-                    int *count_out);
-#define smartlist_get_most_frequent(sl, compare) \
-  smartlist_get_most_frequent_((sl), (compare), NULL)
-void smartlist_uniq(smartlist_t *sl,
-                    int (*compare)(const void **a, const void **b),
-                    void (*free_fn)(void *elt));
-
-void smartlist_sort_strings(smartlist_t *sl);
-void smartlist_sort_digests(smartlist_t *sl);
-void smartlist_sort_digests256(smartlist_t *sl);
-void smartlist_sort_pointers(smartlist_t *sl);
-
-const char *smartlist_get_most_frequent_string(smartlist_t *sl);
-const char *smartlist_get_most_frequent_string_(smartlist_t *sl,
-                                                int *count_out);
-const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl);
-
-void smartlist_uniq_strings(smartlist_t *sl);
-void smartlist_uniq_digests(smartlist_t *sl);
-void smartlist_uniq_digests256(smartlist_t *sl);
-void *smartlist_bsearch(smartlist_t *sl, const void *key,
-                        int (*compare)(const void *key, const void **member));
-int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
-                          int (*compare)(const void *key, const void **member),
-                          int *found_out);
-
-void smartlist_pqueue_add(smartlist_t *sl,
-                          int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
-                          void *item);
-void *smartlist_pqueue_pop(smartlist_t *sl,
-                           int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
-void smartlist_pqueue_remove(smartlist_t *sl,
-                             int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
-                             void *item);
-void smartlist_pqueue_assert_ok(smartlist_t *sl,
-                                int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
-
-#define SPLIT_SKIP_SPACE   0x01
-#define SPLIT_IGNORE_BLANK 0x02
-#define SPLIT_STRIP_SPACE  0x04
-int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
-                           int flags, int max);
-char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
-                             size_t *len_out) ATTR_MALLOC;
-char *smartlist_join_strings2(smartlist_t *sl, const char *join,
-                              size_t join_len, int terminate, size_t *len_out)
-  ATTR_MALLOC;
-
-/** Iterate over the items in a smartlist <b>sl</b>, in order.  For each item,
- * assign it to a new local variable of type <b>type</b> named <b>var</b>, and
- * execute the statements inside the loop body.  Inside the loop, the loop
- * index can be accessed as <b>var</b>_sl_idx and the length of the list can
- * be accessed as <b>var</b>_sl_len.
- *
- * NOTE: Do not change the length of the list while the loop is in progress,
- * unless you adjust the _sl_len variable correspondingly.  See second example
- * below.
- *
- * Example use:
- * <pre>
- *   smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
- *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
- *     printf("%d: %s\n", cp_sl_idx, cp);
- *     tor_free(cp);
- *   } SMARTLIST_FOREACH_END(cp);
- *   smartlist_free(list);
- * </pre>
- *
- * Example use (advanced):
- * <pre>
- *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
- *     if (!strcmp(cp, "junk")) {
- *       tor_free(cp);
- *       SMARTLIST_DEL_CURRENT(list, cp);
- *     }
- *   } SMARTLIST_FOREACH_END(cp);
- * </pre>
- */
-/* Note: these macros use token pasting, and reach into smartlist internals.
- * This can make them a little daunting. Here's the approximate unpacking of
- * the above examples, for entertainment value:
- *
- * <pre>
- * smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
- * {
- *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
- *   char *cp;
- *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
- *     cp = smartlist_get(list, cp_sl_idx);
- *     printf("%d: %s\n", cp_sl_idx, cp);
- *     tor_free(cp);
- *   }
- * }
- * smartlist_free(list);
- * </pre>
- *
- * <pre>
- * {
- *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
- *   char *cp;
- *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
- *     cp = smartlist_get(list, cp_sl_idx);
- *     if (!strcmp(cp, "junk")) {
- *       tor_free(cp);
- *       smartlist_del(list, cp_sl_idx);
- *       --cp_sl_idx;
- *       --cp_sl_len;
- *     }
- *   }
- * }
- * </pre>
- */
-#define SMARTLIST_FOREACH_BEGIN(sl, type, var)  \
-  STMT_BEGIN                                                    \
-    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;          \
-    type var;                                                   \
-    for (var ## _sl_idx = 0; var ## _sl_idx < var ## _sl_len;   \
-         ++var ## _sl_idx) {                                    \
-      var = (sl)->list[var ## _sl_idx];
-
-#define SMARTLIST_FOREACH_END(var)              \
-    var = NULL;                                 \
-    (void) var ## _sl_idx;                      \
-  } STMT_END
-
-/**
- * An alias for SMARTLIST_FOREACH_BEGIN and SMARTLIST_FOREACH_END, using
- * <b>cmd</b> as the loop body.  This wrapper is here for convenience with
- * very short loops.
- *
- * By convention, we do not use this for loops which nest, or for loops over
- * 10 lines or so.  Use SMARTLIST_FOREACH_{BEGIN,END} for those.
- */
-#define SMARTLIST_FOREACH(sl, type, var, cmd)                   \
-  SMARTLIST_FOREACH_BEGIN(sl,type,var) {                        \
-    cmd;                                                        \
-  } SMARTLIST_FOREACH_END(var)
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, remove the current element in a way that
- * won't confuse the loop. */
-#define SMARTLIST_DEL_CURRENT(sl, var)          \
-  STMT_BEGIN                                    \
-    smartlist_del(sl, var ## _sl_idx);          \
-    --var ## _sl_idx;                           \
-    --var ## _sl_len;                           \
-  STMT_END
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, remove the current element in a way that
- * won't confuse the loop. */
-#define SMARTLIST_DEL_CURRENT_KEEPORDER(sl, var)          \
-  STMT_BEGIN                                              \
-     smartlist_del_keeporder(sl, var ## _sl_idx);         \
-     --var ## _sl_idx;                                    \
-     --var ## _sl_len;                                    \
-  STMT_END
-
-/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
- * with the variable <b>var</b>, replace the current element with <b>val</b>.
- * Does not deallocate the current value of <b>var</b>.
- */
-#define SMARTLIST_REPLACE_CURRENT(sl, var, val) \
-  STMT_BEGIN                                    \
-    smartlist_set(sl, var ## _sl_idx, val);     \
-  STMT_END
-
-/* Helper: Given two lists of items, possibly of different types, such that
- * both lists are sorted on some common field (as determined by a comparison
- * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
- * duplicates on the common field, loop through the lists in lockstep, and
- * execute <b>unmatched_var2</b> on items in var2 that do not appear in
- * var1.
- *
- * WARNING: It isn't safe to add remove elements from either list while the
- * loop is in progress.
- *
- * Example use:
- *  SMARTLIST_FOREACH_JOIN(routerstatus_list, routerstatus_t *, rs,
- *                     routerinfo_list, routerinfo_t *, ri,
- *                    tor_memcmp(rs->identity_digest, ri->identity_digest, 20),
- *                     log_info(LD_GENERAL,"No match for %s", ri->nickname)) {
- *    log_info(LD_GENERAL, "%s matches routerstatus %p", ri->nickname, rs);
- * } SMARTLIST_FOREACH_JOIN_END(rs, ri);
- **/
-/* The example above unpacks (approximately) to:
- *  int rs_sl_idx = 0, rs_sl_len = smartlist_len(routerstatus_list);
- *  int ri_sl_idx, ri_sl_len = smartlist_len(routerinfo_list);
- *  int rs_ri_cmp;
- *  routerstatus_t *rs;
- *  routerinfo_t *ri;
- *  for (; ri_sl_idx < ri_sl_len; ++ri_sl_idx) {
- *    ri = smartlist_get(routerinfo_list, ri_sl_idx);
- *    while (rs_sl_idx < rs_sl_len) {
- *      rs = smartlist_get(routerstatus_list, rs_sl_idx);
- *      rs_ri_cmp = tor_memcmp(rs->identity_digest, ri->identity_digest, 20);
- *      if (rs_ri_cmp > 0) {
- *        break;
- *      } else if (rs_ri_cmp == 0) {
- *        goto matched_ri;
- *      } else {
- *        ++rs_sl_idx;
- *      }
- *    }
- *    log_info(LD_GENERAL,"No match for %s", ri->nickname);
- *    continue;
- *   matched_ri: {
- *    log_info(LD_GENERAL,"%s matches with routerstatus %p",ri->nickname,rs);
- *    }
- *  }
- */
-#define SMARTLIST_FOREACH_JOIN(sl1, type1, var1, sl2, type2, var2,      \
-                                cmpexpr, unmatched_var2)                \
-  STMT_BEGIN                                                            \
-  int var1 ## _sl_idx = 0, var1 ## _sl_len=(sl1)->num_used;             \
-  int var2 ## _sl_idx = 0, var2 ## _sl_len=(sl2)->num_used;             \
-  int var1 ## _ ## var2 ## _cmp;                                        \
-  type1 var1;                                                           \
-  type2 var2;                                                           \
-  for (; var2##_sl_idx < var2##_sl_len; ++var2##_sl_idx) {              \
-    var2 = (sl2)->list[var2##_sl_idx];                                  \
-    while (var1##_sl_idx < var1##_sl_len) {                             \
-      var1 = (sl1)->list[var1##_sl_idx];                                \
-      var1##_##var2##_cmp = (cmpexpr);                                  \
-      if (var1##_##var2##_cmp > 0) {                                    \
-        break;                                                          \
-      } else if (var1##_##var2##_cmp == 0) {                            \
-        goto matched_##var2;                                            \
-      } else {                                                          \
-        ++var1##_sl_idx;                                                \
-      }                                                                 \
-    }                                                                   \
-    /* Ran out of v1, or no match for var2. */                          \
-    unmatched_var2;                                                     \
-    continue;                                                           \
-    matched_##var2: ;                                                   \
-
-#define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
-  }                                             \
-  STMT_END
-
-#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##entry_t *prefix##iter_t;                       \
-  MOCK_DECL(maptype*, prefix##new, (void));                             \
-  void* prefix##set(maptype *map, keytype key, void *val);              \
-  void* prefix##get(const maptype *map, keytype key);                   \
-  void* prefix##remove(maptype *map, keytype key);                      \
-  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
-  int prefix##isempty(const maptype *map);                              \
-  int prefix##size(const maptype *map);                                 \
-  prefix##iter_t *prefix##iter_init(maptype *map);                      \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
-  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
-  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
-  int prefix##iter_done(prefix##iter_t *iter);                          \
-  void prefix##assert_ok(const maptype *map)
-
-/* Map from const char * to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
-/* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
-/* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
- * table. */
-DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
-
-#define MAP_FREE_AND_NULL(maptype, map, fn)     \
-  do {                                          \
-    maptype ## _free_((map), (fn));             \
-    (map) = NULL;                               \
-  } while (0)
-
-#define strmap_free(map, fn) MAP_FREE_AND_NULL(strmap, (map), (fn))
-#define digestmap_free(map, fn) MAP_FREE_AND_NULL(digestmap, (map), (fn))
-#define digest256map_free(map, fn) MAP_FREE_AND_NULL(digest256map, (map), (fn))
-
-#undef DECLARE_MAP_FNS
-
-/** Iterates over the key-value pairs in a map <b>map</b> in order.
- * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
- * The map's keys and values are of type keytype and valtype respectively;
- * each iteration assigns them to keyvar and valvar.
- *
- * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
- *     // use k and r
- *   } MAP_FOREACH_END.
- */
-/* Unpacks to, approximately:
- * {
- *   digestmap_iter_t *k_iter;
- *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
- *        k_iter = digestmap_iter_next(m, k_iter)) {
- *     const char *k;
- *     void *r_voidp;
- *     routerinfo_t *r;
- *     digestmap_iter_get(k_iter, &k, &r_voidp);
- *     r = r_voidp;
- *     // use k and r
- *   }
- * }
- */
-#define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
-  STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
-      keytype keyvar;                                                   \
-      void *valvar##_voidp;                                             \
-      valtype valvar;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
-      valvar = valvar##_voidp;
-
-/** As MAP_FOREACH, except allows members to be removed from the map
- * during the iteration via MAP_DEL_CURRENT.  Example use:
- *
- * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
- *      if (is_very_old(r))
- *       MAP_DEL_CURRENT(k);
- *   } MAP_FOREACH_END.
- **/
-/* Unpacks to, approximately:
- * {
- *   digestmap_iter_t *k_iter;
- *   int k_del=0;
- *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
- *        k_iter = k_del ? digestmap_iter_next(m, k_iter)
- *                       : digestmap_iter_next_rmv(m, k_iter)) {
- *     const char *k;
- *     void *r_voidp;
- *     routerinfo_t *r;
- *     k_del=0;
- *     digestmap_iter_get(k_iter, &k, &r_voidp);
- *     r = r_voidp;
- *     if (is_very_old(r)) {
- *       k_del = 1;
- *     }
- *   }
- * }
- */
-#define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
-  STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    int keyvar##_del=0;                                                 \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = keyvar##_del ?                                 \
-           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
-           prefix##iter_next(map, keyvar##_iter)) {                     \
-      keytype keyvar;                                                   \
-      void *valvar##_voidp;                                             \
-      valtype valvar;                                                   \
-      keyvar##_del=0;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
-      valvar = valvar##_voidp;
-
-/** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
- * member of the map.  */
-#define MAP_DEL_CURRENT(keyvar)                   \
-  STMT_BEGIN                                      \
-    keyvar##_del = 1;                             \
-  STMT_END
-
-/** Used to end a MAP_FOREACH() block. */
-#define MAP_FOREACH_END } STMT_END ;
-
-/** As MAP_FOREACH, but does not require declaration of prefix or keytype.
- * Example use:
- *   DIGESTMAP_FOREACH(m, k, routerinfo_t *, r) {
- *     // use k and r
- *   } DIGESTMAP_FOREACH_END.
- */
-#define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
-
-/** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
- * keytype.
- * Example use:
- *   DIGESTMAP_FOREACH_MODIFY(m, k, routerinfo_t *, r) {
- *      if (is_very_old(r))
- *       MAP_DEL_CURRENT(k);
- *   } DIGESTMAP_FOREACH_END.
- */
-#define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
-/** Used to end a DIGESTMAP_FOREACH() block. */
-#define DIGESTMAP_FOREACH_END MAP_FOREACH_END
-
-#define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
-  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
-#define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
-  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
-                     keyvar, valtype, valvar)
-#define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
-
-#define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
-#define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
-#define STRMAP_FOREACH_END MAP_FOREACH_END
-
-void* strmap_set_lc(strmap_t *map, const char *key, void *val);
-void* strmap_get_lc(const strmap_t *map, const char *key);
-void* strmap_remove_lc(strmap_t *map, const char *key);
-
-#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##iter_t *prefix##iter_t;                        \
-  ATTR_UNUSED static inline maptype*                                    \
-  prefix##new(void)                                                     \
-  {                                                                     \
-    return (maptype*)digestmap_new();                                   \
-  }                                                                     \
-  ATTR_UNUSED static inline digestmap_t*                                \
-  prefix##to_digestmap(maptype *map)                                    \
-  {                                                                     \
-    return (digestmap_t*)map;                                           \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##get(maptype *map, const char *key)     \
-  {                                                                     \
-    return (valtype*)digestmap_get((digestmap_t*)map, key);             \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##set(maptype *map, const char *key, valtype *val)              \
-  {                                                                     \
-    return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
-  }                                                                     \
-  ATTR_UNUSED static inline valtype*                                    \
-  prefix##remove(maptype *map, const char *key)                         \
-  {                                                                     \
-    return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
-  }                                                                     \
-  ATTR_UNUSED static inline void                                        \
-  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
-  {                                                                     \
-    digestmap_free_((digestmap_t*)map, free_val);                       \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##isempty(maptype *map)                                         \
-  {                                                                     \
-    return digestmap_isempty((digestmap_t*)map);                        \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##size(maptype *map)                                            \
-  {                                                                     \
-    return digestmap_size((digestmap_t*)map);                           \
-  }                                                                     \
-  ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_init(maptype *map)                       \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
-  }                                                                     \
-  ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next(                       \
-                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
-  }                                                                     \
-  ATTR_UNUSED static inline prefix##iter_t*                             \
-  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
-  {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
-                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
-  }                                                                     \
-  ATTR_UNUSED static inline void                                        \
-  prefix##iter_get(prefix##iter_t *iter,                                \
-                   const char **keyp,                                   \
-                   valtype **valp)                                      \
-  {                                                                     \
-    void *v;                                                            \
-    digestmap_iter_get((digestmap_iter_t*) iter, keyp, &v);             \
-    *valp = v;                                                          \
-  }                                                                     \
-  ATTR_UNUSED static inline int                                         \
-  prefix##iter_done(prefix##iter_t *iter)                               \
-  {                                                                     \
-    return digestmap_iter_done((digestmap_iter_t*)iter);                \
-  }
-
-#if SIZEOF_INT == 4
-#define BITARRAY_SHIFT 5
-#elif SIZEOF_INT == 8
-#define BITARRAY_SHIFT 6
-#else
-#error "int is neither 4 nor 8 bytes. I can't deal with that."
-#endif /* SIZEOF_INT == 4 || ... */
-#define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1)
-
-/** A random-access array of one-bit-wide elements. */
-typedef unsigned int bitarray_t;
-/** Create a new bit array that can hold <b>n_bits</b> bits. */
-static inline bitarray_t *
-bitarray_init_zero(unsigned int n_bits)
-{
-  /* round up to the next int. */
-  size_t sz = (n_bits+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  return tor_calloc(sz, sizeof(unsigned int));
-}
-/** Expand <b>ba</b> from holding <b>n_bits_old</b> to <b>n_bits_new</b>,
- * clearing all new bits.  Returns a possibly changed pointer to the
- * bitarray. */
-static inline bitarray_t *
-bitarray_expand(bitarray_t *ba,
-                unsigned int n_bits_old, unsigned int n_bits_new)
-{
-  size_t sz_old = (n_bits_old+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  size_t sz_new = (n_bits_new+BITARRAY_MASK) >> BITARRAY_SHIFT;
-  char *ptr;
-  if (sz_new <= sz_old)
-    return ba;
-  ptr = tor_reallocarray(ba, sz_new, sizeof(unsigned int));
-  /* This memset does nothing to the older excess bytes.  But they were
-   * already set to 0 by bitarry_init_zero. */
-  memset(ptr+sz_old*sizeof(unsigned int), 0,
-         (sz_new-sz_old)*sizeof(unsigned int));
-  return (bitarray_t*) ptr;
-}
-/** Free the bit array <b>ba</b>. */
-static inline void
-bitarray_free_(bitarray_t *ba)
-{
-  tor_free(ba);
-}
-#define bitarray_free(ba) FREE_AND_NULL(bitarray_t, bitarray_free_, (ba))
-
-/** Set the <b>bit</b>th bit in <b>b</b> to 1. */
-static inline void
-bitarray_set(bitarray_t *b, int bit)
-{
-  b[bit >> BITARRAY_SHIFT] |= (1u << (bit & BITARRAY_MASK));
-}
-/** Set the <b>bit</b>th bit in <b>b</b> to 0. */
-static inline void
-bitarray_clear(bitarray_t *b, int bit)
-{
-  b[bit >> BITARRAY_SHIFT] &= ~ (1u << (bit & BITARRAY_MASK));
-}
-/** Return true iff <b>bit</b>th bit in <b>b</b> is nonzero.  NOTE: does
- * not necessarily return 1 on true. */
-static inline unsigned int
-bitarray_is_set(bitarray_t *b, int bit)
-{
-  return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
-}
-
-/** A set of digests, implemented as a Bloom filter. */
-typedef struct {
-  int mask; /**< One less than the number of bits in <b>ba</b>; always one less
-             * than a power of two. */
-  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
-} digestset_t;
-
-#define BIT(n) ((n) & set->mask)
-/** Add the digest <b>digest</b> to <b>set</b>. */
-static inline void
-digestset_add(digestset_t *set, const char *digest)
-{
-  const uint64_t x = siphash24g(digest, 20);
-  const uint32_t d1 = (uint32_t) x;
-  const uint32_t d2 = (uint32_t)( (x>>16) + x);
-  const uint32_t d3 = (uint32_t)( (x>>32) + x);
-  const uint32_t d4 = (uint32_t)( (x>>48) + x);
-  bitarray_set(set->ba, BIT(d1));
-  bitarray_set(set->ba, BIT(d2));
-  bitarray_set(set->ba, BIT(d3));
-  bitarray_set(set->ba, BIT(d4));
-}
-
-/** If <b>digest</b> is in <b>set</b>, return nonzero.  Otherwise,
- * <em>probably</em> return zero. */
-static inline int
-digestset_contains(const digestset_t *set, const char *digest)
-{
-  const uint64_t x = siphash24g(digest, 20);
-  const uint32_t d1 = (uint32_t) x;
-  const uint32_t d2 = (uint32_t)( (x>>16) + x);
-  const uint32_t d3 = (uint32_t)( (x>>32) + x);
-  const uint32_t d4 = (uint32_t)( (x>>48) + x);
-  return bitarray_is_set(set->ba, BIT(d1)) &&
-         bitarray_is_set(set->ba, BIT(d2)) &&
-         bitarray_is_set(set->ba, BIT(d3)) &&
-         bitarray_is_set(set->ba, BIT(d4));
-}
-#undef BIT
-
-digestset_t *digestset_new(int max_elements);
-void digestset_free_(digestset_t* set);
-#define digestset_free(set) FREE_AND_NULL(digestset_t, digestset_free_, (set))
-
-/* These functions, given an <b>array</b> of <b>n_elements</b>, return the
- * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element;
- * <b>n_elements</b>-1 gives the highest; and (<b>n_elements</b>-1) / 2 gives
- * the median.  As a side effect, the elements of <b>array</b> are sorted. */
-int find_nth_int(int *array, int n_elements, int nth);
-time_t find_nth_time(time_t *array, int n_elements, int nth);
-double find_nth_double(double *array, int n_elements, int nth);
-int32_t find_nth_int32(int32_t *array, int n_elements, int nth);
-uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth);
-long find_nth_long(long *array, int n_elements, int nth);
-static inline int
-median_int(int *array, int n_elements)
-{
-  return find_nth_int(array, n_elements, (n_elements-1)/2);
-}
-static inline time_t
-median_time(time_t *array, int n_elements)
-{
-  return find_nth_time(array, n_elements, (n_elements-1)/2);
-}
-static inline double
-median_double(double *array, int n_elements)
-{
-  return find_nth_double(array, n_elements, (n_elements-1)/2);
-}
-static inline uint32_t
-median_uint32(uint32_t *array, int n_elements)
-{
-  return find_nth_uint32(array, n_elements, (n_elements-1)/2);
-}
-static inline int32_t
-median_int32(int32_t *array, int n_elements)
-{
-  return find_nth_int32(array, n_elements, (n_elements-1)/2);
-}
-
-static inline uint32_t
-third_quartile_uint32(uint32_t *array, int n_elements)
-{
-  return find_nth_uint32(array, n_elements, (n_elements*3)/4);
-}
-
-#endif /* !defined(TOR_CONTAINER_H) */
-

src/common/include.am

@@ -38,11 +38,8 @@ LIBOR_A_SRC = \
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
   src/common/confline.c					\
-  src/common/container.c				\
-  src/common/log.c					\
   src/common/memarea.c					\
   src/common/util.c					\
-  src/common/util_bug.c					\
   src/common/util_format.c				\
   src/common/util_process.c				\
   src/common/sandbox.c					\
@@ -87,7 +84,6 @@ COMMONHEADERS = \
   src/common/compat_threads.h			\
   src/common/compat_time.h			\
   src/common/confline.h				\
-  src/common/container.h			\
   src/common/handles.h				\
   src/common/memarea.h				\
   src/common/linux_syscalls.inc			\
@@ -96,9 +92,7 @@ COMMONHEADERS = \
   src/common/storagedir.h			\
   src/common/timers.h				\
   src/common/token_bucket.h			\
-  src/common/torlog.h				\
   src/common/util.h				\
-  src/common/util_bug.h				\
   src/common/util_format.h			\
   src/common/util_process.h			\
   src/common/workqueue.h

src/common/memarea.c

@@ -12,8 +12,7 @@
 #include "common/memarea.h"
 #include "common/util.h"
 #include "common/compat.h"
-#include "common/torlog.h"
-#include "common/container.h"
+#include "lib/log/torlog.h"
 
 #ifndef DISABLE_MEMORY_SENTINELS
 

src/common/procmon.h

@@ -12,7 +12,7 @@
 #include "common/compat.h"
 #include "common/compat_libevent.h"
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 typedef struct tor_process_monitor_t tor_process_monitor_t;
 

src/common/sandbox.c

@@ -33,15 +33,17 @@
 #include <stdlib.h>
 
 #include "common/sandbox.h"
-#include "common/container.h"
+#include "lib/container/map.h"
 #include "lib/err/torerr.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/cc/torint.h"
 #include "common/util.h"
 #include "tor_queue.h"
 
 #include "ht.h"
 
+#include "siphash.h"
+
 #define DEBUGGING_CLOSE
 
 #if defined(USE_LIBSECCOMP)

src/common/storagedir.c

@@ -1,13 +1,13 @@
 /* Copyright (c) 2017-2018, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#include "common/container.h"
+#include "lib/container/smartlist.h"
 #include "common/compat.h"
 #include "common/confline.h"
 #include "common/memarea.h"
 #include "common/sandbox.h"
 #include "common/storagedir.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 
 #ifdef HAVE_SYS_TYPES_H
@@ -583,4 +583,3 @@ storage_dir_get_max_files(storage_dir_t *d)
 {
   return d->max_files;
 }
-

src/common/timers.c

@@ -34,7 +34,7 @@
 #include "common/compat.h"
 #include "common/compat_libevent.h"
 #include "common/timers.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 
 struct timeout_cb {

src/common/token_bucket.c

@@ -19,7 +19,8 @@
 #define TOKEN_BUCKET_PRIVATE
 
 #include "common/token_bucket.h"
-#include "common/util_bug.h"
+#include "common/compat.h"
+#include "lib/log/util_bug.h"
 
 /**
  * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg.
@@ -252,4 +253,3 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket,
     flags |= TB_WRITE;
   return flags;
 }
-

src/common/util.h

@@ -23,7 +23,16 @@
 #include <sys/stat.h>
 #endif
 #include "lib/err/torerr.h"
-#include "common/util_bug.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/wallclock/approx_time.h"
+#include "lib/string/util_string.h"
+#include "lib/string/scanf.h"
+#include "lib/intmath/bits.h"
+#include "lib/intmath/addsub.h"
+#include "lib/intmath/muldiv.h"
+#include "lib/intmath/cmp.h"
+#include "lib/log/ratelim.h"
+#include "lib/log/util_bug.h"
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -35,90 +44,11 @@
 #define O_NOFOLLOW 0
 #endif
 
-/* Memory management */
-void *tor_malloc_(size_t size) ATTR_MALLOC;
-void *tor_malloc_zero_(size_t size) ATTR_MALLOC;
-void *tor_calloc_(size_t nmemb, size_t size) ATTR_MALLOC;
-void *tor_realloc_(void *ptr, size_t size);
-void *tor_reallocarray_(void *ptr, size_t size1, size_t size2);
-char *tor_strdup_(const char *s) ATTR_MALLOC ATTR_NONNULL((1));
-char *tor_strndup_(const char *s, size_t n)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void *tor_memdup_(const void *mem, size_t len)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void *tor_memdup_nulterm_(const void *mem, size_t len)
-  ATTR_MALLOC ATTR_NONNULL((1));
-void tor_free_(void *mem);
 uint64_t tor_htonll(uint64_t a);
 uint64_t tor_ntohll(uint64_t a);
-/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup,
- * etc.  Unlike the free() function, the tor_free() macro sets the
- * pointer value to NULL after freeing it.
- *
- * This is a macro.  If you need a function pointer to release memory from
- * tor_malloc(), use tor_free_().
- *
- * Note that this macro takes the address of the pointer it is going to
- * free and clear.  If that pointer is stored with a nonstandard
- * alignment (eg because of a "packed" pragma) it is not correct to use
- * tor_free().
- */
-#ifdef __GNUC__
-#define tor_free(p) STMT_BEGIN                                 \
-    typeof(&(p)) tor_free__tmpvar = &(p);                      \
-    raw_free(*tor_free__tmpvar);                               \
-    *tor_free__tmpvar=NULL;                                    \
-  STMT_END
-#else
-#define tor_free(p) STMT_BEGIN                                 \
-  raw_free(p);                                                 \
-  (p)=NULL;                                                    \
-  STMT_END
-#endif
-
-#define tor_malloc(size)       tor_malloc_(size)
-#define tor_malloc_zero(size)  tor_malloc_zero_(size)
-#define tor_calloc(nmemb,size) tor_calloc_(nmemb, size)
-#define tor_realloc(ptr, size) tor_realloc_(ptr, size)
-#define tor_reallocarray(ptr, sz1, sz2) \
-  tor_reallocarray_((ptr), (sz1), (sz2))
-#define tor_strdup(s)          tor_strdup_(s)
-#define tor_strndup(s, n)      tor_strndup_(s, n)
-#define tor_memdup(s, n)       tor_memdup_(s, n)
-#define tor_memdup_nulterm(s, n)       tor_memdup_nulterm_(s, n)
-
-/* Aliases for the underlying system malloc/realloc/free. Only use
- * them to indicate "I really want the underlying system function, I know
- * what I'm doing." */
-#define raw_malloc  malloc
-#define raw_realloc realloc
-#define raw_free    free
-#define raw_strdup  strdup
 
 void tor_log_mallinfo(int severity);
 
-/* Helper macro: free a variable of type 'typename' using freefn, and
- * set the variable to NULL.
- */
-#define FREE_AND_NULL(typename, freefn, var)                            \
-  do {                                                                  \
-    /* only evaluate (var) once. */                                     \
-    typename **tmp__free__ptr ## freefn = &(var);                       \
-    freefn(*tmp__free__ptr ## freefn);                                  \
-    (*tmp__free__ptr ## freefn) = NULL;                                 \
-  } while (0)
-
-/** Macro: yield a pointer to the field at position <b>off</b> within the
- * structure <b>st</b>.  Example:
- * <pre>
- *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = offsetof(struct a, bar);
- *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
- *   *bar_p = 3;
- * </pre>
- */
-#define STRUCT_VAR_P(st, off) ((void*) ( ((char*)(st)) + (off) ) )
-
 /** Macro: yield a pointer to an enclosing structure given a pointer to
  * a substructure at offset <b>off</b>. Example:
  * <pre>
@@ -141,54 +71,13 @@ void tor_log_mallinfo(int severity);
 double tor_mathlog(double d) ATTR_CONST;
 long tor_lround(double d) ATTR_CONST;
 int64_t tor_llround(double d) ATTR_CONST;
-int tor_log2(uint64_t u64) ATTR_CONST;
-uint64_t round_to_power_of_2(uint64_t u64);
-unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
-uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
-uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
 int64_t sample_laplace_distribution(double mu, double b, double p);
 int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
                           double epsilon);
-int n_bits_set_u8(uint8_t v);
 int64_t clamp_double_to_int64(double number);
-void simplify_fraction64(uint64_t *numer, uint64_t *denom);
-
-uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
-
-/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
- * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
- * can overflow. */
-#define CEIL_DIV(a,b) (((a)+((b)-1))/(b))
-
-/* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
- * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
- * <b>b</b> is larger than <b>max</b>.
- *
- * Requires that <b>min</b> is no more than <b>max</b>. May evaluate any of
- * its arguments more than once! */
-#define CLAMP(min,v,max)                        \
-  ( ((v) < (min)) ? (min) :                     \
-    ((v) > (max)) ? (max) :                     \
-    (v) )
 
 /* String manipulation */
 
-/** Allowable characters in a hexadecimal string. */
-#define HEX_CHARACTERS "0123456789ABCDEFabcdef"
-void tor_strlower(char *s) ATTR_NONNULL((1));
-void tor_strupper(char *s) ATTR_NONNULL((1));
-int tor_strisprint(const char *s) ATTR_NONNULL((1));
-int tor_strisnonupper(const char *s) ATTR_NONNULL((1));
-int tor_strisspace(const char *s);
-int strcmp_opt(const char *s1, const char *s2);
-int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2));
-int strcasecmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int strcasecmpend(const char *s1, const char *s2) ATTR_NONNULL((1,2));
-int fast_memcmpstart(const void *mem, size_t memlen, const char *prefix);
-
-void tor_strstrip(char *s, const char *strip) ATTR_NONNULL((1,2));
 long tor_parse_long(const char *s, int base, long min,
                     long max, int *ok, char **next);
 unsigned long tor_parse_ulong(const char *s, int base, unsigned long min,
@@ -197,16 +86,9 @@ double tor_parse_double(const char *s, double min, double max, int *ok,
                         char **next);
 uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
                          uint64_t max, int *ok, char **next);
+
 const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1));
-const char *eat_whitespace(const char *s);
-const char *eat_whitespace_eos(const char *s, const char *eos);
-const char *eat_whitespace_no_nl(const char *s);
-const char *eat_whitespace_eos_no_nl(const char *s, const char *eos);
-const char *find_whitespace(const char *s);
-const char *find_whitespace_eos(const char *s, const char *eos);
-const char *find_str_at_start_of_line(const char *haystack,
-                                      const char *needle);
-int string_is_C_identifier(const char *string);
+
 int string_is_key_value(int severity, const char *string);
 int string_is_valid_dest(const char *string);
 int string_is_valid_nonrfc_hostname(const char *string);
@@ -216,6 +98,7 @@ int string_is_valid_ipv6_address(const char *string);
 int tor_mem_is_zero(const char *mem, size_t len);
 int tor_digest_is_zero(const char *digest);
 int tor_digest256_is_zero(const char *digest);
+
 char *esc_for_log(const char *string) ATTR_MALLOC;
 char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
 const char *escaped(const char *string);
@@ -223,19 +106,6 @@ const char *escaped(const char *string);
 char *tor_escape_str_for_pt_args(const char *string,
                                  const char *chars_to_escape);
 
-struct smartlist_t;
-int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
-  CHECK_SCANF(2, 0);
-int tor_sscanf(const char *buf, const char *pattern, ...)
-  CHECK_SCANF(2, 3);
-
-void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
-  CHECK_PRINTF(2, 3);
-void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
-                             va_list args)
-  CHECK_PRINTF(2, 0);
-void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
-
 /* Time helpers */
 long tv_udiff(const struct timeval *start, const struct timeval *end);
 long tv_mdiff(const struct timeval *start, const struct timeval *end);
@@ -257,52 +127,6 @@ int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
-/* Cached time */
-#ifdef TIME_IS_FAST
-#define approx_time() time(NULL)
-#define update_approx_time(t) STMT_NIL
-#else
-time_t approx_time(void);
-void update_approx_time(time_t now);
-#endif /* defined(TIME_IS_FAST) */
-
-/* Rate-limiter */
-
-/** A ratelim_t remembers how often an event is occurring, and how often
- * it's allowed to occur.  Typical usage is something like:
- *
-   <pre>
-    if (possibly_very_frequent_event()) {
-      const int INTERVAL = 300;
-      static ratelim_t warning_limit = RATELIM_INIT(INTERVAL);
-      char *m;
-      if ((m = rate_limit_log(&warning_limit, approx_time()))) {
-        log_warn(LD_GENERAL, "The event occurred!%s", m);
-        tor_free(m);
-      }
-    }
-   </pre>
-
-   As a convenience wrapper for logging, you can replace the above with:
-   <pre>
-   if (possibly_very_frequent_event()) {
-     static ratelim_t warning_limit = RATELIM_INIT(300);
-     log_fn_ratelim(&warning_limit, LOG_WARN, LD_GENERAL,
-                    "The event occurred!");
-   }
-   </pre>
- */
-typedef struct ratelim_t {
-  int rate;
-  time_t last_allowed;
-  int n_calls_since_last_time;
-} ratelim_t;
-
-#define RATELIM_INIT(r) { (r), 0, 0 }
-#define RATELIM_TOOMANY (16*1000*1000)
-
-char *rate_limit_log(ratelim_t *lim, time_t now);
-
 /* File helpers */
 ssize_t write_all(tor_socket_t fd, const char *buf, size_t count,int isSocket);
 ssize_t read_all(tor_socket_t fd, char *buf, size_t count, int isSocket);
@@ -362,6 +186,7 @@ typedef struct sized_chunk_t {
   const char *bytes;
   size_t len;
 } sized_chunk_t;
+struct smartlist_t;
 int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
                          int bin, int no_tempfile);
 int append_bytes_to_file(const char *fname, const char *str, size_t len,
@@ -538,8 +363,4 @@ STATIC int format_helper_exit_status(unsigned char child_state,
 
 #endif /* defined(UTIL_PRIVATE) */
 
-int size_mul_check(const size_t x, const size_t y);
-
-#define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
-
 #endif /* !defined(TOR_UTIL_H) */

src/common/util_format.c

@@ -12,7 +12,8 @@
  */
 
 #include "orconfig.h"
-#include "common/torlog.h"
+#include <stddef.h>
+#include "lib/log/torlog.h"
 #include "common/util.h"
 #include "common/util_format.h"
 #include "lib/cc/torint.h"
@@ -464,39 +465,6 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   *cp = '\0';
 }
 
-/** Helper: given a hex digit, return its value, or -1 if it isn't hex. */
-static inline int
-hex_decode_digit_(char c)
-{
-  switch (c) {
-    case '0': return 0;
-    case '1': return 1;
-    case '2': return 2;
-    case '3': return 3;
-    case '4': return 4;
-    case '5': return 5;
-    case '6': return 6;
-    case '7': return 7;
-    case '8': return 8;
-    case '9': return 9;
-    case 'A': case 'a': return 10;
-    case 'B': case 'b': return 11;
-    case 'C': case 'c': return 12;
-    case 'D': case 'd': return 13;
-    case 'E': case 'e': return 14;
-    case 'F': case 'f': return 15;
-    default:
-      return -1;
-  }
-}
-
-/** Helper: given a hex digit, return its value, or -1 if it isn't hex. */
-int
-hex_decode_digit(char c)
-{
-  return hex_decode_digit_(c);
-}
-
 /** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode
  * it and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>.
  * Return the number of bytes decoded on success, -1 on failure. If
@@ -519,8 +487,8 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   end = src+srclen;
   while (src<end) {
-    v1 = hex_decode_digit_(*src);
-    v2 = hex_decode_digit_(*(src+1));
+    v1 = hex_decode_digit(*src);
+    v2 = hex_decode_digit(*(src+1));
     if (v1<0||v2<0)
       return -1;
     *(uint8_t*)dest = (v1<<4)|v2;
@@ -532,4 +500,3 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   return (int) (dest-dest_orig);
 }
-

src/common/util_format.h

@@ -44,9 +44,7 @@ void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 size_t base32_encoded_size(size_t srclen);
 
-int hex_decode_digit(char c);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
 #endif /* !defined(TOR_UTIL_FORMAT_H) */
-

src/common/util_process.c

@@ -21,7 +21,7 @@
 
 #include "common/compat.h"
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util_process.h"
 #include "ht.h"
 

src/common/workqueue.c

@@ -31,7 +31,7 @@
 #include "common/util.h"
 #include "common/workqueue.h"
 #include "tor_queue.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 #include <event2/event.h>
 

src/include.am

@@ -3,11 +3,20 @@ include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am
 include src/lib/compress/include.am
+include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
+include src/lib/defs/include.am
+include src/lib/fdio/include.am
 include src/lib/include.libdonna.am
+include src/lib/intmath/include.am
+include src/lib/lock/include.am
+include src/lib/log/include.am
+include src/lib/malloc/include.am
+include src/lib/string/include.am
 include src/lib/testsupport/include.am
 include src/lib/tls/include.am
 include src/lib/trace/include.am
+include src/lib/wallclock/include.am
 include src/common/include.am
 include src/trunnel/include.am
 include src/or/include.am

src/lib/cc/compat_compiler.h

@@ -242,4 +242,18 @@
 #error Unknown: SIZEOF_INTPTR_T
 #endif /* (SIZEOF_INTPTR_T == SIZEOF_INT) || ... */
 
+/** Macro: yield a pointer to the field at position <b>off</b> within the
+ * structure <b>st</b>.  Example:
+ * <pre>
+ *   struct a { int foo; int bar; } x;
+ *   off_t bar_offset = offsetof(struct a, bar);
+ *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
+ *   *bar_p = 3;
+ * </pre>
+ */
+#define STRUCT_VAR_P(st, off) ((void*) ( ((char*)(st)) + (off) ) )
+
+/** Macro: Yields the number of elements in array x. */
+#define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
+
 #endif /* !defined(TOR_COMPAT_H) */

src/lib/compress/.may_include

@@ -1,6 +1,7 @@
 orconfig.h
 lib/cc/*.h
 lib/compress/*.h
+lib/log/*.h
 
 # XXX I'd like to remove this.
 common/*.h

src/lib/compress/compress.c

@@ -20,7 +20,7 @@
 #endif
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_none.h"

src/lib/compress/compress_lzma.c

@@ -14,7 +14,7 @@
 #include "orconfig.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 

src/lib/compress/compress_none.c

@@ -17,7 +17,7 @@
 #include "orconfig.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_none.h"
 

src/lib/compress/compress_zlib.c

@@ -14,7 +14,7 @@
 #include "orconfig.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zlib.h"
 

src/lib/compress/compress_zstd.c

@@ -14,7 +14,7 @@
 #include "orconfig.h"
 
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zstd.h"
 

src/lib/container/.may_include

@@ -0,0 +1,16 @@
+orconfig.h
+lib/cc/*.h
+lib/container/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/malloc/*.h
+lib/err/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
+lib/intmath/*.h
+
+ht.h
+siphash.h
+
+# XXX I'd like to remove this.
+lib/log/util_bug.h

src/lib/container/bitarray.h

@@ -0,0 +1,80 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BITARRAY_H
+#define TOR_BITARRAY_H
+
+#include "orconfig.h"
+#include <string.h>
+#include "lib/cc/torint.h"
+#include "lib/malloc/util_malloc.h"
+
+#if SIZEOF_INT == 4
+#define BITARRAY_SHIFT 5
+#elif SIZEOF_INT == 8
+#define BITARRAY_SHIFT 6
+#else
+#error "int is neither 4 nor 8 bytes. I can't deal with that."
+#endif /* SIZEOF_INT == 4 || ... */
+#define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1)
+
+/** A random-access array of one-bit-wide elements. */
+typedef unsigned int bitarray_t;
+/** Create a new bit array that can hold <b>n_bits</b> bits. */
+static inline bitarray_t *
+bitarray_init_zero(unsigned int n_bits)
+{
+  /* round up to the next int. */
+  size_t sz = (n_bits+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  return tor_calloc(sz, sizeof(unsigned int));
+}
+/** Expand <b>ba</b> from holding <b>n_bits_old</b> to <b>n_bits_new</b>,
+ * clearing all new bits.  Returns a possibly changed pointer to the
+ * bitarray. */
+static inline bitarray_t *
+bitarray_expand(bitarray_t *ba,
+                unsigned int n_bits_old, unsigned int n_bits_new)
+{
+  size_t sz_old = (n_bits_old+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  size_t sz_new = (n_bits_new+BITARRAY_MASK) >> BITARRAY_SHIFT;
+  char *ptr;
+  if (sz_new <= sz_old)
+    return ba;
+  ptr = tor_reallocarray(ba, sz_new, sizeof(unsigned int));
+  /* This memset does nothing to the older excess bytes.  But they were
+   * already set to 0 by bitarry_init_zero. */
+  memset(ptr+sz_old*sizeof(unsigned int), 0,
+         (sz_new-sz_old)*sizeof(unsigned int));
+  return (bitarray_t*) ptr;
+}
+/** Free the bit array <b>ba</b>. */
+static inline void
+bitarray_free_(bitarray_t *ba)
+{
+  tor_free(ba);
+}
+#define bitarray_free(ba) FREE_AND_NULL(bitarray_t, bitarray_free_, (ba))
+
+/** Set the <b>bit</b>th bit in <b>b</b> to 1. */
+static inline void
+bitarray_set(bitarray_t *b, int bit)
+{
+  b[bit >> BITARRAY_SHIFT] |= (1u << (bit & BITARRAY_MASK));
+}
+/** Set the <b>bit</b>th bit in <b>b</b> to 0. */
+static inline void
+bitarray_clear(bitarray_t *b, int bit)
+{
+  b[bit >> BITARRAY_SHIFT] &= ~ (1u << (bit & BITARRAY_MASK));
+}
+/** Return true iff <b>bit</b>th bit in <b>b</b> is nonzero.  NOTE: does
+ * not necessarily return 1 on true. */
+static inline unsigned int
+bitarray_is_set(bitarray_t *b, int bit)
+{
+  return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
+}
+
+#endif /* !defined(TOR_CONTAINER_H) */

src/lib/container/bloomfilt.c

@@ -0,0 +1,49 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file container.c
+ * \brief Implements a smartlist (a resizable array) along
+ * with helper functions to use smartlists.  Also includes
+ * hash table implementations of a string-to-void* map, and of
+ * a digest-to-void* map.
+ **/
+
+#include <stdlib.h>
+#include "lib/malloc/util_malloc.h"
+#include "lib/container/bloomfilt.h"
+#include "lib/intmath/bits.h"
+
+/** Return a newly allocated digestset_t, optimized to hold a total of
+ * <b>max_elements</b> digests with a reasonably low false positive weight. */
+digestset_t *
+digestset_new(int max_elements)
+{
+  /* The probability of false positives is about P=(1 - exp(-kn/m))^k, where k
+   * is the number of hash functions per entry, m is the bits in the array,
+   * and n is the number of elements inserted.  For us, k==4, n<=max_elements,
+   * and m==n_bits= approximately max_elements*32.  This gives
+   *   P<(1-exp(-4*n/(32*n)))^4 == (1-exp(1/-8))^4 == .00019
+   *
+   * It would be more optimal in space vs false positives to get this false
+   * positive rate by going for k==13, and m==18.5n, but we also want to
+   * conserve CPU, and k==13 is pretty big.
+   */
+  int n_bits = 1u << (tor_log2(max_elements)+5);
+  digestset_t *r = tor_malloc(sizeof(digestset_t));
+  r->mask = n_bits - 1;
+  r->ba = bitarray_init_zero(n_bits);
+  return r;
+}
+
+/** Free all storage held in <b>set</b>. */
+void
+digestset_free_(digestset_t *set)
+{
+  if (!set)
+    return;
+  bitarray_free(set->ba);
+  tor_free(set);
+}

src/lib/container/bloomfilt.h

@@ -0,0 +1,58 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BLOOMFILT_H
+#define TOR_BLOOMFILT_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/container/bitarray.h"
+#include "siphash.h"
+
+/** A set of digests, implemented as a Bloom filter. */
+typedef struct {
+  int mask; /**< One less than the number of bits in <b>ba</b>; always one less
+             * than a power of two. */
+  bitarray_t *ba; /**< A bit array to implement the Bloom filter. */
+} digestset_t;
+
+#define BIT(n) ((n) & set->mask)
+/** Add the digest <b>digest</b> to <b>set</b>. */
+static inline void
+digestset_add(digestset_t *set, const char *digest)
+{
+  const uint64_t x = siphash24g(digest, 20);
+  const uint32_t d1 = (uint32_t) x;
+  const uint32_t d2 = (uint32_t)( (x>>16) + x);
+  const uint32_t d3 = (uint32_t)( (x>>32) + x);
+  const uint32_t d4 = (uint32_t)( (x>>48) + x);
+  bitarray_set(set->ba, BIT(d1));
+  bitarray_set(set->ba, BIT(d2));
+  bitarray_set(set->ba, BIT(d3));
+  bitarray_set(set->ba, BIT(d4));
+}
+
+/** If <b>digest</b> is in <b>set</b>, return nonzero.  Otherwise,
+ * <em>probably</em> return zero. */
+static inline int
+digestset_contains(const digestset_t *set, const char *digest)
+{
+  const uint64_t x = siphash24g(digest, 20);
+  const uint32_t d1 = (uint32_t) x;
+  const uint32_t d2 = (uint32_t)( (x>>16) + x);
+  const uint32_t d3 = (uint32_t)( (x>>32) + x);
+  const uint32_t d4 = (uint32_t)( (x>>48) + x);
+  return bitarray_is_set(set->ba, BIT(d1)) &&
+         bitarray_is_set(set->ba, BIT(d2)) &&
+         bitarray_is_set(set->ba, BIT(d3)) &&
+         bitarray_is_set(set->ba, BIT(d4));
+}
+#undef BIT
+
+digestset_t *digestset_new(int max_elements);
+void digestset_free_(digestset_t* set);
+#define digestset_free(set) FREE_AND_NULL(digestset_t, digestset_free_, (set))
+
+#endif /* !defined(TOR_CONTAINER_H) */

src/lib/container/include.am

@@ -0,0 +1,24 @@
+
+noinst_LIBRARIES += src/lib/libtor-container.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-container-testing.a
+endif
+
+src_lib_libtor_container_a_SOURCES =			\
+	src/lib/container/bloomfilt.c			\
+	src/lib/container/map.c				\
+	src/lib/container/order.c			\
+	src/lib/container/smartlist.c
+
+src_lib_libtor_container_testing_a_SOURCES = \
+	$(src_lib_libtor_container_a_SOURCES)
+src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/container/bitarray.h			\
+	src/lib/container/bloomfilt.h			\
+	src/lib/container/map.h				\
+	src/lib/container/order.h			\
+	src/lib/container/smartlist.h

src/lib/container/map.c

@@ -0,0 +1,414 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file container.c
+ * \brief Implements a smartlist (a resizable array) along
+ * with helper functions to use smartlists.  Also includes
+ * hash table implementations of a string-to-void* map, and of
+ * a digest-to-void* map.
+ **/
+
+#include "lib/container/map.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/string/util_string.h"
+#include "lib/malloc/util_malloc.h"
+
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "ht.h"
+
+/** Helper: Declare an entry type and a map type to implement a mapping using
+ * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
+ * entry is declared using the C declaration <b>keydecl</b>.  All functions
+ * and types associated with the map get prefixed with <b>prefix</b> */
+#define DEFINE_MAP_STRUCTS(maptype, keydecl, prefix)      \
+  typedef struct prefix ## entry_t {                      \
+    HT_ENTRY(prefix ## entry_t) node;                     \
+    void *val;                                            \
+    keydecl;                                              \
+  } prefix ## entry_t;                                    \
+  struct maptype {                                        \
+    HT_HEAD(prefix ## impl, prefix ## entry_t) head;      \
+  }
+
+DEFINE_MAP_STRUCTS(strmap_t, char *key, strmap_);
+DEFINE_MAP_STRUCTS(digestmap_t, char key[DIGEST_LEN], digestmap_);
+DEFINE_MAP_STRUCTS(digest256map_t, uint8_t key[DIGEST256_LEN], digest256map_);
+
+/** Helper: compare strmap_entry_t objects by key value. */
+static inline int
+strmap_entries_eq(const strmap_entry_t *a, const strmap_entry_t *b)
+{
+  return !strcmp(a->key, b->key);
+}
+
+/** Helper: return a hash value for a strmap_entry_t. */
+static inline unsigned int
+strmap_entry_hash(const strmap_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, strlen(a->key));
+}
+
+/** Helper: compare digestmap_entry_t objects by key value. */
+static inline int
+digestmap_entries_eq(const digestmap_entry_t *a, const digestmap_entry_t *b)
+{
+  return tor_memeq(a->key, b->key, DIGEST_LEN);
+}
+
+/** Helper: return a hash value for a digest_map_t. */
+static inline unsigned int
+digestmap_entry_hash(const digestmap_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, DIGEST_LEN);
+}
+
+/** Helper: compare digestmap_entry_t objects by key value. */
+static inline int
+digest256map_entries_eq(const digest256map_entry_t *a,
+                        const digest256map_entry_t *b)
+{
+  return tor_memeq(a->key, b->key, DIGEST256_LEN);
+}
+
+/** Helper: return a hash value for a digest_map_t. */
+static inline unsigned int
+digest256map_entry_hash(const digest256map_entry_t *a)
+{
+  return (unsigned) siphash24g(a->key, DIGEST256_LEN);
+}
+
+HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
+             strmap_entries_eq)
+HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
+             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
+             digestmap_entries_eq)
+HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
+             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
+             digest256map_entry_hash,
+             digest256map_entries_eq)
+HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
+             digest256map_entry_hash,
+             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+#define strmap_entry_free(ent) \
+  FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
+#define digestmap_entry_free(ent) \
+  FREE_AND_NULL(digestmap_entry_t, digestmap_entry_free_, (ent))
+#define digest256map_entry_free(ent) \
+  FREE_AND_NULL(digest256map_entry_t, digest256map_entry_free_, (ent))
+
+static inline void
+strmap_entry_free_(strmap_entry_t *ent)
+{
+  tor_free(ent->key);
+  tor_free(ent);
+}
+static inline void
+digestmap_entry_free_(digestmap_entry_t *ent)
+{
+  tor_free(ent);
+}
+static inline void
+digest256map_entry_free_(digest256map_entry_t *ent)
+{
+  tor_free(ent);
+}
+
+static inline void
+strmap_assign_tmp_key(strmap_entry_t *ent, const char *key)
+{
+  ent->key = (char*)key;
+}
+static inline void
+digestmap_assign_tmp_key(digestmap_entry_t *ent, const char *key)
+{
+  memcpy(ent->key, key, DIGEST_LEN);
+}
+static inline void
+digest256map_assign_tmp_key(digest256map_entry_t *ent, const uint8_t *key)
+{
+  memcpy(ent->key, key, DIGEST256_LEN);
+}
+static inline void
+strmap_assign_key(strmap_entry_t *ent, const char *key)
+{
+  ent->key = tor_strdup(key);
+}
+static inline void
+digestmap_assign_key(digestmap_entry_t *ent, const char *key)
+{
+  memcpy(ent->key, key, DIGEST_LEN);
+}
+static inline void
+digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key)
+{
+  memcpy(ent->key, key, DIGEST256_LEN);
+}
+
+/**
+ * Macro: implement all the functions for a map that are declared in
+ * map.h by the DECLARE_MAP_FNS() macro.  You must additionally define a
+ * prefix_entry_free_() function to free entries (and their keys), a
+ * prefix_assign_tmp_key() function to temporarily set a stack-allocated
+ * entry to hold a key, and a prefix_assign_key() function to set a
+ * heap-allocated entry to hold a key.
+ */
+#define IMPLEMENT_MAP_FNS(maptype, keytype, prefix)                     \
+  /** Create and return a new empty map. */                             \
+  MOCK_IMPL(maptype *,                                                  \
+  prefix##_new,(void))                                                  \
+  {                                                                     \
+    maptype *result;                                                    \
+    result = tor_malloc(sizeof(maptype));                               \
+    HT_INIT(prefix##_impl, &result->head);                              \
+    return result;                                                      \
+  }                                                                     \
+                                                                        \
+  /** Return the item from <b>map</b> whose key matches <b>key</b>, or  \
+   * NULL if no such value exists. */                                   \
+  void *                                                                \
+  prefix##_get(const maptype *map, const keytype key)                   \
+  {                                                                     \
+    prefix ##_entry_t *resolve;                                         \
+    prefix ##_entry_t search;                                           \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    prefix ##_assign_tmp_key(&search, key);                             \
+    resolve = HT_FIND(prefix ##_impl, &map->head, &search);             \
+    if (resolve) {                                                      \
+      return resolve->val;                                              \
+    } else {                                                            \
+      return NULL;                                                      \
+    }                                                                   \
+  }                                                                     \
+                                                                        \
+  /** Add an entry to <b>map</b> mapping <b>key</b> to <b>val</b>;      \
+   * return the previous value, or NULL if no such value existed. */     \
+  void *                                                                \
+  prefix##_set(maptype *map, const keytype key, void *val)              \
+  {                                                                     \
+    prefix##_entry_t search;                                            \
+    void *oldval;                                                       \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    tor_assert(val);                                                    \
+    prefix##_assign_tmp_key(&search, key);                              \
+    /* We a lot of our time in this function, so the code below is */   \
+    /* meant to optimize the check/alloc/set cycle by avoiding the two */\
+    /* trips to the hash table that we would do in the unoptimized */   \
+    /* version of this code. (Each of HT_INSERT and HT_FIND calls */     \
+    /* HT_SET_HASH and HT_FIND_P.) */                                   \
+    HT_FIND_OR_INSERT_(prefix##_impl, node, prefix##_entry_hash,        \
+                       &(map->head),                                    \
+                       prefix##_entry_t, &search, ptr,                  \
+                       {                                                \
+                         /* we found an entry. */                       \
+                         oldval = (*ptr)->val;                          \
+                         (*ptr)->val = val;                             \
+                         return oldval;                                 \
+                       },                                               \
+                       {                                                \
+                         /* We didn't find the entry. */                \
+                         prefix##_entry_t *newent =                     \
+                           tor_malloc_zero(sizeof(prefix##_entry_t));   \
+                         prefix##_assign_key(newent, key);              \
+                         newent->val = val;                             \
+                         HT_FOI_INSERT_(node, &(map->head),             \
+                            &search, newent, ptr);                      \
+                         return NULL;                                   \
+    });                                                                 \
+  }                                                                     \
+                                                                        \
+  /** Remove the value currently associated with <b>key</b> from the map. \
+   * Return the value if one was set, or NULL if there was no entry for \
+   * <b>key</b>.                                                        \
+   *                                                                    \
+   * Note: you must free any storage associated with the returned value. \
+   */                                                                   \
+  void *                                                                \
+  prefix##_remove(maptype *map, const keytype key)                      \
+  {                                                                     \
+    prefix##_entry_t *resolve;                                          \
+    prefix##_entry_t search;                                            \
+    void *oldval;                                                       \
+    tor_assert(map);                                                    \
+    tor_assert(key);                                                    \
+    prefix##_assign_tmp_key(&search, key);                              \
+    resolve = HT_REMOVE(prefix##_impl, &map->head, &search);            \
+    if (resolve) {                                                      \
+      oldval = resolve->val;                                            \
+      prefix##_entry_free(resolve);                                     \
+      return oldval;                                                    \
+    } else {                                                            \
+      return NULL;                                                      \
+    }                                                                   \
+  }                                                                     \
+                                                                        \
+  /** Return the number of elements in <b>map</b>. */                   \
+  int                                                                   \
+  prefix##_size(const maptype *map)                                     \
+  {                                                                     \
+    return HT_SIZE(&map->head);                                         \
+  }                                                                     \
+                                                                        \
+  /** Return true iff <b>map</b> has no entries. */                     \
+  int                                                                   \
+  prefix##_isempty(const maptype *map)                                  \
+  {                                                                     \
+    return HT_EMPTY(&map->head);                                        \
+  }                                                                     \
+                                                                        \
+  /** Assert that <b>map</b> is not corrupt. */                         \
+  void                                                                  \
+  prefix##_assert_ok(const maptype *map)                                \
+  {                                                                     \
+    tor_assert(!prefix##_impl_HT_REP_IS_BAD_(&map->head));              \
+  }                                                                     \
+                                                                        \
+  /** Remove all entries from <b>map</b>, and deallocate storage for    \
+   * those entries.  If free_val is provided, invoked it every value in \
+   * <b>map</b>. */                                                     \
+  MOCK_IMPL(void,                                                       \
+  prefix##_free_, (maptype *map, void (*free_val)(void*)))              \
+  {                                                                     \
+    prefix##_entry_t **ent, **next, *this;                              \
+    if (!map)                                                           \
+      return;                                                           \
+    for (ent = HT_START(prefix##_impl, &map->head); ent != NULL;        \
+         ent = next) {                                                  \
+      this = *ent;                                                      \
+      next = HT_NEXT_RMV(prefix##_impl, &map->head, ent);               \
+      if (free_val)                                                     \
+        free_val(this->val);                                            \
+      prefix##_entry_free(this);                                        \
+    }                                                                   \
+    tor_assert(HT_EMPTY(&map->head));                                   \
+    HT_CLEAR(prefix##_impl, &map->head);                                \
+    tor_free(map);                                                      \
+  }                                                                     \
+                                                                        \
+  /** return an <b>iterator</b> pointer to the front of a map.          \
+   *                                                                    \
+   * Iterator example:                                                  \
+   *                                                                    \
+   * \code                                                              \
+   * // uppercase values in "map", removing empty values.               \
+   *                                                                    \
+   * strmap_iter_t *iter;                                               \
+   * const char *key;                                                   \
+   * void *val;                                                         \
+   * char *cp;                                                          \
+   *                                                                    \
+   * for (iter = strmap_iter_init(map); !strmap_iter_done(iter); ) {    \
+   *    strmap_iter_get(iter, &key, &val);                              \
+   *    cp = (char*)val;                                                \
+   *    if (!*cp) {                                                     \
+   *       iter = strmap_iter_next_rmv(map,iter);                       \
+   *       free(val);                                                   \
+   *    } else {                                                        \
+   *       for (;*cp;cp++) *cp = TOR_TOUPPER(*cp);                      \
+   */                                                                   \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_init(maptype *map)                                      \
+  {                                                                     \
+    tor_assert(map);                                                    \
+    return HT_START(prefix##_impl, &map->head);                         \
+  }                                                                     \
+                                                                        \
+  /** Advance <b>iter</b> a single step to the next entry, and return   \
+   * its new value. */                                                  \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_next(maptype *map, prefix##_iter_t *iter)               \
+  {                                                                     \
+    tor_assert(map);                                                    \
+    tor_assert(iter);                                                   \
+    return HT_NEXT(prefix##_impl, &map->head, iter);                    \
+  }                                                                     \
+  /** Advance <b>iter</b> a single step to the next entry, removing the \
+   * current entry, and return its new value. */                        \
+  prefix##_iter_t *                                                     \
+  prefix##_iter_next_rmv(maptype *map, prefix##_iter_t *iter)           \
+  {                                                                     \
+    prefix##_entry_t *rmv;                                              \
+    tor_assert(map);                                                    \
+    tor_assert(iter);                                                   \
+    tor_assert(*iter);                                                  \
+    rmv = *iter;                                                        \
+    iter = HT_NEXT_RMV(prefix##_impl, &map->head, iter);                \
+    prefix##_entry_free(rmv);                                           \
+    return iter;                                                        \
+  }                                                                     \
+  /** Set *<b>keyp</b> and *<b>valp</b> to the current entry pointed    \
+   * to by iter. */                                                     \
+  void                                                                  \
+  prefix##_iter_get(prefix##_iter_t *iter, const keytype *keyp,         \
+                    void **valp)                                        \
+  {                                                                     \
+    tor_assert(iter);                                                   \
+    tor_assert(*iter);                                                  \
+    tor_assert(keyp);                                                   \
+    tor_assert(valp);                                                   \
+    *keyp = (*iter)->key;                                               \
+    *valp = (*iter)->val;                                               \
+  }                                                                     \
+  /** Return true iff <b>iter</b> has advanced past the last entry of   \
+   * <b>map</b>. */                                                     \
+  int                                                                   \
+  prefix##_iter_done(prefix##_iter_t *iter)                             \
+  {                                                                     \
+    return iter == NULL;                                                \
+  }
+
+IMPLEMENT_MAP_FNS(strmap_t, char *, strmap)
+IMPLEMENT_MAP_FNS(digestmap_t, char *, digestmap)
+IMPLEMENT_MAP_FNS(digest256map_t, uint8_t *, digest256map)
+
+/** Same as strmap_set, but first converts <b>key</b> to lowercase. */
+void *
+strmap_set_lc(strmap_t *map, const char *key, void *val)
+{
+  /* We could be a little faster by using strcasecmp instead, and a separate
+   * type, but I don't think it matters. */
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_set(map,lc_key,val);
+  tor_free(lc_key);
+  return v;
+}
+
+/** Same as strmap_get, but first converts <b>key</b> to lowercase. */
+void *
+strmap_get_lc(const strmap_t *map, const char *key)
+{
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_get(map,lc_key);
+  tor_free(lc_key);
+  return v;
+}
+
+/** Same as strmap_remove, but first converts <b>key</b> to lowercase */
+void *
+strmap_remove_lc(strmap_t *map, const char *key)
+{
+  void *v;
+  char *lc_key = tor_strdup(key);
+  tor_strlower(lc_key);
+  v = strmap_remove(map,lc_key);
+  tor_free(lc_key);
+  return v;
+}

src/lib/container/map.h

@@ -0,0 +1,255 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MAP_H
+#define TOR_MAP_H
+
+#include "lib/testsupport/testsupport.h"
+#include "lib/cc/torint.h"
+
+#include "siphash.h"
+
+#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
+  typedef struct maptype maptype;                                       \
+  typedef struct prefix##entry_t *prefix##iter_t;                       \
+  MOCK_DECL(maptype*, prefix##new, (void));                             \
+  void* prefix##set(maptype *map, keytype key, void *val);              \
+  void* prefix##get(const maptype *map, keytype key);                   \
+  void* prefix##remove(maptype *map, keytype key);                      \
+  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
+  int prefix##isempty(const maptype *map);                              \
+  int prefix##size(const maptype *map);                                 \
+  prefix##iter_t *prefix##iter_init(maptype *map);                      \
+  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
+  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
+  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
+  int prefix##iter_done(prefix##iter_t *iter);                          \
+  void prefix##assert_ok(const maptype *map)
+
+/* Map from const char * to void *. Implemented with a hash table. */
+DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
+/* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
+DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
+/* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
+ * table. */
+DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
+
+#define MAP_FREE_AND_NULL(maptype, map, fn)     \
+  do {                                          \
+    maptype ## _free_((map), (fn));             \
+    (map) = NULL;                               \
+  } while (0)
+
+#define strmap_free(map, fn) MAP_FREE_AND_NULL(strmap, (map), (fn))
+#define digestmap_free(map, fn) MAP_FREE_AND_NULL(digestmap, (map), (fn))
+#define digest256map_free(map, fn) MAP_FREE_AND_NULL(digest256map, (map), (fn))
+
+#undef DECLARE_MAP_FNS
+
+/** Iterates over the key-value pairs in a map <b>map</b> in order.
+ * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
+ * The map's keys and values are of type keytype and valtype respectively;
+ * each iteration assigns them to keyvar and valvar.
+ *
+ * Example use:
+ *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *     // use k and r
+ *   } MAP_FOREACH_END.
+ */
+/* Unpacks to, approximately:
+ * {
+ *   digestmap_iter_t *k_iter;
+ *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
+ *        k_iter = digestmap_iter_next(m, k_iter)) {
+ *     const char *k;
+ *     void *r_voidp;
+ *     routerinfo_t *r;
+ *     digestmap_iter_get(k_iter, &k, &r_voidp);
+ *     r = r_voidp;
+ *     // use k and r
+ *   }
+ * }
+ */
+#define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
+  STMT_BEGIN                                                            \
+    prefix##iter_t *keyvar##_iter;                                      \
+    for (keyvar##_iter = prefix##iter_init(map);                        \
+         !prefix##iter_done(keyvar##_iter);                             \
+         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
+      keytype keyvar;                                                   \
+      void *valvar##_voidp;                                             \
+      valtype valvar;                                                   \
+      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      valvar = valvar##_voidp;
+
+/** As MAP_FOREACH, except allows members to be removed from the map
+ * during the iteration via MAP_DEL_CURRENT.  Example use:
+ *
+ * Example use:
+ *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *      if (is_very_old(r))
+ *       MAP_DEL_CURRENT(k);
+ *   } MAP_FOREACH_END.
+ **/
+/* Unpacks to, approximately:
+ * {
+ *   digestmap_iter_t *k_iter;
+ *   int k_del=0;
+ *   for (k_iter = digestmap_iter_init(m); !digestmap_iter_done(k_iter);
+ *        k_iter = k_del ? digestmap_iter_next(m, k_iter)
+ *                       : digestmap_iter_next_rmv(m, k_iter)) {
+ *     const char *k;
+ *     void *r_voidp;
+ *     routerinfo_t *r;
+ *     k_del=0;
+ *     digestmap_iter_get(k_iter, &k, &r_voidp);
+ *     r = r_voidp;
+ *     if (is_very_old(r)) {
+ *       k_del = 1;
+ *     }
+ *   }
+ * }
+ */
+#define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
+  STMT_BEGIN                                                            \
+    prefix##iter_t *keyvar##_iter;                                      \
+    int keyvar##_del=0;                                                 \
+    for (keyvar##_iter = prefix##iter_init(map);                        \
+         !prefix##iter_done(keyvar##_iter);                             \
+         keyvar##_iter = keyvar##_del ?                                 \
+           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
+           prefix##iter_next(map, keyvar##_iter)) {                     \
+      keytype keyvar;                                                   \
+      void *valvar##_voidp;                                             \
+      valtype valvar;                                                   \
+      keyvar##_del=0;                                                   \
+      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      valvar = valvar##_voidp;
+
+/** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
+ * member of the map.  */
+#define MAP_DEL_CURRENT(keyvar)                   \
+  STMT_BEGIN                                      \
+    keyvar##_del = 1;                             \
+  STMT_END
+
+/** Used to end a MAP_FOREACH() block. */
+#define MAP_FOREACH_END } STMT_END ;
+
+/** As MAP_FOREACH, but does not require declaration of prefix or keytype.
+ * Example use:
+ *   DIGESTMAP_FOREACH(m, k, routerinfo_t *, r) {
+ *     // use k and r
+ *   } DIGESTMAP_FOREACH_END.
+ */
+#define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
+  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
+
+/** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
+ * keytype.
+ * Example use:
+ *   DIGESTMAP_FOREACH_MODIFY(m, k, routerinfo_t *, r) {
+ *      if (is_very_old(r))
+ *       MAP_DEL_CURRENT(k);
+ *   } DIGESTMAP_FOREACH_END.
+ */
+#define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
+  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
+/** Used to end a DIGESTMAP_FOREACH() block. */
+#define DIGESTMAP_FOREACH_END MAP_FOREACH_END
+
+#define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
+  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
+#define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
+  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
+                     keyvar, valtype, valvar)
+#define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
+
+#define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
+  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
+#define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
+  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
+#define STRMAP_FOREACH_END MAP_FOREACH_END
+
+void* strmap_set_lc(strmap_t *map, const char *key, void *val);
+void* strmap_get_lc(const strmap_t *map, const char *key);
+void* strmap_remove_lc(strmap_t *map, const char *key);
+
+#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
+  typedef struct maptype maptype;                                       \
+  typedef struct prefix##iter_t *prefix##iter_t;                        \
+  ATTR_UNUSED static inline maptype*                                    \
+  prefix##new(void)                                                     \
+  {                                                                     \
+    return (maptype*)digestmap_new();                                   \
+  }                                                                     \
+  ATTR_UNUSED static inline digestmap_t*                                \
+  prefix##to_digestmap(maptype *map)                                    \
+  {                                                                     \
+    return (digestmap_t*)map;                                           \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##get(maptype *map, const char *key)     \
+  {                                                                     \
+    return (valtype*)digestmap_get((digestmap_t*)map, key);             \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##set(maptype *map, const char *key, valtype *val)              \
+  {                                                                     \
+    return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
+  }                                                                     \
+  ATTR_UNUSED static inline valtype*                                    \
+  prefix##remove(maptype *map, const char *key)                         \
+  {                                                                     \
+    return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
+  {                                                                     \
+    digestmap_free_((digestmap_t*)map, free_val);                       \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##isempty(maptype *map)                                         \
+  {                                                                     \
+    return digestmap_isempty((digestmap_t*)map);                        \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##size(maptype *map)                                            \
+  {                                                                     \
+    return digestmap_size((digestmap_t*)map);                           \
+  }                                                                     \
+  ATTR_UNUSED static inline                                             \
+  prefix##iter_t *prefix##iter_init(maptype *map)                       \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
+  }                                                                     \
+  ATTR_UNUSED static inline                                             \
+  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_next(                       \
+                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
+  }                                                                     \
+  ATTR_UNUSED static inline prefix##iter_t*                             \
+  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
+  {                                                                     \
+    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
+                       (digestmap_t*)map, (digestmap_iter_t*)iter);     \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  prefix##iter_get(prefix##iter_t *iter,                                \
+                   const char **keyp,                                   \
+                   valtype **valp)                                      \
+  {                                                                     \
+    void *v;                                                            \
+    digestmap_iter_get((digestmap_iter_t*) iter, keyp, &v);             \
+    *valp = v;                                                          \
+  }                                                                     \
+  ATTR_UNUSED static inline int                                         \
+  prefix##iter_done(prefix##iter_t *iter)                               \
+  {                                                                     \
+    return digestmap_iter_done((digestmap_iter_t*)iter);                \
+  }
+
+#endif /* !defined(TOR_CONTAINER_H) */

src/lib/container/order.c

@@ -0,0 +1,51 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file container.c
+ * \brief Implements a smartlist (a resizable array) along
+ * with helper functions to use smartlists.  Also includes
+ * hash table implementations of a string-to-void* map, and of
+ * a digest-to-void* map.
+ **/
+
+#include <stdlib.h>
+
+#include "lib/container/order.h"
+#include "lib/log/util_bug.h"
+
+/** Declare a function called <b>funcname</b> that acts as a find_nth_FOO
+ * function for an array of type <b>elt_t</b>*.
+ *
+ * NOTE: The implementation kind of sucks: It's O(n log n), whereas finding
+ * the kth element of an n-element list can be done in O(n).  Then again, this
+ * implementation is not in critical path, and it is obviously correct. */
+#define IMPLEMENT_ORDER_FUNC(funcname, elt_t)                   \
+  static int                                                    \
+  _cmp_ ## elt_t(const void *_a, const void *_b)                \
+  {                                                             \
+    const elt_t *a = _a, *b = _b;                               \
+    if (*a<*b)                                                  \
+      return -1;                                                \
+    else if (*a>*b)                                             \
+      return 1;                                                 \
+    else                                                        \
+      return 0;                                                 \
+  }                                                             \
+  elt_t                                                         \
+  funcname(elt_t *array, int n_elements, int nth)               \
+  {                                                             \
+    tor_assert(nth >= 0);                                       \
+    tor_assert(nth < n_elements);                               \
+    qsort(array, n_elements, sizeof(elt_t), _cmp_ ##elt_t);     \
+    return array[nth];                                          \
+  }
+
+IMPLEMENT_ORDER_FUNC(find_nth_int, int)
+IMPLEMENT_ORDER_FUNC(find_nth_time, time_t)
+IMPLEMENT_ORDER_FUNC(find_nth_double, double)
+IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t)
+IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t)
+IMPLEMENT_ORDER_FUNC(find_nth_long, long)

src/lib/container/order.h

@@ -0,0 +1,54 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_ORDER_H
+#define TOR_ORDER_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+/* These functions, given an <b>array</b> of <b>n_elements</b>, return the
+ * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element;
+ * <b>n_elements</b>-1 gives the highest; and (<b>n_elements</b>-1) / 2 gives
+ * the median.  As a side effect, the elements of <b>array</b> are sorted. */
+int find_nth_int(int *array, int n_elements, int nth);
+time_t find_nth_time(time_t *array, int n_elements, int nth);
+double find_nth_double(double *array, int n_elements, int nth);
+int32_t find_nth_int32(int32_t *array, int n_elements, int nth);
+uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth);
+long find_nth_long(long *array, int n_elements, int nth);
+static inline int
+median_int(int *array, int n_elements)
+{
+  return find_nth_int(array, n_elements, (n_elements-1)/2);
+}
+static inline time_t
+median_time(time_t *array, int n_elements)
+{
+  return find_nth_time(array, n_elements, (n_elements-1)/2);
+}
+static inline double
+median_double(double *array, int n_elements)
+{
+  return find_nth_double(array, n_elements, (n_elements-1)/2);
+}
+static inline uint32_t
+median_uint32(uint32_t *array, int n_elements)
+{
+  return find_nth_uint32(array, n_elements, (n_elements-1)/2);
+}
+static inline int32_t
+median_int32(int32_t *array, int n_elements)
+{
+  return find_nth_int32(array, n_elements, (n_elements-1)/2);
+}
+
+static inline uint32_t
+third_quartile_uint32(uint32_t *array, int n_elements)
+{
+  return find_nth_uint32(array, n_elements, (n_elements*3)/4);
+}
+
+#endif /* !defined(TOR_CONTAINER_H) */

src/common/container.c → src/lib/container/smartlist.c

@@ -11,17 +11,21 @@
  * a digest-to-void* map.
  **/
 
-#include "common/compat.h"
-#include "common/util.h"
-#include "common/torlog.h"
-#include "common/container.h"
-#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/container/smartlist.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/util_string.h"
+#include "lib/string/printf.h"
+
+#include "lib/log/util_bug.h"
 
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
-
 /** All newly allocated smartlists have this capacity. */
 #define SMARTLIST_DEFAULT_CAPACITY 16
 
@@ -113,6 +117,41 @@ smartlist_add_all(smartlist_t *s1, const smartlist_t *s2)
   s1->num_used = (int) new_size;
 }
 
+/** Append a copy of string to sl */
+void
+smartlist_add_strdup(struct smartlist_t *sl, const char *string)
+{
+  char *copy;
+
+  copy = tor_strdup(string);
+
+  smartlist_add(sl, copy);
+}
+
+/** Append the string produced by tor_asprintf(<b>pattern</b>, <b>...</b>)
+ * to <b>sl</b>. */
+void
+smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
+{
+  va_list ap;
+  va_start(ap, pattern);
+  smartlist_add_vasprintf(sl, pattern, ap);
+  va_end(ap);
+}
+
+/** va_list-based backend of smartlist_add_asprintf. */
+void
+smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
+                        va_list args)
+{
+  char *str = NULL;
+
+  tor_vasprintf(&str, pattern, args);
+  tor_assert(str != NULL);
+
+  smartlist_add(sl, str);
+}
+
 /** Remove all elements E from sl such that E==element.  Preserve
  * the order of any elements before E, but elements after E can be
  * rearranged.
@@ -1083,458 +1122,3 @@ smartlist_uniq_digests256(smartlist_t *sl)
 {
   smartlist_uniq(sl, compare_digests256_, tor_free_);
 }
-
-/** Helper: Declare an entry type and a map type to implement a mapping using
- * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
- * entry is declared using the C declaration <b>keydecl</b>.  All functions
- * and types associated with the map get prefixed with <b>prefix</b> */
-#define DEFINE_MAP_STRUCTS(maptype, keydecl, prefix)      \
-  typedef struct prefix ## entry_t {                      \
-    HT_ENTRY(prefix ## entry_t) node;                     \
-    void *val;                                            \
-    keydecl;                                              \
-  } prefix ## entry_t;                                    \
-  struct maptype {                                        \
-    HT_HEAD(prefix ## impl, prefix ## entry_t) head;      \
-  }
-
-DEFINE_MAP_STRUCTS(strmap_t, char *key, strmap_);
-DEFINE_MAP_STRUCTS(digestmap_t, char key[DIGEST_LEN], digestmap_);
-DEFINE_MAP_STRUCTS(digest256map_t, uint8_t key[DIGEST256_LEN], digest256map_);
-
-/** Helper: compare strmap_entry_t objects by key value. */
-static inline int
-strmap_entries_eq(const strmap_entry_t *a, const strmap_entry_t *b)
-{
-  return !strcmp(a->key, b->key);
-}
-
-/** Helper: return a hash value for a strmap_entry_t. */
-static inline unsigned int
-strmap_entry_hash(const strmap_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, strlen(a->key));
-}
-
-/** Helper: compare digestmap_entry_t objects by key value. */
-static inline int
-digestmap_entries_eq(const digestmap_entry_t *a, const digestmap_entry_t *b)
-{
-  return tor_memeq(a->key, b->key, DIGEST_LEN);
-}
-
-/** Helper: return a hash value for a digest_map_t. */
-static inline unsigned int
-digestmap_entry_hash(const digestmap_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, DIGEST_LEN);
-}
-
-/** Helper: compare digestmap_entry_t objects by key value. */
-static inline int
-digest256map_entries_eq(const digest256map_entry_t *a,
-                        const digest256map_entry_t *b)
-{
-  return tor_memeq(a->key, b->key, DIGEST256_LEN);
-}
-
-/** Helper: return a hash value for a digest_map_t. */
-static inline unsigned int
-digest256map_entry_hash(const digest256map_entry_t *a)
-{
-  return (unsigned) siphash24g(a->key, DIGEST256_LEN);
-}
-
-HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq)
-HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq)
-HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
-             digest256map_entry_hash,
-             digest256map_entries_eq)
-HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
-             digest256map_entry_hash,
-             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-#define strmap_entry_free(ent) \
-  FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
-#define digestmap_entry_free(ent) \
-  FREE_AND_NULL(digestmap_entry_t, digestmap_entry_free_, (ent))
-#define digest256map_entry_free(ent) \
-  FREE_AND_NULL(digest256map_entry_t, digest256map_entry_free_, (ent))
-
-static inline void
-strmap_entry_free_(strmap_entry_t *ent)
-{
-  tor_free(ent->key);
-  tor_free(ent);
-}
-static inline void
-digestmap_entry_free_(digestmap_entry_t *ent)
-{
-  tor_free(ent);
-}
-static inline void
-digest256map_entry_free_(digest256map_entry_t *ent)
-{
-  tor_free(ent);
-}
-
-static inline void
-strmap_assign_tmp_key(strmap_entry_t *ent, const char *key)
-{
-  ent->key = (char*)key;
-}
-static inline void
-digestmap_assign_tmp_key(digestmap_entry_t *ent, const char *key)
-{
-  memcpy(ent->key, key, DIGEST_LEN);
-}
-static inline void
-digest256map_assign_tmp_key(digest256map_entry_t *ent, const uint8_t *key)
-{
-  memcpy(ent->key, key, DIGEST256_LEN);
-}
-static inline void
-strmap_assign_key(strmap_entry_t *ent, const char *key)
-{
-  ent->key = tor_strdup(key);
-}
-static inline void
-digestmap_assign_key(digestmap_entry_t *ent, const char *key)
-{
-  memcpy(ent->key, key, DIGEST_LEN);
-}
-static inline void
-digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key)
-{
-  memcpy(ent->key, key, DIGEST256_LEN);
-}
-
-/**
- * Macro: implement all the functions for a map that are declared in
- * container.h by the DECLARE_MAP_FNS() macro.  You must additionally define a
- * prefix_entry_free_() function to free entries (and their keys), a
- * prefix_assign_tmp_key() function to temporarily set a stack-allocated
- * entry to hold a key, and a prefix_assign_key() function to set a
- * heap-allocated entry to hold a key.
- */
-#define IMPLEMENT_MAP_FNS(maptype, keytype, prefix)                     \
-  /** Create and return a new empty map. */                             \
-  MOCK_IMPL(maptype *,                                                  \
-  prefix##_new,(void))                                                  \
-  {                                                                     \
-    maptype *result;                                                    \
-    result = tor_malloc(sizeof(maptype));                               \
-    HT_INIT(prefix##_impl, &result->head);                              \
-    return result;                                                      \
-  }                                                                     \
-                                                                        \
-  /** Return the item from <b>map</b> whose key matches <b>key</b>, or  \
-   * NULL if no such value exists. */                                   \
-  void *                                                                \
-  prefix##_get(const maptype *map, const keytype key)                   \
-  {                                                                     \
-    prefix ##_entry_t *resolve;                                         \
-    prefix ##_entry_t search;                                           \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    prefix ##_assign_tmp_key(&search, key);                             \
-    resolve = HT_FIND(prefix ##_impl, &map->head, &search);             \
-    if (resolve) {                                                      \
-      return resolve->val;                                              \
-    } else {                                                            \
-      return NULL;                                                      \
-    }                                                                   \
-  }                                                                     \
-                                                                        \
-  /** Add an entry to <b>map</b> mapping <b>key</b> to <b>val</b>;      \
-   * return the previous value, or NULL if no such value existed. */     \
-  void *                                                                \
-  prefix##_set(maptype *map, const keytype key, void *val)              \
-  {                                                                     \
-    prefix##_entry_t search;                                            \
-    void *oldval;                                                       \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    tor_assert(val);                                                    \
-    prefix##_assign_tmp_key(&search, key);                              \
-    /* We a lot of our time in this function, so the code below is */   \
-    /* meant to optimize the check/alloc/set cycle by avoiding the two */\
-    /* trips to the hash table that we would do in the unoptimized */   \
-    /* version of this code. (Each of HT_INSERT and HT_FIND calls */     \
-    /* HT_SET_HASH and HT_FIND_P.) */                                   \
-    HT_FIND_OR_INSERT_(prefix##_impl, node, prefix##_entry_hash,        \
-                       &(map->head),                                    \
-                       prefix##_entry_t, &search, ptr,                  \
-                       {                                                \
-                         /* we found an entry. */                       \
-                         oldval = (*ptr)->val;                          \
-                         (*ptr)->val = val;                             \
-                         return oldval;                                 \
-                       },                                               \
-                       {                                                \
-                         /* We didn't find the entry. */                \
-                         prefix##_entry_t *newent =                     \
-                           tor_malloc_zero(sizeof(prefix##_entry_t));   \
-                         prefix##_assign_key(newent, key);              \
-                         newent->val = val;                             \
-                         HT_FOI_INSERT_(node, &(map->head),             \
-                            &search, newent, ptr);                      \
-                         return NULL;                                   \
-    });                                                                 \
-  }                                                                     \
-                                                                        \
-  /** Remove the value currently associated with <b>key</b> from the map. \
-   * Return the value if one was set, or NULL if there was no entry for \
-   * <b>key</b>.                                                        \
-   *                                                                    \
-   * Note: you must free any storage associated with the returned value. \
-   */                                                                   \
-  void *                                                                \
-  prefix##_remove(maptype *map, const keytype key)                      \
-  {                                                                     \
-    prefix##_entry_t *resolve;                                          \
-    prefix##_entry_t search;                                            \
-    void *oldval;                                                       \
-    tor_assert(map);                                                    \
-    tor_assert(key);                                                    \
-    prefix##_assign_tmp_key(&search, key);                              \
-    resolve = HT_REMOVE(prefix##_impl, &map->head, &search);            \
-    if (resolve) {                                                      \
-      oldval = resolve->val;                                            \
-      prefix##_entry_free(resolve);                                     \
-      return oldval;                                                    \
-    } else {                                                            \
-      return NULL;                                                      \
-    }                                                                   \
-  }                                                                     \
-                                                                        \
-  /** Return the number of elements in <b>map</b>. */                   \
-  int                                                                   \
-  prefix##_size(const maptype *map)                                     \
-  {                                                                     \
-    return HT_SIZE(&map->head);                                         \
-  }                                                                     \
-                                                                        \
-  /** Return true iff <b>map</b> has no entries. */                     \
-  int                                                                   \
-  prefix##_isempty(const maptype *map)                                  \
-  {                                                                     \
-    return HT_EMPTY(&map->head);                                        \
-  }                                                                     \
-                                                                        \
-  /** Assert that <b>map</b> is not corrupt. */                         \
-  void                                                                  \
-  prefix##_assert_ok(const maptype *map)                                \
-  {                                                                     \
-    tor_assert(!prefix##_impl_HT_REP_IS_BAD_(&map->head));              \
-  }                                                                     \
-                                                                        \
-  /** Remove all entries from <b>map</b>, and deallocate storage for    \
-   * those entries.  If free_val is provided, invoked it every value in \
-   * <b>map</b>. */                                                     \
-  MOCK_IMPL(void,                                                       \
-  prefix##_free_, (maptype *map, void (*free_val)(void*)))              \
-  {                                                                     \
-    prefix##_entry_t **ent, **next, *this;                              \
-    if (!map)                                                           \
-      return;                                                           \
-    for (ent = HT_START(prefix##_impl, &map->head); ent != NULL;        \
-         ent = next) {                                                  \
-      this = *ent;                                                      \
-      next = HT_NEXT_RMV(prefix##_impl, &map->head, ent);               \
-      if (free_val)                                                     \
-        free_val(this->val);                                            \
-      prefix##_entry_free(this);                                        \
-    }                                                                   \
-    tor_assert(HT_EMPTY(&map->head));                                   \
-    HT_CLEAR(prefix##_impl, &map->head);                                \
-    tor_free(map);                                                      \
-  }                                                                     \
-                                                                        \
-  /** return an <b>iterator</b> pointer to the front of a map.          \
-   *                                                                    \
-   * Iterator example:                                                  \
-   *                                                                    \
-   * \code                                                              \
-   * // uppercase values in "map", removing empty values.               \
-   *                                                                    \
-   * strmap_iter_t *iter;                                               \
-   * const char *key;                                                   \
-   * void *val;                                                         \
-   * char *cp;                                                          \
-   *                                                                    \
-   * for (iter = strmap_iter_init(map); !strmap_iter_done(iter); ) {    \
-   *    strmap_iter_get(iter, &key, &val);                              \
-   *    cp = (char*)val;                                                \
-   *    if (!*cp) {                                                     \
-   *       iter = strmap_iter_next_rmv(map,iter);                       \
-   *       free(val);                                                   \
-   *    } else {                                                        \
-   *       for (;*cp;cp++) *cp = TOR_TOUPPER(*cp);                      \
-   */                                                                   \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_init(maptype *map)                                      \
-  {                                                                     \
-    tor_assert(map);                                                    \
-    return HT_START(prefix##_impl, &map->head);                         \
-  }                                                                     \
-                                                                        \
-  /** Advance <b>iter</b> a single step to the next entry, and return   \
-   * its new value. */                                                  \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_next(maptype *map, prefix##_iter_t *iter)               \
-  {                                                                     \
-    tor_assert(map);                                                    \
-    tor_assert(iter);                                                   \
-    return HT_NEXT(prefix##_impl, &map->head, iter);                    \
-  }                                                                     \
-  /** Advance <b>iter</b> a single step to the next entry, removing the \
-   * current entry, and return its new value. */                        \
-  prefix##_iter_t *                                                     \
-  prefix##_iter_next_rmv(maptype *map, prefix##_iter_t *iter)           \
-  {                                                                     \
-    prefix##_entry_t *rmv;                                              \
-    tor_assert(map);                                                    \
-    tor_assert(iter);                                                   \
-    tor_assert(*iter);                                                  \
-    rmv = *iter;                                                        \
-    iter = HT_NEXT_RMV(prefix##_impl, &map->head, iter);                \
-    prefix##_entry_free(rmv);                                           \
-    return iter;                                                        \
-  }                                                                     \
-  /** Set *<b>keyp</b> and *<b>valp</b> to the current entry pointed    \
-   * to by iter. */                                                     \
-  void                                                                  \
-  prefix##_iter_get(prefix##_iter_t *iter, const keytype *keyp,         \
-                    void **valp)                                        \
-  {                                                                     \
-    tor_assert(iter);                                                   \
-    tor_assert(*iter);                                                  \
-    tor_assert(keyp);                                                   \
-    tor_assert(valp);                                                   \
-    *keyp = (*iter)->key;                                               \
-    *valp = (*iter)->val;                                               \
-  }                                                                     \
-  /** Return true iff <b>iter</b> has advanced past the last entry of   \
-   * <b>map</b>. */                                                     \
-  int                                                                   \
-  prefix##_iter_done(prefix##_iter_t *iter)                             \
-  {                                                                     \
-    return iter == NULL;                                                \
-  }
-
-IMPLEMENT_MAP_FNS(strmap_t, char *, strmap)
-IMPLEMENT_MAP_FNS(digestmap_t, char *, digestmap)
-IMPLEMENT_MAP_FNS(digest256map_t, uint8_t *, digest256map)
-
-/** Same as strmap_set, but first converts <b>key</b> to lowercase. */
-void *
-strmap_set_lc(strmap_t *map, const char *key, void *val)
-{
-  /* We could be a little faster by using strcasecmp instead, and a separate
-   * type, but I don't think it matters. */
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_set(map,lc_key,val);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Same as strmap_get, but first converts <b>key</b> to lowercase. */
-void *
-strmap_get_lc(const strmap_t *map, const char *key)
-{
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_get(map,lc_key);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Same as strmap_remove, but first converts <b>key</b> to lowercase */
-void *
-strmap_remove_lc(strmap_t *map, const char *key)
-{
-  void *v;
-  char *lc_key = tor_strdup(key);
-  tor_strlower(lc_key);
-  v = strmap_remove(map,lc_key);
-  tor_free(lc_key);
-  return v;
-}
-
-/** Declare a function called <b>funcname</b> that acts as a find_nth_FOO
- * function for an array of type <b>elt_t</b>*.
- *
- * NOTE: The implementation kind of sucks: It's O(n log n), whereas finding
- * the kth element of an n-element list can be done in O(n).  Then again, this
- * implementation is not in critical path, and it is obviously correct. */
-#define IMPLEMENT_ORDER_FUNC(funcname, elt_t)                   \
-  static int                                                    \
-  _cmp_ ## elt_t(const void *_a, const void *_b)                \
-  {                                                             \
-    const elt_t *a = _a, *b = _b;                               \
-    if (*a<*b)                                                  \
-      return -1;                                                \
-    else if (*a>*b)                                             \
-      return 1;                                                 \
-    else                                                        \
-      return 0;                                                 \
-  }                                                             \
-  elt_t                                                         \
-  funcname(elt_t *array, int n_elements, int nth)               \
-  {                                                             \
-    tor_assert(nth >= 0);                                       \
-    tor_assert(nth < n_elements);                               \
-    qsort(array, n_elements, sizeof(elt_t), _cmp_ ##elt_t);     \
-    return array[nth];                                          \
-  }
-
-IMPLEMENT_ORDER_FUNC(find_nth_int, int)
-IMPLEMENT_ORDER_FUNC(find_nth_time, time_t)
-IMPLEMENT_ORDER_FUNC(find_nth_double, double)
-IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t)
-IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t)
-IMPLEMENT_ORDER_FUNC(find_nth_long, long)
-
-/** Return a newly allocated digestset_t, optimized to hold a total of
- * <b>max_elements</b> digests with a reasonably low false positive weight. */
-digestset_t *
-digestset_new(int max_elements)
-{
-  /* The probability of false positives is about P=(1 - exp(-kn/m))^k, where k
-   * is the number of hash functions per entry, m is the bits in the array,
-   * and n is the number of elements inserted.  For us, k==4, n<=max_elements,
-   * and m==n_bits= approximately max_elements*32.  This gives
-   *   P<(1-exp(-4*n/(32*n)))^4 == (1-exp(1/-8))^4 == .00019
-   *
-   * It would be more optimal in space vs false positives to get this false
-   * positive rate by going for k==13, and m==18.5n, but we also want to
-   * conserve CPU, and k==13 is pretty big.
-   */
-  int n_bits = 1u << (tor_log2(max_elements)+5);
-  digestset_t *r = tor_malloc(sizeof(digestset_t));
-  r->mask = n_bits - 1;
-  r->ba = bitarray_init_zero(n_bits);
-  return r;
-}
-
-/** Free all storage held in <b>set</b>. */
-void
-digestset_free_(digestset_t *set)
-{
-  if (!set)
-    return;
-  bitarray_free(set->ba);
-  tor_free(set);
-}

src/lib/container/smartlist.h

@@ -0,0 +1,360 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SMARTLIST_H
+#define TOR_SMARTLIST_H
+
+#include <stddef.h>
+#include <stdarg.h>
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/** A resizeable list of pointers, with associated helpful functionality.
+ *
+ * The members of this struct are exposed only so that macros and inlines can
+ * use them; all access to smartlist internals should go through the functions
+ * and macros defined here.
+ **/
+typedef struct smartlist_t {
+  /** @{ */
+  /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements
+   * before it needs to be resized.  Only the first <b>num_used</b> (\<=
+   * capacity) elements point to valid data.
+   */
+  void **list;
+  int num_used;
+  int capacity;
+  /** @} */
+} smartlist_t;
+
+MOCK_DECL(smartlist_t *, smartlist_new, (void));
+MOCK_DECL(void, smartlist_free_, (smartlist_t *sl));
+#define smartlist_free(sl) FREE_AND_NULL(smartlist_t, smartlist_free_, (sl))
+
+void smartlist_clear(smartlist_t *sl);
+void smartlist_add(smartlist_t *sl, void *element);
+void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
+void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
+  CHECK_PRINTF(2, 3);
+void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
+                             va_list args)
+  CHECK_PRINTF(2, 0);
+void smartlist_remove(smartlist_t *sl, const void *element);
+void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
+void *smartlist_pop_last(smartlist_t *sl);
+void smartlist_reverse(smartlist_t *sl);
+void smartlist_string_remove(smartlist_t *sl, const char *element);
+int smartlist_contains(const smartlist_t *sl, const void *element);
+int smartlist_contains_string(const smartlist_t *sl, const char *element);
+int smartlist_pos(const smartlist_t *sl, const void *element);
+int smartlist_string_pos(const smartlist_t *, const char *elt);
+int smartlist_contains_string_case(const smartlist_t *sl, const char *element);
+int smartlist_contains_int_as_string(const smartlist_t *sl, int num);
+int smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2);
+int smartlist_contains_digest(const smartlist_t *sl, const char *element);
+int smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2);
+int smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2);
+void smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2);
+void smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2);
+
+/* smartlist_choose() is defined in crypto.[ch] */
+#ifdef DEBUG_SMARTLIST
+#include "lib/err/torerr.h"
+#include <stdlib.h>
+/** Return the number of items in sl.
+ */
+static inline int smartlist_len(const smartlist_t *sl);
+static inline int smartlist_len(const smartlist_t *sl) {
+  raw_assert(sl);
+  return (sl)->num_used;
+}
+/** Return the <b>idx</b>th element of sl.
+ */
+static inline void *smartlist_get(const smartlist_t *sl, int idx);
+static inline void *smartlist_get(const smartlist_t *sl, int idx) {
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(sl->num_used > idx);
+  return sl->list[idx];
+}
+static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
+  raw_assert(sl);
+  raw_assert(idx>=0);
+  raw_assert(sl->num_used > idx);
+  sl->list[idx] = val;
+}
+#else /* !(defined(DEBUG_SMARTLIST)) */
+#define smartlist_len(sl) ((sl)->num_used)
+#define smartlist_get(sl, idx) ((sl)->list[idx])
+#define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
+#endif /* defined(DEBUG_SMARTLIST) */
+
+/** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the
+ * smartlist <b>sl</b>. */
+static inline void smartlist_swap(smartlist_t *sl, int idx1, int idx2)
+{
+  if (idx1 != idx2) {
+    void *elt = smartlist_get(sl, idx1);
+    smartlist_set(sl, idx1, smartlist_get(sl, idx2));
+    smartlist_set(sl, idx2, elt);
+  }
+}
+
+void smartlist_del(smartlist_t *sl, int idx);
+void smartlist_del_keeporder(smartlist_t *sl, int idx);
+void smartlist_insert(smartlist_t *sl, int idx, void *val);
+void smartlist_sort(smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b));
+void *smartlist_get_most_frequent_(const smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b),
+                    int *count_out);
+#define smartlist_get_most_frequent(sl, compare) \
+  smartlist_get_most_frequent_((sl), (compare), NULL)
+void smartlist_uniq(smartlist_t *sl,
+                    int (*compare)(const void **a, const void **b),
+                    void (*free_fn)(void *elt));
+
+void smartlist_sort_strings(smartlist_t *sl);
+void smartlist_sort_digests(smartlist_t *sl);
+void smartlist_sort_digests256(smartlist_t *sl);
+void smartlist_sort_pointers(smartlist_t *sl);
+
+const char *smartlist_get_most_frequent_string(smartlist_t *sl);
+const char *smartlist_get_most_frequent_string_(smartlist_t *sl,
+                                                int *count_out);
+const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl);
+
+void smartlist_uniq_strings(smartlist_t *sl);
+void smartlist_uniq_digests(smartlist_t *sl);
+void smartlist_uniq_digests256(smartlist_t *sl);
+void *smartlist_bsearch(smartlist_t *sl, const void *key,
+                        int (*compare)(const void *key, const void **member));
+int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
+                          int (*compare)(const void *key, const void **member),
+                          int *found_out);
+
+void smartlist_pqueue_add(smartlist_t *sl,
+                          int (*compare)(const void *a, const void *b),
+                          int idx_field_offset,
+                          void *item);
+void *smartlist_pqueue_pop(smartlist_t *sl,
+                           int (*compare)(const void *a, const void *b),
+                           int idx_field_offset);
+void smartlist_pqueue_remove(smartlist_t *sl,
+                             int (*compare)(const void *a, const void *b),
+                             int idx_field_offset,
+                             void *item);
+void smartlist_pqueue_assert_ok(smartlist_t *sl,
+                                int (*compare)(const void *a, const void *b),
+                                int idx_field_offset);
+
+#define SPLIT_SKIP_SPACE   0x01
+#define SPLIT_IGNORE_BLANK 0x02
+#define SPLIT_STRIP_SPACE  0x04
+int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
+                           int flags, int max);
+char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
+                             size_t *len_out) ATTR_MALLOC;
+char *smartlist_join_strings2(smartlist_t *sl, const char *join,
+                              size_t join_len, int terminate, size_t *len_out)
+  ATTR_MALLOC;
+
+/** Iterate over the items in a smartlist <b>sl</b>, in order.  For each item,
+ * assign it to a new local variable of type <b>type</b> named <b>var</b>, and
+ * execute the statements inside the loop body.  Inside the loop, the loop
+ * index can be accessed as <b>var</b>_sl_idx and the length of the list can
+ * be accessed as <b>var</b>_sl_len.
+ *
+ * NOTE: Do not change the length of the list while the loop is in progress,
+ * unless you adjust the _sl_len variable correspondingly.  See second example
+ * below.
+ *
+ * Example use:
+ * <pre>
+ *   smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
+ *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
+ *     printf("%d: %s\n", cp_sl_idx, cp);
+ *     tor_free(cp);
+ *   } SMARTLIST_FOREACH_END(cp);
+ *   smartlist_free(list);
+ * </pre>
+ *
+ * Example use (advanced):
+ * <pre>
+ *   SMARTLIST_FOREACH_BEGIN(list, char *, cp) {
+ *     if (!strcmp(cp, "junk")) {
+ *       tor_free(cp);
+ *       SMARTLIST_DEL_CURRENT(list, cp);
+ *     }
+ *   } SMARTLIST_FOREACH_END(cp);
+ * </pre>
+ */
+/* Note: these macros use token pasting, and reach into smartlist internals.
+ * This can make them a little daunting. Here's the approximate unpacking of
+ * the above examples, for entertainment value:
+ *
+ * <pre>
+ * smartlist_t *list = smartlist_split("A:B:C", ":", 0, 0);
+ * {
+ *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
+ *   char *cp;
+ *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
+ *     cp = smartlist_get(list, cp_sl_idx);
+ *     printf("%d: %s\n", cp_sl_idx, cp);
+ *     tor_free(cp);
+ *   }
+ * }
+ * smartlist_free(list);
+ * </pre>
+ *
+ * <pre>
+ * {
+ *   int cp_sl_idx, cp_sl_len = smartlist_len(list);
+ *   char *cp;
+ *   for (cp_sl_idx = 0; cp_sl_idx < cp_sl_len; ++cp_sl_idx) {
+ *     cp = smartlist_get(list, cp_sl_idx);
+ *     if (!strcmp(cp, "junk")) {
+ *       tor_free(cp);
+ *       smartlist_del(list, cp_sl_idx);
+ *       --cp_sl_idx;
+ *       --cp_sl_len;
+ *     }
+ *   }
+ * }
+ * </pre>
+ */
+#define SMARTLIST_FOREACH_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                    \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;          \
+    type var;                                                   \
+    for (var ## _sl_idx = 0; var ## _sl_idx < var ## _sl_len;   \
+         ++var ## _sl_idx) {                                    \
+      var = (sl)->list[var ## _sl_idx];
+
+#define SMARTLIST_FOREACH_END(var)              \
+    var = NULL;                                 \
+    (void) var ## _sl_idx;                      \
+  } STMT_END
+
+/**
+ * An alias for SMARTLIST_FOREACH_BEGIN and SMARTLIST_FOREACH_END, using
+ * <b>cmd</b> as the loop body.  This wrapper is here for convenience with
+ * very short loops.
+ *
+ * By convention, we do not use this for loops which nest, or for loops over
+ * 10 lines or so.  Use SMARTLIST_FOREACH_{BEGIN,END} for those.
+ */
+#define SMARTLIST_FOREACH(sl, type, var, cmd)                   \
+  SMARTLIST_FOREACH_BEGIN(sl,type,var) {                        \
+    cmd;                                                        \
+  } SMARTLIST_FOREACH_END(var)
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, remove the current element in a way that
+ * won't confuse the loop. */
+#define SMARTLIST_DEL_CURRENT(sl, var)          \
+  STMT_BEGIN                                    \
+    smartlist_del(sl, var ## _sl_idx);          \
+    --var ## _sl_idx;                           \
+    --var ## _sl_len;                           \
+  STMT_END
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, remove the current element in a way that
+ * won't confuse the loop. */
+#define SMARTLIST_DEL_CURRENT_KEEPORDER(sl, var)          \
+  STMT_BEGIN                                              \
+     smartlist_del_keeporder(sl, var ## _sl_idx);         \
+     --var ## _sl_idx;                                    \
+     --var ## _sl_len;                                    \
+  STMT_END
+
+/** Helper: While in a SMARTLIST_FOREACH loop over the list <b>sl</b> indexed
+ * with the variable <b>var</b>, replace the current element with <b>val</b>.
+ * Does not deallocate the current value of <b>var</b>.
+ */
+#define SMARTLIST_REPLACE_CURRENT(sl, var, val) \
+  STMT_BEGIN                                    \
+    smartlist_set(sl, var ## _sl_idx, val);     \
+  STMT_END
+
+/* Helper: Given two lists of items, possibly of different types, such that
+ * both lists are sorted on some common field (as determined by a comparison
+ * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
+ * duplicates on the common field, loop through the lists in lockstep, and
+ * execute <b>unmatched_var2</b> on items in var2 that do not appear in
+ * var1.
+ *
+ * WARNING: It isn't safe to add remove elements from either list while the
+ * loop is in progress.
+ *
+ * Example use:
+ *  SMARTLIST_FOREACH_JOIN(routerstatus_list, routerstatus_t *, rs,
+ *                     routerinfo_list, routerinfo_t *, ri,
+ *                    tor_memcmp(rs->identity_digest, ri->identity_digest, 20),
+ *                     log_info(LD_GENERAL,"No match for %s", ri->nickname)) {
+ *    log_info(LD_GENERAL, "%s matches routerstatus %p", ri->nickname, rs);
+ * } SMARTLIST_FOREACH_JOIN_END(rs, ri);
+ **/
+/* The example above unpacks (approximately) to:
+ *  int rs_sl_idx = 0, rs_sl_len = smartlist_len(routerstatus_list);
+ *  int ri_sl_idx, ri_sl_len = smartlist_len(routerinfo_list);
+ *  int rs_ri_cmp;
+ *  routerstatus_t *rs;
+ *  routerinfo_t *ri;
+ *  for (; ri_sl_idx < ri_sl_len; ++ri_sl_idx) {
+ *    ri = smartlist_get(routerinfo_list, ri_sl_idx);
+ *    while (rs_sl_idx < rs_sl_len) {
+ *      rs = smartlist_get(routerstatus_list, rs_sl_idx);
+ *      rs_ri_cmp = tor_memcmp(rs->identity_digest, ri->identity_digest, 20);
+ *      if (rs_ri_cmp > 0) {
+ *        break;
+ *      } else if (rs_ri_cmp == 0) {
+ *        goto matched_ri;
+ *      } else {
+ *        ++rs_sl_idx;
+ *      }
+ *    }
+ *    log_info(LD_GENERAL,"No match for %s", ri->nickname);
+ *    continue;
+ *   matched_ri: {
+ *    log_info(LD_GENERAL,"%s matches with routerstatus %p",ri->nickname,rs);
+ *    }
+ *  }
+ */
+#define SMARTLIST_FOREACH_JOIN(sl1, type1, var1, sl2, type2, var2,      \
+                                cmpexpr, unmatched_var2)                \
+  STMT_BEGIN                                                            \
+  int var1 ## _sl_idx = 0, var1 ## _sl_len=(sl1)->num_used;             \
+  int var2 ## _sl_idx = 0, var2 ## _sl_len=(sl2)->num_used;             \
+  int var1 ## _ ## var2 ## _cmp;                                        \
+  type1 var1;                                                           \
+  type2 var2;                                                           \
+  for (; var2##_sl_idx < var2##_sl_len; ++var2##_sl_idx) {              \
+    var2 = (sl2)->list[var2##_sl_idx];                                  \
+    while (var1##_sl_idx < var1##_sl_len) {                             \
+      var1 = (sl1)->list[var1##_sl_idx];                                \
+      var1##_##var2##_cmp = (cmpexpr);                                  \
+      if (var1##_##var2##_cmp > 0) {                                    \
+        break;                                                          \
+      } else if (var1##_##var2##_cmp == 0) {                            \
+        goto matched_##var2;                                            \
+      } else {                                                          \
+        ++var1##_sl_idx;                                                \
+      }                                                                 \
+    }                                                                   \
+    /* Ran out of v1, or no match for var2. */                          \
+    unmatched_var2;                                                     \
+    continue;                                                           \
+    matched_##var2: ;                                                   \
+
+#define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
+  }                                             \
+  STMT_END
+
+#endif /* !defined(TOR_CONTAINER_H) */

src/lib/crypt_ops/.may_include

@@ -1,14 +1,22 @@
 orconfig.h
 lib/cc/*.h
+lib/container/*.h
 lib/crypt_ops/*.h
 lib/ctime/*.h
+lib/defs/*.h
+lib/malloc/*.h
 lib/err/*.h
+lib/intmath/*.h
+lib/string/*.h
 lib/testsupport/testsupport.h
+lib/log/*.h
 
 trunnel/pwbox.h
 
 keccak-tiny/*.h
 ed25519/*.h
 
+siphash.h
+
 # XXX I'd like to remove this.
 common/*.h

src/lib/crypt_ops/aes.c

@@ -38,7 +38,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "common/compat.h"
 #include "lib/crypt_ops/aes.h"
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/ctime/di_ops.h"
 
 #ifdef ANDROID

src/lib/crypt_ops/crypto.c

@@ -62,17 +62,18 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include <unistd.h>
 #endif
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "lib/cc/torint.h"
 #include "lib/crypt_ops/aes.h"
 #include "common/util.h"
-#include "common/container.h"
 #include "common/compat.h"
 #include "common/sandbox.h"
 #include "common/util_format.h"
 
 #include "keccak-tiny/keccak-tiny.h"
 
+#include "siphash.h"
+
 /** Boolean: has OpenSSL's crypto been initialized? */
 static int crypto_early_initialized_ = 0;
 

src/lib/crypt_ops/crypto_curve25519.c

@@ -20,14 +20,13 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#include "common/container.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_digest.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 #include "ed25519/donna/ed25519_donna_tor.h"
 

src/lib/crypt_ops/crypto_dh.c

@@ -23,7 +23,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 
 #include <openssl/bn.h>
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake
  * while we're waiting for the second.*/

src/lib/crypt_ops/crypto_digest.c

@@ -10,11 +10,11 @@
  * operations.
  **/
 
-#include "common/container.h"
+#include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_digest.h"
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 #include "keccak-tiny/keccak-tiny.h"
 
@@ -580,4 +580,3 @@ crypto_xof_free_(crypto_xof_t *xof)
   memwipe(xof, 0, sizeof(crypto_xof_t));
   tor_free(xof);
 }
-

src/lib/crypt_ops/crypto_digest.h

@@ -13,18 +13,9 @@
 #ifndef TOR_CRYPTO_DIGEST_H
 #define TOR_CRYPTO_DIGEST_H
 
-#include <stdio.h>
-
-#include "common/container.h"
 #include "lib/cc/torint.h"
-
-/** Length of the output of our message digest. */
-#define DIGEST_LEN 20
-/** Length of the output of our second (improved) message digests.  (For now
- * this is just sha256, but it could be any other 256-bit digest.) */
-#define DIGEST256_LEN 32
-/** Length of the output of our 64-bit optimized message digests (SHA512). */
-#define DIGEST512_LEN 64
+#include "lib/defs/digest_sizes.h"
+#include "lib/malloc/util_malloc.h"
 
 /** Length of a sha1 message digest when encoded in base32 with trailing =
  * signs removed. */
@@ -78,6 +69,8 @@ typedef struct {
 typedef struct crypto_digest_t crypto_digest_t;
 typedef struct crypto_xof_t crypto_xof_t;
 
+struct smartlist_t;
+
 /* SHA-1 and other digests */
 int crypto_digest(char *digest, const char *m, size_t len);
 int crypto_digest256(char *digest, const char *m, size_t len,
@@ -133,4 +126,3 @@ digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
 #endif
 
 #endif /* !defined(TOR_CRYPTO_DIGEST_H) */
-

src/lib/crypt_ops/crypto_ed25519.c

@@ -27,7 +27,7 @@
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 #include "common/util_format.h"
 

src/lib/crypt_ops/crypto_format.c

@@ -14,15 +14,16 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#include "common/container.h"
+#include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_digest.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
+#include "lib/string/util_string.h"
 #include "common/util.h"
 #include "common/util_format.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 /** Write the <b>datalen</b> bytes from <b>data</b> to the file named
  * <b>fname</b> in the tagged-data format.  This format contains a
@@ -296,4 +297,3 @@ digest256_from_base64(char *digest, const char *d64)
   else
     return -1;
 }
-

src/lib/crypt_ops/crypto_openssl_mgt.c

@@ -12,6 +12,7 @@
 
 #include "lib/crypt_ops/compat_openssl.h"
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
+#include "lib/string/util_string.h"
 
 DISABLE_GCC_WARNING(redundant-decls)
 
@@ -158,4 +159,3 @@ crypto_openssl_free_all(void)
   }
 #endif /* !defined(NEW_THREAD_API) */
 }
-

src/lib/crypt_ops/crypto_pwbox.c

@@ -15,6 +15,7 @@
 #include "lib/crypt_ops/crypto_s2k.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/ctime/di_ops.h"
+#include "lib/intmath/muldiv.h"
 #include "common/util.h"
 #include "trunnel/pwbox.h"
 
@@ -212,4 +213,3 @@ crypto_unpwbox(uint8_t **out, size_t *outlen_out,
   memwipe(keys, 0, sizeof(keys));
   return rv;
 }
-

src/lib/crypt_ops/crypto_rand.c

@@ -21,13 +21,13 @@
 #include <wincrypt.h>
 #endif /* defined(_WIN32) */
 
-#include "common/container.h"
+#include "lib/container/smartlist.h"
 #include "common/compat.h"
 #include "lib/crypt_ops/compat_openssl.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "common/sandbox.h"
 #include "lib/testsupport/testsupport.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 #include "common/util_format.h"
 
@@ -612,4 +612,3 @@ crypto_force_rand_ssleay(void)
 }
 
 #endif /* !defined(CRYPTO_RAND_PRIVATE) */
-

src/lib/crypt_ops/crypto_rsa.c

@@ -33,7 +33,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 
 ENABLE_GCC_WARNING(redundant-decls)
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 #include "common/util.h"
 #include "common/util_format.h"
 

src/lib/crypt_ops/crypto_rsa.h

@@ -21,7 +21,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "common/compat.h"
 #include "common/util.h"
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 /** Length of our public keys. */
 #define PK_BYTES (1024/8)

src/lib/crypt_ops/crypto_util.c

@@ -32,7 +32,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 
 ENABLE_GCC_WARNING(redundant-decls)
 
-#include "common/torlog.h"
+#include "lib/log/torlog.h"
 
 /**
  * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to

src/lib/ctime/.may_include

@@ -1,6 +1,5 @@
 orconfig.h
 lib/cc/*.h
 lib/ctime/*.h
-
-# XXXX I'd like to remove this
-common/util.h
+lib/err/*.h
+lib/malloc/*.h

src/lib/ctime/di_ops.c

@@ -8,7 +8,10 @@
 
 #include "orconfig.h"
 #include "lib/ctime/di_ops.h"
-#include "common/util.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/util_malloc.h"
+
+#include <string.h>
 
 /**
  * Timing-safe version of memcmp.  As memcmp, compare the <b>sz</b> bytes at
@@ -170,8 +173,8 @@ dimap_add_entry(di_digest256_map_t **map,
   di_digest256_map_t *new_ent;
   {
     void *old_val = dimap_search(*map, key, NULL);
-    tor_assert(! old_val);
-    tor_assert(val);
+    raw_assert(! old_val);
+    raw_assert(val);
   }
   new_ent = tor_malloc_zero(sizeof(di_digest256_map_t));
   new_ent->next = *map;
@@ -263,10 +266,10 @@ select_array_member_cumulative_timei(const uint64_t *entries, int n_entries,
       rand_val = INT64_MAX;
     }
   }
-  tor_assert(total_so_far == total);
-  tor_assert(n_chosen == 1);
-  tor_assert(i_chosen >= 0);
-  tor_assert(i_chosen < n_entries);
+  raw_assert(total_so_far == total);
+  raw_assert(n_chosen == 1);
+  raw_assert(i_chosen >= 0);
+  raw_assert(i_chosen < n_entries);
 
   return i_chosen;
 }

src/lib/defs/.may_include

@@ -0,0 +1 @@
+orconfig.h

src/lib/defs/digest_sizes.h

@@ -0,0 +1,18 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIGEST_SIZES_H
+#define TOR_DIGEST_SIZES_H
+
+/** Length of the output of our message digest. */
+#define DIGEST_LEN 20
+/** Length of the output of our second (improved) message digests.  (For now
+ * this is just sha256, but it could be any other 256-bit digest.) */
+#define DIGEST256_LEN 32
+/** Length of the output of our 64-bit optimized message digests (SHA512). */
+#define DIGEST512_LEN 64
+
+#endif

src/lib/defs/include.am

@@ -0,0 +1,3 @@
+
+noinst_HEADERS += \
+	src/lib/defs/digest_sizes.h

src/lib/fdio/.may_include

@@ -0,0 +1,4 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/fdio/*.h

src/lib/fdio/fdio.c

@@ -0,0 +1,109 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include "lib/fdio/fdio.h"
+#include "lib/cc/torint.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+/** @{ */
+/** Some old versions of Unix didn't define constants for these values,
+ * and instead expect you to say 0, 1, or 2. */
+#ifndef SEEK_SET
+#define SEEK_SET 0
+#endif
+#ifndef SEEK_CUR
+#define SEEK_CUR 1
+#endif
+#ifndef SEEK_END
+#define SEEK_END 2
+#endif
+/** @} */
+
+/** Return the position of <b>fd</b> with respect to the start of the file. */
+off_t
+tor_fd_getpos(int fd)
+{
+#ifdef _WIN32
+  return (off_t) _lseek(fd, 0, SEEK_CUR);
+#else
+  return (off_t) lseek(fd, 0, SEEK_CUR);
+#endif
+}
+
+/** Move <b>fd</b> to the end of the file. Return -1 on error, 0 on success.
+ * If the file is a pipe, do nothing and succeed.
+ **/
+int
+tor_fd_seekend(int fd)
+{
+#ifdef _WIN32
+  return _lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
+#else
+  off_t rc = lseek(fd, 0, SEEK_END) < 0 ? -1 : 0;
+#ifdef ESPIPE
+  /* If we get an error and ESPIPE, then it's a pipe or a socket of a fifo:
+   * no need to worry. */
+  if (rc < 0 && errno == ESPIPE)
+    rc = 0;
+#endif /* defined(ESPIPE) */
+  return (rc < 0) ? -1 : 0;
+#endif /* defined(_WIN32) */
+}
+
+/** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0
+ * on success. */
+int
+tor_fd_setpos(int fd, off_t pos)
+{
+#ifdef _WIN32
+  return _lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
+#else
+  return lseek(fd, pos, SEEK_SET) < 0 ? -1 : 0;
+#endif
+}
+
+/** Replacement for ftruncate(fd, 0): move to the front of the file and remove
+ * all the rest of the file. Return -1 on error, 0 on success. */
+int
+tor_ftruncate(int fd)
+{
+  /* Rumor has it that some versions of ftruncate do not move the file pointer.
+   */
+  if (tor_fd_setpos(fd, 0) < 0)
+    return -1;
+
+#ifdef _WIN32
+  return _chsize(fd, 0);
+#else
+  return ftruncate(fd, 0);
+#endif
+}
+
+/** Minimal version of write_all, for use by logging. */
+int
+write_all_to_fd(int fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written < count) {
+    ssize_t result = write(fd, buf+written, count-written);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return 0;
+}

src/lib/fdio/fdio.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_FDIO_H
+#define TOR_FDIO_H
+
+#include <stddef.h>
+
+off_t tor_fd_getpos(int fd);
+int tor_fd_setpos(int fd, off_t pos);
+int tor_fd_seekend(int fd);
+int tor_ftruncate(int fd);
+int write_all_to_fd(int fd, const char *buf, size_t count);
+
+#endif /* !defined(TOR_FDIO_H) */

src/lib/fdio/include.am

@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-fdio.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
+endif
+
+src_lib_libtor_fdio_a_SOURCES =			\
+	src/lib/fdio/fdio.c
+
+src_lib_libtor_fdio_testing_a_SOURCES = \
+	$(src_lib_libtor_fdio_a_SOURCES)
+src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/fdio/fdio.h

src/lib/intmath/.may_include

@@ -0,0 +1,4 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/intmath/*.h

src/lib/intmath/addsub.c

@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/intmath/addsub.h"
+#include "lib/cc/compat_compiler.h"
+
+/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather
+ * than overflow */
+uint32_t
+tor_add_u32_nowrap(uint32_t a, uint32_t b)
+{
+  /* a+b > UINT32_MAX check, without overflow */
+  if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) {
+    return UINT32_MAX;
+  } else {
+    return a+b;
+  }
+}

src/lib/intmath/addsub.h

@@ -0,0 +1,13 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_INTMATH_ADDSUB_H
+#define TOR_INTMATH_ADDSUB_H
+
+#include "lib/cc/torint.h"
+
+uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
+
+#endif /* !defined(TOR_INTMATH_MULDIV_H) */

src/lib/intmath/bits.c

@@ -0,0 +1,88 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/intmath/bits.h"
+
+/** Returns floor(log2(u64)).  If u64 is 0, (incorrectly) returns 0. */
+int
+tor_log2(uint64_t u64)
+{
+  int r = 0;
+  if (u64 >= (U64_LITERAL(1)<<32)) {
+    u64 >>= 32;
+    r = 32;
+  }
+  if (u64 >= (U64_LITERAL(1)<<16)) {
+    u64 >>= 16;
+    r += 16;
+  }
+  if (u64 >= (U64_LITERAL(1)<<8)) {
+    u64 >>= 8;
+    r += 8;
+  }
+  if (u64 >= (U64_LITERAL(1)<<4)) {
+    u64 >>= 4;
+    r += 4;
+  }
+  if (u64 >= (U64_LITERAL(1)<<2)) {
+    u64 >>= 2;
+    r += 2;
+  }
+  if (u64 >= (U64_LITERAL(1)<<1)) {
+    // u64 >>= 1; // not using this any more.
+    r += 1;
+  }
+  return r;
+}
+
+/** Return the power of 2 in range [1,UINT64_MAX] closest to <b>u64</b>.  If
+ * there are two powers of 2 equally close, round down. */
+uint64_t
+round_to_power_of_2(uint64_t u64)
+{
+  int lg2;
+  uint64_t low;
+  uint64_t high;
+  if (u64 == 0)
+    return 1;
+
+  lg2 = tor_log2(u64);
+  low = U64_LITERAL(1) << lg2;
+
+  if (lg2 == 63)
+    return low;
+
+  high = U64_LITERAL(1) << (lg2+1);
+  if (high - u64 < u64 - low)
+    return high;
+  else
+    return low;
+}
+
+/** Return the number of bits set in <b>v</b>. */
+int
+n_bits_set_u8(uint8_t v)
+{
+  static const int nybble_table[] = {
+    0, /* 0000 */
+    1, /* 0001 */
+    1, /* 0010 */
+    2, /* 0011 */
+    1, /* 0100 */
+    2, /* 0101 */
+    2, /* 0110 */
+    3, /* 0111 */
+    1, /* 1000 */
+    2, /* 1001 */
+    2, /* 1010 */
+    3, /* 1011 */
+    2, /* 1100 */
+    3, /* 1101 */
+    3, /* 1110 */
+    4, /* 1111 */
+  };
+
+  return nybble_table[v & 15] + nybble_table[v>>4];
+}

src/lib/intmath/bits.h

@@ -0,0 +1,16 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_INTMATH_BITS_H
+#define TOR_INTMATH_BITS_H
+
+#include "lib/cc/torint.h"
+#include "lib/cc/compat_compiler.h"
+
+int tor_log2(uint64_t u64) ATTR_CONST;
+uint64_t round_to_power_of_2(uint64_t u64);
+int n_bits_set_u8(uint8_t v);
+
+#endif /* !defined(TOR_INTMATH_BITS_H) */

src/lib/intmath/cmp.h

@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_INTMATH_CMP_H
+#define TOR_INTMATH_CMP_H
+
+/* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
+ * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
+ * <b>b</b> is larger than <b>max</b>.
+ *
+ * Requires that <b>min</b> is no more than <b>max</b>. May evaluate any of
+ * its arguments more than once! */
+#define CLAMP(min,v,max)                        \
+  ( ((v) < (min)) ? (min) :                     \
+    ((v) > (max)) ? (max) :                     \
+    (v) )
+
+#endif /* !defined(TOR_INTMATH_CMP_H) */

src/lib/intmath/include.am

@@ -0,0 +1,22 @@
+
+noinst_LIBRARIES += src/lib/libtor-intmath.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
+endif
+
+src_lib_libtor_intmath_a_SOURCES =			\
+	src/lib/intmath/addsub.c			\
+	src/lib/intmath/bits.c				\
+	src/lib/intmath/muldiv.c
+
+src_lib_libtor_intmath_testing_a_SOURCES = \
+	$(src_lib_libtor_intmath_a_SOURCES)
+src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/intmath/addsub.h			\
+	src/lib/intmath/cmp.h				\
+	src/lib/intmath/bits.h				\
+	src/lib/intmath/muldiv.h

src/lib/intmath/muldiv.c

@@ -0,0 +1,75 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/intmath/muldiv.h"
+#include "lib/err/torerr.h"
+
+#include <stdlib.h>
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0.  If no such x can be expressed as an unsigned, return
+ * UINT_MAX. Asserts if divisor is zero. */
+unsigned
+round_to_next_multiple_of(unsigned number, unsigned divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT_MAX - divisor + 1 < number)
+    return UINT_MAX;
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return
+ * UINT32_MAX. Asserts if divisor is zero. */
+uint32_t
+round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT32_MAX - divisor + 1 < number)
+    return UINT32_MAX;
+
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/** Return the lowest x such that x is at least <b>number</b>, and x modulo
+ * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return
+ * UINT64_MAX. Asserts if divisor is zero. */
+uint64_t
+round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
+{
+  raw_assert(divisor > 0);
+  if (UINT64_MAX - divisor + 1 < number)
+    return UINT64_MAX;
+  number += divisor - 1;
+  number -= number % divisor;
+  return number;
+}
+
+/* Helper: return greatest common divisor of a,b */
+static uint64_t
+gcd64(uint64_t a, uint64_t b)
+{
+  while (b) {
+    uint64_t t = b;
+    b = a % b;
+    a = t;
+  }
+  return a;
+}
+
+/* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it.
+ * Requires that the denominator is greater than 0. */
+void
+simplify_fraction64(uint64_t *numer, uint64_t *denom)
+{
+  raw_assert(denom);
+  uint64_t gcd = gcd64(*numer, *denom);
+  *numer /= gcd;
+  *denom /= gcd;
+}

src/lib/intmath/muldiv.h

@@ -0,0 +1,22 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_INTMATH_MULDIV_H
+#define TOR_INTMATH_MULDIV_H
+
+#include "lib/cc/torint.h"
+
+unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
+uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
+uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
+
+void simplify_fraction64(uint64_t *numer, uint64_t *denom);
+
+/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
+ * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
+ * can overflow. */
+#define CEIL_DIV(a,b) (((a)+((b)-1))/(b))
+
+#endif /* !defined(TOR_INTMATH_MULDIV_H) */

src/lib/lock/.may_include

@@ -0,0 +1,5 @@
+orconfig.h
+lib/cc/*.h
+lib/err/*.h
+lib/lock/*.h
+lib/malloc/*.h

src/lib/lock/compat_mutex.c

@@ -0,0 +1,34 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/malloc/util_malloc.h"
+
+/** Return a newly allocated, ready-for-use mutex. */
+tor_mutex_t *
+tor_mutex_new(void)
+{
+  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
+  tor_mutex_init(m);
+  return m;
+}
+/** Return a newly allocated, ready-for-use mutex.  This one might be
+ * non-recursive, if that's faster. */
+tor_mutex_t *
+tor_mutex_new_nonrecursive(void)
+{
+  tor_mutex_t *m = tor_malloc_zero(sizeof(tor_mutex_t));
+  tor_mutex_init_nonrecursive(m);
+  return m;
+}
+/** Release all storage and system resources held by <b>m</b>. */
+void
+tor_mutex_free_(tor_mutex_t *m)
+{
+  if (!m)
+    return;
+  tor_mutex_uninit(m);
+  tor_free(m);
+}

src/lib/lock/compat_mutex.h

@@ -0,0 +1,60 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_COMPAT_MUTEX_H
+#define TOR_COMPAT_MUTEX_H
+
+#include "orconfig.h"
+#include "lib/cc/torint.h"
+#include "lib/malloc/util_malloc.h"
+
+#if defined(HAVE_PTHREAD_H) && !defined(_WIN32)
+#include <pthread.h>
+#endif
+
+#if defined(_WIN32)
+#include <windows.h>
+#endif
+
+#if defined(_WIN32)
+#define USE_WIN32_THREADS
+#elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE)
+#define USE_PTHREADS
+#else
+#error "No threading system was found"
+#endif /* defined(_WIN32) || ... */
+
+/* Because we use threads instead of processes on most platforms (Windows,
+ * Linux, etc), we need locking for them.  On platforms with poor thread
+ * support or broken gethostbyname_r, these functions are no-ops. */
+
+/** A generic lock structure for multithreaded builds. */
+typedef struct tor_mutex_t {
+#if defined(USE_WIN32_THREADS)
+  /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */
+  CRITICAL_SECTION mutex;
+#elif defined(USE_PTHREADS)
+  /** Pthreads-only: with pthreads, we implement locks with
+   * pthread_mutex_t. */
+  pthread_mutex_t mutex;
+#else
+  /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
+  int _unused;
+#endif /* defined(USE_WIN32_MUTEX) || ... */
+} tor_mutex_t;
+
+tor_mutex_t *tor_mutex_new(void);
+tor_mutex_t *tor_mutex_new_nonrecursive(void);
+void tor_mutex_init(tor_mutex_t *m);
+void tor_mutex_init_nonrecursive(tor_mutex_t *m);
+void tor_mutex_acquire(tor_mutex_t *m);
+void tor_mutex_release(tor_mutex_t *m);
+void tor_mutex_free_(tor_mutex_t *m);
+#define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
+void tor_mutex_uninit(tor_mutex_t *m);
+
+void tor_locking_init(void);
+
+#endif /* !defined(TOR_COMPAT_MUTEX_H) */

src/lib/lock/compat_mutex_pthreads.c

@@ -0,0 +1,97 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/err/torerr.h"
+
+/** A mutex attribute that we're going to use to tell pthreads that we want
+ * "recursive" mutexes (i.e., once we can re-lock if we're already holding
+ * them.) */
+static pthread_mutexattr_t attr_recursive;
+static int attr_initialized = 0;
+
+void
+tor_locking_init(void)
+{
+  if (!attr_initialized) {
+    pthread_mutexattr_init(&attr_recursive);
+    pthread_mutexattr_settype(&attr_recursive, PTHREAD_MUTEX_RECURSIVE);
+    attr_initialized = 1;
+  }
+}
+
+/** Initialize <b>mutex</b> so it can be locked.  Every mutex must be set
+ * up with tor_mutex_init() or tor_mutex_new(); not both. */
+void
+tor_mutex_init(tor_mutex_t *mutex)
+{
+  if (PREDICT_UNLIKELY(!attr_initialized))
+    tor_locking_init(); // LCOV_EXCL_LINE
+  const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error creating a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+
+/** As tor_mutex_init, but initialize a mutex suitable that may be
+ * non-recursive, if the OS supports that. */
+void
+tor_mutex_init_nonrecursive(tor_mutex_t *mutex)
+{
+  int err;
+  if (!attr_initialized)
+    tor_locking_init(); // LCOV_EXCL_LINE
+  err = pthread_mutex_init(&mutex->mutex, NULL);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error creating a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+
+/** Wait until <b>m</b> is free, then acquire it. */
+void
+tor_mutex_acquire(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_lock(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error locking a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+/** Release the lock <b>m</b> so another thread can have it. */
+void
+tor_mutex_release(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_unlock(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error unlocking a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}
+/** Clean up the mutex <b>m</b> so that it no longer uses any system
+ * resources.  Does not free <b>m</b>.  This function must only be called on
+ * mutexes from tor_mutex_init(). */
+void
+tor_mutex_uninit(tor_mutex_t *m)
+{
+  int err;
+  raw_assert(m);
+  err = pthread_mutex_destroy(&m->mutex);
+  if (PREDICT_UNLIKELY(err)) {
+    // LCOV_EXCL_START
+    raw_assert_unreached_msg("Error destroying a mutex.");
+    // LCOV_EXCL_STOP
+  }
+}

src/lib/lock/compat_mutex_winthreads.c

@@ -0,0 +1,40 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/lock/compat_mutex.h"
+#include "lib/err/torerr.h"
+
+void
+tor_locking_init(void)
+{
+}
+
+void
+tor_mutex_init(tor_mutex_t *m)
+{
+  InitializeCriticalSection(&m->mutex);
+}
+void
+tor_mutex_init_nonrecursive(tor_mutex_t *m)
+{
+  InitializeCriticalSection(&m->mutex);
+}
+
+void
+tor_mutex_uninit(tor_mutex_t *m)
+{
+  DeleteCriticalSection(&m->mutex);
+}
+void
+tor_mutex_acquire(tor_mutex_t *m)
+{
+  raw_assert(m);
+  EnterCriticalSection(&m->mutex);
+}
+void
+tor_mutex_release(tor_mutex_t *m)
+{
+  LeaveCriticalSection(&m->mutex);
+}

src/lib/lock/include.am

@@ -0,0 +1,24 @@
+
+noinst_LIBRARIES += src/lib/libtor-lock.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-lock-testing.a
+endif
+
+src_lib_libtor_lock_a_SOURCES =			\
+	src/lib/lock/compat_mutex.c
+
+if THREADS_PTHREADS
+src_lib_libtor_lock_a_SOURCES += src/lib/lock/compat_mutex_pthreads.c
+endif
+if THREADS_WIN32
+src_lib_libtor_lock_a_SOURCES += src/lib/lock/compat_mutex_winthreads.c
+endif
+
+src_lib_libtor_lock_testing_a_SOURCES = \
+	$(src_lib_libtor_lock_a_SOURCES)
+src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/lock/compat_mutex.h

src/lib/log/.may_include

@@ -0,0 +1,15 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/smartlist.h
+lib/err/*.h
+lib/fdio/*.h
+lib/intmath/*.h
+lib/lock/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+lib/wallclock/*.h
+
+micro-revision.i

src/lib/log/include.am

@@ -0,0 +1,24 @@
+
+noinst_LIBRARIES += src/lib/libtor-log.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-log-testing.a
+endif
+
+src_lib_libtor_log_a_SOURCES =			\
+	src/lib/log/ratelim.c			\
+	src/lib/log/torlog.c			\
+	src/lib/log/util_bug.c
+
+src_lib_libtor_log_testing_a_SOURCES = \
+	$(src_lib_libtor_log_a_SOURCES)
+src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+src/lib/log/torlog.$(OBJEXT) \
+  src/lib/log/src_lib_libtor_log_testing_a-torlog.$(OBJEXT): micro-revision.i
+
+noinst_HEADERS +=					\
+	src/lib/log/ratelim.h				\
+	src/lib/log/torlog.h				\
+	src/lib/log/util_bug.h

src/lib/log/ratelim.c

@@ -0,0 +1,55 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/log/ratelim.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/string/printf.h"
+
+/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return the number
+ * of calls to rate_limit_is_ready (including this one!) since the last time
+ * rate_limit_is_ready returned nonzero.  Otherwise return 0.
+ * If the call number hits <b>RATELIM_TOOMANY</b> limit, drop a warning
+ * about this event and stop counting. */
+static int
+rate_limit_is_ready(ratelim_t *lim, time_t now)
+{
+  if (lim->rate + lim->last_allowed <= now) {
+    int res = lim->n_calls_since_last_time + 1;
+    lim->last_allowed = now;
+    lim->n_calls_since_last_time = 0;
+    return res;
+  } else {
+    if (lim->n_calls_since_last_time <= RATELIM_TOOMANY) {
+      ++lim->n_calls_since_last_time;
+    }
+
+    return 0;
+  }
+}
+
+/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return a newly
+ * allocated string indicating how many messages were suppressed, suitable to
+ * append to a log message.  Otherwise return NULL. */
+char *
+rate_limit_log(ratelim_t *lim, time_t now)
+{
+  int n;
+  if ((n = rate_limit_is_ready(lim, now))) {
+    if (n == 1) {
+      return tor_strdup("");
+    } else {
+      char *cp=NULL;
+      const char *opt_over = (n >= RATELIM_TOOMANY) ? "over " : "";
+      /* XXXX this is not exactly correct: the messages could have occurred
+       * any time between the old value of lim->allowed and now. */
+      tor_asprintf(&cp,
+                   " [%s%d similar message(s) suppressed in last %d seconds]",
+                   opt_over, n-1, lim->rate);
+      return cp;
+    }
+  } else {
+    return NULL;
+  }
+}

src/lib/log/ratelim.h

@@ -0,0 +1,48 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_RATELIM_H
+#define TOR_RATELIM_H
+
+#include <time.h>
+
+/* Rate-limiter */
+
+/** A ratelim_t remembers how often an event is occurring, and how often
+ * it's allowed to occur.  Typical usage is something like:
+ *
+   <pre>
+    if (possibly_very_frequent_event()) {
+      const int INTERVAL = 300;
+      static ratelim_t warning_limit = RATELIM_INIT(INTERVAL);
+      char *m;
+      if ((m = rate_limit_log(&warning_limit, approx_time()))) {
+        log_warn(LD_GENERAL, "The event occurred!%s", m);
+        tor_free(m);
+      }
+    }
+   </pre>
+
+   As a convenience wrapper for logging, you can replace the above with:
+   <pre>
+   if (possibly_very_frequent_event()) {
+     static ratelim_t warning_limit = RATELIM_INIT(300);
+     log_fn_ratelim(&warning_limit, LOG_WARN, LD_GENERAL,
+                    "The event occurred!");
+   }
+   </pre>
+ */
+typedef struct ratelim_t {
+  int rate;
+  time_t last_allowed;
+  int n_calls_since_last_time;
+} ratelim_t;
+
+#define RATELIM_INIT(r) { (r), 0, 0 }
+#define RATELIM_TOOMANY (16*1000*1000)
+
+char *rate_limit_log(ratelim_t *lim, time_t now);
+
+#endif

src/common/log.c → src/lib/log/torlog.c

@@ -11,7 +11,7 @@
 
 #include "orconfig.h"
 #include <stdarg.h>
-// #include <stdio.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #ifdef HAVE_SYS_TIME_H
@@ -29,12 +29,22 @@
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
 #endif
-#include "common/compat.h"
-#include "common/util.h"
+
 #define LOG_PRIVATE
-#include "common/torlog.h"
-#include "common/container.h"
+#include "lib/log/torlog.h"
+#include "lib/log/ratelim.h"
+#include "lib/lock/compat_mutex.h"
+#include "lib/container/smartlist.h"
 #include "lib/err/torerr.h"
+#include "lib/intmath/bits.h"
+#include "lib/string/compat_string.h"
+#include "lib/string/printf.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/string/util_string.h"
+#include "lib/wallclock/tor_gettimeofday.h"
+#include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/tm_cvt.h"
+#include "lib/fdio/fdio.h"
 
 #ifdef HAVE_ANDROID_LOG_H
 #include <android/log.h>
@@ -194,12 +204,12 @@ static int pretty_fn_has_parens = 0;
 
 /** Lock the log_mutex to prevent others from changing the logfile_t list */
 #define LOCK_LOGS() STMT_BEGIN                                          \
-  tor_assert(log_mutex_initialized);                                    \
+  raw_assert(log_mutex_initialized);                                    \
   tor_mutex_acquire(&log_mutex);                                        \
   STMT_END
 /** Unlock the log_mutex */
 #define UNLOCK_LOGS() STMT_BEGIN                                        \
-  tor_assert(log_mutex_initialized);                                    \
+  raw_assert(log_mutex_initialized);                                    \
   tor_mutex_release(&log_mutex);                                        \
   STMT_END
 
@@ -292,7 +302,8 @@ log_prefix_(char *buf, size_t buf_len, int severity)
     ms -= ((int)now.tv_usec / 1000) % log_time_granularity;
   }
 
-  n = strftime(buf, buf_len, "%b %d %H:%M:%S", tor_localtime_r(&t, &tm));
+  n = strftime(buf, buf_len, "%b %d %H:%M:%S",
+               tor_localtime_r_msg(&t, &tm, NULL));
   r = tor_snprintf(buf+n, buf_len-n, ".%.3i [%s] ", ms,
                    sev_to_string(severity));
 
@@ -335,7 +346,7 @@ log_tor_version(logfile_t *lf, int reset)
     tor_snprintf(buf+n, sizeof(buf)-n,
                  "Tor %s opening %slog file.\n", VERSION, is_new?"new ":"");
   }
-  if (write_all(lf->fd, buf, strlen(buf), 0) < 0) /* error */
+  if (write_all_to_fd(lf->fd, buf, strlen(buf)) < 0) /* error */
     return -1; /* failed */
   return 0;
 }
@@ -549,7 +560,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
       lf->callback(severity, domain, msg_after_prefix);
     }
   } else {
-    if (write_all(lf->fd, buf, msg_len, 0) < 0) { /* error */
+    if (write_all_to_fd(lf->fd, buf, msg_len) < 0) { /* error */
       /* don't log the error! mark this log entry to be blown away, and
        * continue. */
       lf->seems_dead = 1;
@@ -572,7 +583,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   char *end_of_prefix=NULL;
   int callbacks_deferred = 0;
 
-  /* Call assert, not tor_assert, since tor_assert calls log on failure. */
+  /* Call assert, not raw_assert, since raw_assert calls log on failure. */
   raw_assert(format);
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
@@ -687,7 +698,7 @@ tor_log_update_sigsafe_err_fds(void)
   if (!found_real_stderr &&
       int_array_contains(fds, n_fds, STDOUT_FILENO)) {
     /* Don't use a virtual stderr when we're also logging to stdout. */
-    raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
+    raw_assert(n_fds >= 2); /* Don't raw_assert inside log fns */
     fds[0] = fds[--n_fds];
   }
 
@@ -702,7 +713,7 @@ void
 tor_log_get_logfile_names(smartlist_t *out)
 {
   logfile_t *lf;
-  tor_assert(out);
+  raw_assert(out);
 
   LOCK_LOGS();
 
@@ -815,8 +826,8 @@ delete_log(logfile_t *victim)
     logfiles = victim->next;
   else {
     for (tmpl = logfiles; tmpl && tmpl->next != victim; tmpl=tmpl->next) ;
-//    tor_assert(tmpl);
-//    tor_assert(tmpl->next == victim);
+//    raw_assert(tmpl);
+//    raw_assert(tmpl->next == victim);
     if (!tmpl)
       return;
     tmpl->next = victim->next;
@@ -850,9 +861,9 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
                         log_severity_list_t *severity_out)
 {
   int i;
-  tor_assert(loglevelMin >= loglevelMax);
-  tor_assert(loglevelMin >= LOG_ERR && loglevelMin <= LOG_DEBUG);
-  tor_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
+  raw_assert(loglevelMin >= loglevelMax);
+  raw_assert(loglevelMin >= LOG_ERR && loglevelMin <= LOG_DEBUG);
+  raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
     severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
@@ -1122,20 +1133,17 @@ mark_logs_temp(void)
 }
 
 /**
- * Add a log handler to send messages to <b>filename</b>. If opening the
- * logfile fails, -1 is returned and errno is set appropriately (by open(2)).
+ * Add a log handler to send messages to <b>filename</b> via <b>fd</b>. If
+ * opening the logfile failed, -1 is returned and errno is set appropriately
+ * (by open(2)).  Takes ownership of fd.
  */
 int
-add_file_log(const log_severity_list_t *severity, const char *filename,
-             const int truncate_log)
+add_file_log(const log_severity_list_t *severity,
+             const char *filename,
+             int fd)
 {
-  int fd;
   logfile_t *lf;
 
-  int open_flags = O_WRONLY|O_CREAT;
-  open_flags |= truncate_log ? O_TRUNC : O_APPEND;
-
-  fd = tor_open_cloexec(filename, open_flags, 0640);
   if (fd<0)
     return -1;
   if (tor_fd_seekend(fd)<0) {

src/common/torlog.h → src/lib/log/torlog.h

@@ -12,7 +12,9 @@
 
 #ifndef TOR_TORLOG_H
 
-#include "common/compat.h"
+#include <stdarg.h>
+#include "lib/cc/torint.h"
+#include "lib/cc/compat_compiler.h"
 #include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
@@ -143,8 +145,10 @@ void set_log_severity_config(int minSeverity, int maxSeverity,
                              log_severity_list_t *severity_out);
 void add_stream_log(const log_severity_list_t *severity, const char *name,
                     int fd);
-int add_file_log(const log_severity_list_t *severity, const char *filename,
-                 const int truncate);
+int add_file_log(const log_severity_list_t *severity,
+                 const char *filename,
+                 int fd);
+
 #ifdef HAVE_SYSLOG_H
 int add_syslog_log(const log_severity_list_t *severity,
                    const char* syslog_identity_tag);

src/common/util_bug.c → src/lib/log/util_bug.c

@@ -8,10 +8,16 @@
  **/
 
 #include "orconfig.h"
-#include "common/util_bug.h"
-#include "common/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/torlog.h"
 #include "lib/err/backtrace.h"
-#include "common/container.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/container/smartlist.h"
+#endif
+#include "lib/malloc/util_malloc.h"
+#include "lib/string/printf.h"
+
+#include <string.h>
 
 #ifdef __COVERITY__
 int bug_macro_deadcode_dummy__ = 0;
@@ -117,3 +123,28 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 #endif
 }
 
+#ifdef _WIN32
+/** Take a filename and return a pointer to its final element.  This
+ * function is called on __FILE__ to fix a MSVC nit where __FILE__
+ * contains the full path to the file.  This is bad, because it
+ * confuses users to find the home directory of the person who
+ * compiled the binary in their warning messages.
+ */
+const char *
+tor_fix_source_file(const char *fname)
+{
+  const char *cp1, *cp2, *r;
+  cp1 = strrchr(fname, '/');
+  cp2 = strrchr(fname, '\\');
+  if (cp1 && cp2) {
+    r = (cp1<cp2)?(cp2+1):(cp1+1);
+  } else if (cp1) {
+    r = cp1+1;
+  } else if (cp2) {
+    r = cp2+1;
+  } else {
+    r = fname;
+  }
+  return r;
+}
+#endif /* defined(_WIN32) */

src/common/util_bug.h → src/lib/log/util_bug.h

@@ -37,7 +37,8 @@
 #define TOR_UTIL_BUG_H
 
 #include "orconfig.h"
-#include "common/compat.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/log/torlog.h"
 #include "lib/testsupport/testsupport.h"
 
 /* Replace assert() with a variant that sends failures to the log before
@@ -191,6 +192,14 @@ void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
                        int once);
 
+#ifdef _WIN32
+#define SHORT_FILE__ (tor_fix_source_file(__FILE__))
+const char *tor_fix_source_file(const char *fname);
+#else
+#define SHORT_FILE__ (__FILE__)
+#define tor_fix_source_file(s) (s)
+#endif /* defined(_WIN32) */
+
 #ifdef TOR_UNIT_TESTS
 void tor_capture_bugs_(int n);
 void tor_end_capture_bugs_(void);

src/lib/malloc/.may_include

@@ -0,0 +1,6 @@
+orconfig.h
+
+lib/cc/*.h
+lib/err/*.h
+lib/malloc/*.h
+lib/testsupport/testsupport.h