Merge branch 'fs_refactor'

.gitignore

@@ -175,6 +175,8 @@ uptime-*.json
 /src/lib/libtor-err-testing.a
 /src/lib/libtor-fdio.a
 /src/lib/libtor-fdio-testing.a
+/src/lib/libtor-fs.a
+/src/lib/libtor-fs-testing.a
 /src/lib/libtor-intmath.a
 /src/lib/libtor-intmath-testing.a
 /src/lib/libtor-lock.a

Makefile.am

@@ -40,6 +40,7 @@ endif
 # "Common" libraries used to link tor's utility code.
 TOR_UTIL_LIBS = \
 	src/common/libor.a \
+        src/lib/libtor-fs.a \
         src/lib/libtor-sandbox.a \
 	src/lib/libtor-container.a \
 	src/lib/libtor-net.a \
@@ -58,6 +59,7 @@ TOR_UTIL_LIBS = \
 # and tests)
 TOR_UTIL_TESTING_LIBS = \
 	src/common/libor-testing.a \
+        src/lib/libtor-fs-testing.a \
         src/lib/libtor-sandbox-testing.a \
 	src/lib/libtor-container-testing.a \
 	src/lib/libtor-net-testing.a \

configure.ac

@@ -422,7 +422,7 @@ AH_BOTTOM([
 #endif
 ])
 
-
+AM_CONDITIONAL(WIN32, test "x$bwin32" = "xtrue")
 AM_CONDITIONAL(BUILD_NT_SERVICES, test "x$bwin32" = "xtrue")
 AM_CONDITIONAL(BUILD_LIBTORRUNNER, test "x$bwin32" != "xtrue")
 

src/common/compat.c

@@ -131,267 +131,6 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #include "lib/net/address.h"
 #include "lib/sandbox/sandbox.h"
 
-/** As open(path, flags, mode), but return an fd with the close-on-exec mode
- * set. */
-int
-tor_open_cloexec(const char *path, int flags, unsigned mode)
-{
-  int fd;
-  const char *p = sandbox_intern_string(path);
-#ifdef O_CLOEXEC
-  fd = open(p, flags|O_CLOEXEC, mode);
-  if (fd >= 0)
-    return fd;
-  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
-   * even though we were built on a system with O_CLOEXEC support, we
-   * are running on one without. */
-  if (errno != EINVAL)
-    return -1;
-#endif /* defined(O_CLOEXEC) */
-
-  log_debug(LD_FS, "Opening %s with flags %x", p, flags);
-  fd = open(p, flags, mode);
-#ifdef FD_CLOEXEC
-  if (fd >= 0) {
-    if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      close(fd);
-      return -1;
-    }
-  }
-#endif /* defined(FD_CLOEXEC) */
-  return fd;
-}
-
-/** As fopen(path,mode), but ensures that the O_CLOEXEC bit is set on the
- * underlying file handle. */
-FILE *
-tor_fopen_cloexec(const char *path, const char *mode)
-{
-  FILE *result = fopen(path, mode);
-#ifdef FD_CLOEXEC
-  if (result != NULL) {
-    if (fcntl(fileno(result), F_SETFD, FD_CLOEXEC) == -1) {
-      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
-      fclose(result);
-      return NULL;
-    }
-  }
-#endif /* defined(FD_CLOEXEC) */
-  return result;
-}
-
-/** As rename(), but work correctly with the sandbox. */
-int
-tor_rename(const char *path_old, const char *path_new)
-{
-  log_debug(LD_FS, "Renaming %s to %s", path_old, path_new);
-  return rename(sandbox_intern_string(path_old),
-                sandbox_intern_string(path_new));
-}
-
-#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
-/** Try to create a memory mapping for <b>filename</b> and return it.  On
- * failure, return NULL. Sets errno properly, using ERANGE to mean
- * "empty file". Must only be called on trusted Tor-owned files, as changing
- * the underlying file's size causes unspecified behavior. */
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  int fd; /* router file */
-  char *string;
-  int result;
-  tor_mmap_t *res;
-  size_t size, filesize;
-  struct stat st;
-
-  tor_assert(filename);
-
-  fd = tor_open_cloexec(filename, O_RDONLY, 0);
-  if (fd<0) {
-    int save_errno = errno;
-    int severity = (errno == ENOENT) ? LOG_INFO : LOG_WARN;
-    log_fn(severity, LD_FS,"Could not open \"%s\" for mmap(): %s",filename,
-           strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  /* Get the size of the file */
-  result = fstat(fd, &st);
-  if (result != 0) {
-    int save_errno = errno;
-    log_warn(LD_FS,
-             "Couldn't fstat opened descriptor for \"%s\" during mmap: %s",
-             filename, strerror(errno));
-    close(fd);
-    errno = save_errno;
-    return NULL;
-  }
-  size = filesize = (size_t)(st.st_size);
-
-  if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
-    log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
-    errno = EFBIG;
-    close(fd);
-    return NULL;
-  }
-  if (!size) {
-    /* Zero-length file. If we call mmap on it, it will succeed but
-     * return NULL, and bad things will happen. So just fail. */
-    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
-    errno = ERANGE;
-    close(fd);
-    return NULL;
-  }
-
-  string = mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
-  close(fd);
-  if (string == MAP_FAILED) {
-    int save_errno = errno;
-    log_warn(LD_FS,"Could not mmap file \"%s\": %s", filename,
-             strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  res = tor_malloc_zero(sizeof(tor_mmap_t));
-  res->data = string;
-  res->size = filesize;
-  res->mapping_size = size;
-
-  return res;
-}
-/** Release storage held for a memory mapping; returns 0 on success,
- * or -1 on failure (and logs a warning). */
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  int res;
-
-  if (handle == NULL)
-    return 0;
-
-  res = munmap((char*)handle->data, handle->mapping_size);
-  if (res == 0) {
-    /* munmap() succeeded */
-    tor_free(handle);
-  } else {
-    log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s",
-             strerror(errno));
-    res = -1;
-  }
-
-  return res;
-}
-#elif defined(_WIN32)
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  TCHAR tfilename[MAX_PATH]= {0};
-  tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
-  int empty = 0;
-  HANDLE file_handle = INVALID_HANDLE_VALUE;
-  DWORD size_low, size_high;
-  uint64_t real_size;
-  res->mmap_handle = NULL;
-#ifdef UNICODE
-  mbstowcs(tfilename,filename,MAX_PATH);
-#else
-  strlcpy(tfilename,filename,MAX_PATH);
-#endif
-  file_handle = CreateFile(tfilename,
-                           GENERIC_READ, FILE_SHARE_READ,
-                           NULL,
-                           OPEN_EXISTING,
-                           FILE_ATTRIBUTE_NORMAL,
-                           0);
-
-  if (file_handle == INVALID_HANDLE_VALUE)
-    goto win_err;
-
-  size_low = GetFileSize(file_handle, &size_high);
-
-  if (size_low == INVALID_FILE_SIZE && GetLastError() != NO_ERROR) {
-    log_warn(LD_FS,"Error getting size of \"%s\".",filename);
-    goto win_err;
-  }
-  if (size_low == 0 && size_high == 0) {
-    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
-    empty = 1;
-    goto err;
-  }
-  real_size = (((uint64_t)size_high)<<32) | size_low;
-  if (real_size > SIZE_MAX) {
-    log_warn(LD_FS,"File \"%s\" is too big to map; not trying.",filename);
-    goto err;
-  }
-  res->size = real_size;
-
-  res->mmap_handle = CreateFileMapping(file_handle,
-                                       NULL,
-                                       PAGE_READONLY,
-                                       size_high,
-                                       size_low,
-                                       NULL);
-  if (res->mmap_handle == NULL)
-    goto win_err;
-  res->data = (char*) MapViewOfFile(res->mmap_handle,
-                                    FILE_MAP_READ,
-                                    0, 0, 0);
-  if (!res->data)
-    goto win_err;
-
-  CloseHandle(file_handle);
-  return res;
- win_err: {
-    DWORD e = GetLastError();
-    int severity = (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) ?
-      LOG_INFO : LOG_WARN;
-    char *msg = format_win32_error(e);
-    log_fn(severity, LD_FS, "Couldn't mmap file \"%s\": %s", filename, msg);
-    tor_free(msg);
-    if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND)
-      errno = ENOENT;
-    else
-      errno = EINVAL;
-  }
- err:
-  if (empty)
-    errno = ERANGE;
-  if (file_handle != INVALID_HANDLE_VALUE)
-    CloseHandle(file_handle);
-  tor_munmap_file(res);
-  return NULL;
-}
-
-/* Unmap the file, and return 0 for success or -1 for failure */
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  if (handle == NULL)
-    return 0;
-
-  if (handle->data) {
-    /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would
-       have to be redefined as non-const. */
-    BOOL ok = UnmapViewOfFile( (LPVOID) handle->data);
-    if (!ok) {
-      log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d",
-               (int)GetLastError());
-    }
-  }
-
-  if (handle->mmap_handle != NULL)
-    CloseHandle(handle->mmap_handle);
-  tor_free(handle);
-
-  return 0;
-}
-#else
-#error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
-
 /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at
  * <b>needle</b>, return a pointer to the first occurrence of the needle
  * within the haystack, or NULL if there is no such occurrence.
@@ -553,45 +292,6 @@ set_uint64(void *cp, uint64_t v)
   memcpy(cp,&v,8);
 }
 
-/**
- * Rename the file <b>from</b> to the file <b>to</b>.  On Unix, this is
- * the same as rename(2).  On windows, this removes <b>to</b> first if
- * it already exists.
- * Returns 0 on success.  Returns -1 and sets errno on failure.
- */
-int
-replace_file(const char *from, const char *to)
-{
-#ifndef _WIN32
-  return tor_rename(from, to);
-#else
-  switch (file_status(to))
-    {
-    case FN_NOENT:
-      break;
-    case FN_FILE:
-    case FN_EMPTY:
-      if (unlink(to)) return -1;
-      break;
-    case FN_ERROR:
-      return -1;
-    case FN_DIR:
-      errno = EISDIR;
-      return -1;
-    }
-  return tor_rename(from,to);
-#endif /* !defined(_WIN32) */
-}
-
-/** Change <b>fname</b>'s modification time to now. */
-int
-touch_file(const char *fname)
-{
-  if (utime(fname, NULL)!=0)
-    return -1;
-  return 0;
-}
-
 /** Represents a lockfile on which we hold the lock. */
 struct tor_lockfile_t {
   /** Name of the file */
@@ -928,109 +628,6 @@ log_credential_status(void)
 }
 #endif /* !defined(_WIN32) */
 
-#ifndef _WIN32
-/** Cached struct from the last getpwname() call we did successfully. */
-static struct passwd *passwd_cached = NULL;
-
-/** Helper: copy a struct passwd object.
- *
- * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir.  Tor doesn't use
- * any others, and I don't want to run into incompatibilities.
- */
-static struct passwd *
-tor_passwd_dup(const struct passwd *pw)
-{
-  struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd));
-  if (pw->pw_name)
-    new_pw->pw_name = tor_strdup(pw->pw_name);
-  if (pw->pw_dir)
-    new_pw->pw_dir = tor_strdup(pw->pw_dir);
-  new_pw->pw_uid = pw->pw_uid;
-  new_pw->pw_gid = pw->pw_gid;
-
-  return new_pw;
-}
-
-#define tor_passwd_free(pw) \
-  FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw))
-
-/** Helper: free one of our cached 'struct passwd' values. */
-static void
-tor_passwd_free_(struct passwd *pw)
-{
-  if (!pw)
-    return;
-
-  tor_free(pw->pw_name);
-  tor_free(pw->pw_dir);
-  tor_free(pw);
-}
-
-/** Wrapper around getpwnam() that caches result. Used so that we don't need
- * to give the sandbox access to /etc/passwd.
- *
- * The following fields alone will definitely be copied in the output: pw_uid,
- * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
- *
- * When called with a NULL argument, this function clears storage associated
- * with static variables it uses.
- **/
-const struct passwd *
-tor_getpwnam(const char *username)
-{
-  struct passwd *pw;
-
-  if (username == NULL) {
-    tor_passwd_free(passwd_cached);
-    passwd_cached = NULL;
-    return NULL;
-  }
-
-  if ((pw = getpwnam(username))) {
-    tor_passwd_free(passwd_cached);
-    passwd_cached = tor_passwd_dup(pw);
-    log_info(LD_GENERAL, "Caching new entry %s for %s",
-             passwd_cached->pw_name, username);
-    return pw;
-  }
-
-  /* Lookup failed */
-  if (! passwd_cached || ! passwd_cached->pw_name)
-    return NULL;
-
-  if (! strcmp(username, passwd_cached->pw_name))
-    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
-
-  return NULL;
-}
-
-/** Wrapper around getpwnam() that can use cached result from
- * tor_getpwnam(). Used so that we don't need to give the sandbox access to
- * /etc/passwd.
- *
- * The following fields alone will definitely be copied in the output: pw_uid,
- * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
- */
-const struct passwd *
-tor_getpwuid(uid_t uid)
-{
-  struct passwd *pw;
-
-  if ((pw = getpwuid(uid))) {
-    return pw;
-  }
-
-  /* Lookup failed */
-  if (! passwd_cached)
-    return NULL;
-
-  if (uid == passwd_cached->pw_uid)
-    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
-
-  return NULL;
-}
-#endif /* !defined(_WIN32) */
-
 /** Return true iff we were compiled with capability support, and capabilities
  * seem to work. **/
 int
@@ -1322,159 +919,6 @@ tor_disable_debugger_attach(void)
   return r;
 }
 
-#ifdef HAVE_PWD_H
-/** Allocate and return a string containing the home directory for the
- * user <b>username</b>. Only works on posix-like systems. */
-char *
-get_user_homedir(const char *username)
-{
-  const struct passwd *pw;
-  tor_assert(username);
-
-  if (!(pw = tor_getpwnam(username))) {
-    log_err(LD_CONFIG,"User \"%s\" not found.", username);
-    return NULL;
-  }
-  return tor_strdup(pw->pw_dir);
-}
-#endif /* defined(HAVE_PWD_H) */
-
-/** Modify <b>fname</b> to contain the name of its parent directory.  Doesn't
- * actually examine the filesystem; does a purely syntactic modification.
- *
- * The parent of the root director is considered to be iteself.
- *
- * Path separators are the forward slash (/) everywhere and additionally
- * the backslash (\) on Win32.
- *
- * Cuts off any number of trailing path separators but otherwise ignores
- * them for purposes of finding the parent directory.
- *
- * Returns 0 if a parent directory was successfully found, -1 otherwise (fname
- * did not have any path separators or only had them at the end).
- * */
-int
-get_parent_directory(char *fname)
-{
-  char *cp;
-  int at_end = 1;
-  tor_assert(fname);
-#ifdef _WIN32
-  /* If we start with, say, c:, then don't consider that the start of the path
-   */
-  if (fname[0] && fname[1] == ':') {
-    fname += 2;
-  }
-#endif /* defined(_WIN32) */
-  /* Now we want to remove all path-separators at the end of the string,
-   * and to remove the end of the string starting with the path separator
-   * before the last non-path-separator.  In perl, this would be
-   *   s#[/]*$##; s#/[^/]*$##;
-   * on a unixy platform.
-   */
-  cp = fname + strlen(fname);
-  at_end = 1;
-  while (--cp >= fname) {
-    int is_sep = (*cp == '/'
-#ifdef _WIN32
-                  || *cp == '\\'
-#endif
-                  );
-    if (is_sep) {
-      if (cp == fname) {
-        /* This is the first separator in the file name; don't remove it! */
-        cp[1] = '\0';
-        return 0;
-      }
-      *cp = '\0';
-      if (! at_end)
-        return 0;
-    } else {
-      at_end = 0;
-    }
-  }
-  return -1;
-}
-
-#ifndef _WIN32
-/** Return a newly allocated string containing the output of getcwd(). Return
- * NULL on failure. (We can't just use getcwd() into a PATH_MAX buffer, since
- * Hurd hasn't got a PATH_MAX.)
- */
-static char *
-alloc_getcwd(void)
-{
-#ifdef HAVE_GET_CURRENT_DIR_NAME
-  /* Glibc makes this nice and simple for us. */
-  char *cwd = get_current_dir_name();
-  char *result = NULL;
-  if (cwd) {
-    /* We make a copy here, in case tor_malloc() is not malloc(). */
-    result = tor_strdup(cwd);
-    raw_free(cwd); // alias for free to avoid tripping check-spaces.
-  }
-  return result;
-#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
-  size_t size = 1024;
-  char *buf = NULL;
-  char *ptr = NULL;
-
-  while (ptr == NULL) {
-    buf = tor_realloc(buf, size);
-    ptr = getcwd(buf, size);
-
-    if (ptr == NULL && errno != ERANGE) {
-      tor_free(buf);
-      return NULL;
-    }
-
-    size *= 2;
-  }
-  return buf;
-#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */
-}
-#endif /* !defined(_WIN32) */
-
-/** Expand possibly relative path <b>fname</b> to an absolute path.
- * Return a newly allocated string, possibly equal to <b>fname</b>. */
-char *
-make_path_absolute(char *fname)
-{
-#ifdef _WIN32
-  char *absfname_malloced = _fullpath(NULL, fname, 1);
-
-  /* We don't want to assume that tor_free can free a string allocated
-   * with malloc.  On failure, return fname (it's better than nothing). */
-  char *absfname = tor_strdup(absfname_malloced ? absfname_malloced : fname);
-  if (absfname_malloced) raw_free(absfname_malloced);
-
-  return absfname;
-#else /* !(defined(_WIN32)) */
-  char *absfname = NULL, *path = NULL;
-
-  tor_assert(fname);
-
-  if (fname[0] == '/') {
-    absfname = tor_strdup(fname);
-  } else {
-    path = alloc_getcwd();
-    if (path) {
-      tor_asprintf(&absfname, "%s/%s", path, fname);
-      tor_free(path);
-    } else {
-      /* LCOV_EXCL_START Can't make getcwd fail. */
-      /* If getcwd failed, the best we can do here is keep using the
-       * relative path.  (Perhaps / isn't readable by this UID/GID.) */
-      log_warn(LD_GENERAL, "Unable to find current working directory: %s",
-               strerror(errno));
-      absfname = tor_strdup(fname);
-      /* LCOV_EXCL_STOP */
-    }
-  }
-  return absfname;
-#endif /* defined(_WIN32) */
-}
-
 #ifndef HAVE__NSGETENVIRON
 #ifndef HAVE_EXTERN_ENVIRON_DECLARED
 /* Some platforms declare environ under some circumstances, others don't. */
@@ -1934,51 +1378,6 @@ network_init(void)
   return 0;
 }
 
-#ifdef _WIN32
-/** Return a newly allocated string describing the windows system error code
- * <b>err</b>.  Note that error codes are different from errno.  Error codes
- * come from GetLastError() when a winapi call fails.  errno is set only when
- * ANSI functions fail.  Whee. */
-char *
-format_win32_error(DWORD err)
-{
-  TCHAR *str = NULL;
-  char *result;
-  DWORD n;
-
-  /* Somebody once decided that this interface was better than strerror(). */
-  n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                 FORMAT_MESSAGE_FROM_SYSTEM |
-                 FORMAT_MESSAGE_IGNORE_INSERTS,
-                 NULL, err,
-                 MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
-                 (LPVOID)&str,
-                 0, NULL);
-
-  if (str && n) {
-#ifdef UNICODE
-    size_t len;
-    if (n > 128*1024)
-      len = (128 * 1024) * 2 + 1; /* This shouldn't be possible, but let's
-                                   * make sure. */
-    else
-      len = n * 2 + 1;
-    result = tor_malloc(len);
-    wcstombs(result,str,len);
-    result[len-1] = '\0';
-#else /* !(defined(UNICODE)) */
-    result = tor_strdup(str);
-#endif /* defined(UNICODE) */
-  } else {
-    result = tor_strdup("<unformattable error>");
-  }
-  if (str) {
-    LocalFree(str); /* LocalFree != free() */
-  }
-  return result;
-}
-#endif /* defined(_WIN32) */
-
 #if defined(HW_PHYSMEM64)
 /* This appears to be an OpenBSD thing */
 #define INT64_HW_MEM HW_PHYSMEM64

src/common/compat.h

@@ -50,35 +50,18 @@
 #include "lib/string/compat_ctype.h"
 #include "lib/string/compat_string.h"
 #include "lib/string/printf.h"
+#include "lib/log/win32err.h"
 #include "lib/net/socket.h"
 #include "lib/net/ipv4.h"
 #include "lib/net/ipv6.h"
 #include "lib/net/resolve.h"
+#include "lib/fs/files.h"
+#include "lib/fs/mmap.h"
+#include "lib/fs/userdb.h"
 
 #include <stdio.h>
 #include <errno.h>
 
-/* ===== Compiler compatibility */
-
-/** Represents an mmaped file. Allocated via tor_mmap_file; freed with
- * tor_munmap_file. */
-typedef struct tor_mmap_t {
-  const char *data; /**< Mapping of the file's contents. */
-  size_t size; /**< Size of the file. */
-
-  /* None of the fields below should be accessed from outside compat.c */
-#ifdef HAVE_MMAP
-  size_t mapping_size; /**< Size of the actual mapping. (This is this file
-                        * size, rounded up to the nearest page.) */
-#elif defined _WIN32
-  HANDLE mmap_handle;
-#endif /* defined(HAVE_MMAP) || ... */
-
-} tor_mmap_t;
-
-tor_mmap_t *tor_mmap_file(const char *filename) ATTR_NONNULL((1));
-int tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1));
-
 const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen) ATTR_NONNULL((1,3));
 static const void *tor_memstr(const void *haystack, size_t hlen,
@@ -144,26 +127,11 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
 #endif /* !defined(timercmp) */
 
 /* ===== File compatibility */
-int tor_open_cloexec(const char *path, int flags, unsigned mode);
-FILE *tor_fopen_cloexec(const char *path, const char *mode);
-int tor_rename(const char *path_old, const char *path_new);
-
-int replace_file(const char *from, const char *to);
-int touch_file(const char *fname);
-
 typedef struct tor_lockfile_t tor_lockfile_t;
 tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-int64_t tor_get_avail_disk_space(const char *path);
-
-#ifdef _WIN32
-#define PATH_SEPARATOR "\\"
-#else
-#define PATH_SEPARATOR "/"
-#endif
-
 /* ===== Net compatibility */
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
@@ -217,17 +185,6 @@ int have_capability_support(void);
 /** Flag for switch_id; see switch_id() for documentation */
 #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
 int switch_id(const char *user, unsigned flags);
-#ifdef HAVE_PWD_H
-char *get_user_homedir(const char *username);
-#endif
-
-#ifndef _WIN32
-const struct passwd *tor_getpwnam(const char *username);
-const struct passwd *tor_getpwuid(uid_t uid);
-#endif
-
-int get_parent_directory(char *fname);
-char *make_path_absolute(char *fname);
 
 char **get_environment(void);
 
@@ -250,11 +207,6 @@ int tor_mlockall(void);
 #define MIN(a,b) ( ((a)>(b)) ? (b) : (a) )
 #endif
 
-/* Platform-specific helpers. */
-#ifdef _WIN32
-char *format_win32_error(DWORD err);
-#endif
-
 /*for some reason my compiler doesn't have these version flags defined
   a nice homework assignment for someone one day is to define the rest*/
 //these are the values as given on MSDN

src/common/util.c

@@ -1076,861 +1076,10 @@ format_time_interval(char *out, size_t out_len, long interval)
  * File helpers
  * ===== */
 
-/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>.  <b>isSocket</b>
- * must be 1 if fd was returned by socket() or accept(), and 0 if fd
- * was returned by open().  Return the number of bytes written, or -1
- * on error.  Only use if fd is a blocking fd.  */
-ssize_t
-write_all(tor_socket_t fd, const char *buf, size_t count, int isSocket)
-{
-  size_t written = 0;
-  ssize_t result;
-  raw_assert(count < SSIZE_MAX);
-
-  while (written != count) {
-    if (isSocket)
-      result = tor_socket_send(fd, buf+written, count-written, 0);
-    else
-      result = write((int)fd, buf+written, count-written);
-    if (result<0)
-      return -1;
-    written += result;
-  }
-  return (ssize_t)count;
-}
-
-/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes
- * or reach the end of the file. <b>isSocket</b> must be 1 if fd
- * was returned by socket() or accept(), and 0 if fd was returned by
- * open().  Return the number of bytes read, or -1 on error. Only use
- * if fd is a blocking fd. */
-ssize_t
-read_all(tor_socket_t fd, char *buf, size_t count, int isSocket)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
-    errno = EINVAL;
-    return -1;
-  }
-
-  while (numread < count) {
-    if (isSocket)
-      result = tor_socket_recv(fd, buf+numread, count-numread, 0);
-    else
-      result = read((int)fd, buf+numread, count-numread);
-    if (result<0)
-      return -1;
-    else if (result == 0)
-      break;
-    numread += result;
-  }
-  return (ssize_t)numread;
-}
-
 /*
  *    Filesystem operations.
  */
 
-/** Clean up <b>name</b> so that we can use it in a call to "stat".  On Unix,
- * we do nothing.  On Windows, we remove a trailing slash, unless the path is
- * the root of a disk. */
-static void
-clean_name_for_stat(char *name)
-{
-#ifdef _WIN32
-  size_t len = strlen(name);
-  if (!len)
-    return;
-  if (name[len-1]=='\\' || name[len-1]=='/') {
-    if (len == 1 || (len==3 && name[1]==':'))
-      return;
-    name[len-1]='\0';
-  }
-#else /* !(defined(_WIN32)) */
-  (void)name;
-#endif /* defined(_WIN32) */
-}
-
-/** Wrapper for unlink() to make it mockable for the test suite; returns 0
- * if unlinking the file succeeded, -1 and sets errno if unlinking fails.
- */
-
-MOCK_IMPL(int,
-tor_unlink,(const char *pathname))
-{
-  return unlink(pathname);
-}
-
-/** Return:
- * FN_ERROR if filename can't be read, is NULL, or is zero-length,
- * FN_NOENT if it doesn't exist,
- * FN_FILE if it is a non-empty regular file, or a FIFO on unix-like systems,
- * FN_EMPTY for zero-byte regular files,
- * FN_DIR if it's a directory, and
- * FN_ERROR for any other file type.
- * On FN_ERROR and FN_NOENT, sets errno.  (errno is not set when FN_ERROR
- * is returned due to an unhandled file type.) */
-file_status_t
-file_status(const char *fname)
-{
-  struct stat st;
-  char *f;
-  int r;
-  if (!fname || strlen(fname) == 0) {
-    return FN_ERROR;
-  }
-  f = tor_strdup(fname);
-  clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", f);
-  r = stat(sandbox_intern_string(f), &st);
-  tor_free(f);
-  if (r) {
-    if (errno == ENOENT) {
-      return FN_NOENT;
-    }
-    return FN_ERROR;
-  }
-  if (st.st_mode & S_IFDIR) {
-    return FN_DIR;
-  } else if (st.st_mode & S_IFREG) {
-    if (st.st_size > 0) {
-      return FN_FILE;
-    } else if (st.st_size == 0) {
-      return FN_EMPTY;
-    } else {
-      return FN_ERROR;
-    }
-#ifndef _WIN32
-  } else if (st.st_mode & S_IFIFO) {
-    return FN_FILE;
-#endif
-  } else {
-    return FN_ERROR;
-  }
-}
-
-/** Check whether <b>dirname</b> exists and is private.  If yes return 0.
- * If <b>dirname</b> does not exist:
- *  - if <b>check</b>&CPD_CREATE, try to create it and return 0 on success.
- *  - if <b>check</b>&CPD_CHECK, and we think we can create it, return 0.
- *  - if <b>check</b>&CPD_CHECK is false, and the directory exists, return 0.
- *  - otherwise, return -1.
- * If CPD_GROUP_OK is set, then it's okay if the directory
- * is group-readable, but in all cases we create the directory mode 0700.
- * If CPD_GROUP_READ is set, existing directory behaves as CPD_GROUP_OK and
- * if the directory is created it will use mode 0750 with group read
- * permission. Group read privileges also assume execute permission
- * as norm for directories. If CPD_CHECK_MODE_ONLY is set, then we don't
- * alter the directory permissions if they are too permissive:
- * we just return -1.
- * When effective_user is not NULL, check permissions against the given user
- * and its primary group.
- */
-MOCK_IMPL(int,
-check_private_dir,(const char *dirname, cpd_check_t check,
-                   const char *effective_user))
-{
-  int r;
-  struct stat st;
-
-  tor_assert(dirname);
-
-#ifndef _WIN32
-  int fd;
-  const struct passwd *pw = NULL;
-  uid_t running_uid;
-  gid_t running_gid;
-
-  /*
-   * Goal is to harden the implementation by removing any
-   * potential for race between stat() and chmod().
-   * chmod() accepts filename as argument. If an attacker can move
-   * the file between stat() and chmod(), a potential race exists.
-   *
-   * Several suggestions taken from:
-   * https://developer.apple.com/library/mac/documentation/
-   *     Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
-   */
-
-  /* Open directory.
-   * O_NOFOLLOW to ensure that it does not follow symbolic links */
-  fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
-
-  /* Was there an error? Maybe the directory does not exist? */
-  if (fd == -1) {
-
-    if (errno != ENOENT) {
-      /* Other directory error */
-      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
-               strerror(errno));
-      return -1;
-    }
-
-    /* Received ENOENT: Directory does not exist */
-
-    /* Should we create the directory? */
-    if (check & CPD_CREATE) {
-      log_info(LD_GENERAL, "Creating directory %s", dirname);
-      if (check & CPD_GROUP_READ) {
-        r = mkdir(dirname, 0750);
-      } else {
-        r = mkdir(dirname, 0700);
-      }
-
-      /* check for mkdir() error */
-      if (r) {
-        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
-            strerror(errno));
-        return -1;
-      }
-
-      /* we just created the directory. try to open it again.
-       * permissions on the directory will be checked again below.*/
-      fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
-
-      if (fd == -1) {
-        log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
-                 dirname,
-                 strerror(errno));
-        return -1;
-      } else {
-        close(fd);
-      }
-
-    } else if (!(check & CPD_CHECK)) {
-      log_warn(LD_FS, "Directory %s does not exist.", dirname);
-      return -1;
-    }
-
-    /* XXXX In the case where check==CPD_CHECK, we should look at the
-     * parent directory a little harder. */
-    return 0;
-  }
-
-  tor_assert(fd >= 0);
-
-  //f = tor_strdup(dirname);
-  //clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", dirname);
-  //r = stat(sandbox_intern_string(f), &st);
-  r = fstat(fd, &st);
-  if (r == -1) {
-      log_warn(LD_FS, "fstat() on directory %s failed.", dirname);
-      close(fd);
-      return -1;
-  }
-  //tor_free(f);
-
-  /* check that dirname is a directory */
-  if (!(st.st_mode & S_IFDIR)) {
-    log_warn(LD_FS, "%s is not a directory", dirname);
-    close(fd);
-    return -1;
-  }
-
-  if (effective_user) {
-    /* Look up the user and group information.
-     * If we have a problem, bail out. */
-    pw = tor_getpwnam(effective_user);
-    if (pw == NULL) {
-      log_warn(LD_CONFIG, "Error setting configured user: %s not found",
-               effective_user);
-      close(fd);
-      return -1;
-    }
-    running_uid = pw->pw_uid;
-    running_gid = pw->pw_gid;
-  } else {
-    running_uid = getuid();
-    running_gid = getgid();
-  }
-  if (st.st_uid != running_uid) {
-    char *process_ownername = NULL, *file_ownername = NULL;
-
-    {
-      const struct passwd *pw_running = tor_getpwuid(running_uid);
-      process_ownername = pw_running ? tor_strdup(pw_running->pw_name) :
-        tor_strdup("<unknown>");
-    }
-
-    {
-      const struct passwd *pw_stat = tor_getpwuid(st.st_uid);
-      file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) :
-        tor_strdup("<unknown>");
-    }
-
-    log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
-        "%s (%d). Perhaps you are running Tor as the wrong user?",
-             dirname, process_ownername, (int)running_uid,
-             file_ownername, (int)st.st_uid);
-
-    tor_free(process_ownername);
-    tor_free(file_ownername);
-    close(fd);
-    return -1;
-  }
-  if ( (check & (CPD_GROUP_OK|CPD_GROUP_READ))
-       && (st.st_gid != running_gid) && (st.st_gid != 0)) {
-    struct group *gr;
-    char *process_groupname = NULL;
-    gr = getgrgid(running_gid);
-    process_groupname = gr ? tor_strdup(gr->gr_name) : tor_strdup("<unknown>");
-    gr = getgrgid(st.st_gid);
-
-    log_warn(LD_FS, "%s is not owned by this group (%s, %d) but by group "
-             "%s (%d).  Are you running Tor as the wrong user?",
-             dirname, process_groupname, (int)running_gid,
-             gr ?  gr->gr_name : "<unknown>", (int)st.st_gid);
-
-    tor_free(process_groupname);
-    close(fd);
-    return -1;
-  }
-  unsigned unwanted_bits = 0;
-  if (check & (CPD_GROUP_OK|CPD_GROUP_READ)) {
-    unwanted_bits = 0027;
-  } else {
-    unwanted_bits = 0077;
-  }
-  unsigned check_bits_filter = ~0;
-  if (check & CPD_RELAX_DIRMODE_CHECK) {
-    check_bits_filter = 0022;
-  }
-  if ((st.st_mode & unwanted_bits & check_bits_filter) != 0) {
-    unsigned new_mode;
-    if (check & CPD_CHECK_MODE_ONLY) {
-      log_warn(LD_FS, "Permissions on directory %s are too permissive.",
-               dirname);
-      close(fd);
-      return -1;
-    }
-    log_warn(LD_FS, "Fixing permissions on directory %s", dirname);
-    new_mode = st.st_mode;
-    new_mode |= 0700; /* Owner should have rwx */
-    if (check & CPD_GROUP_READ) {
-      new_mode |= 0050; /* Group should have rx */
-    }
-    new_mode &= ~unwanted_bits; /* Clear the bits that we didn't want set...*/
-    if (fchmod(fd, new_mode)) {
-      log_warn(LD_FS, "Could not chmod directory %s: %s", dirname,
-               strerror(errno));
-      close(fd);
-      return -1;
-    } else {
-      close(fd);
-      return 0;
-    }
-  }
-  close(fd);
-#else /* !(!defined(_WIN32)) */
-  /* Win32 case: we can't open() a directory. */
-  (void)effective_user;
-
-  char *f = tor_strdup(dirname);
-  clean_name_for_stat(f);
-  log_debug(LD_FS, "stat()ing %s", f);
-  r = stat(sandbox_intern_string(f), &st);
-  tor_free(f);
-  if (r) {
-    if (errno != ENOENT) {
-      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
-               strerror(errno));
-      return -1;
-    }
-    if (check & CPD_CREATE) {
-      log_info(LD_GENERAL, "Creating directory %s", dirname);
-      r = mkdir(dirname);
-      if (r) {
-        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
-                 strerror(errno));
-        return -1;
-      }
-    } else if (!(check & CPD_CHECK)) {
-      log_warn(LD_FS, "Directory %s does not exist.", dirname);
-      return -1;
-    }
-    return 0;
-  }
-  if (!(st.st_mode & S_IFDIR)) {
-    log_warn(LD_FS, "%s is not a directory", dirname);
-    return -1;
-  }
-
-#endif /* !defined(_WIN32) */
-  return 0;
-}
-
-/** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
- * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
- *
- * This function replaces the old file atomically, if possible.  This
- * function, and all other functions in util.c that create files, create them
- * with mode 0600.
- */
-MOCK_IMPL(int,
-write_str_to_file,(const char *fname, const char *str, int bin))
-{
-#ifdef _WIN32
-  if (!bin && strchr(str, '\r')) {
-    log_warn(LD_BUG,
-             "We're writing a text string that already contains a CR to %s",
-             escaped(fname));
-  }
-#endif /* defined(_WIN32) */
-  return write_bytes_to_file(fname, str, strlen(str), bin);
-}
-
-/** Represents a file that we're writing to, with support for atomic commit:
- * we can write into a temporary file, and either remove the file on
- * failure, or replace the original file on success. */
-struct open_file_t {
-  char *tempname; /**< Name of the temporary file. */
-  char *filename; /**< Name of the original file. */
-  unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
-  unsigned binary:1; /**< Did we open in binary mode? */
-  int fd; /**< fd for the open file. */
-  FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
-};
-
-/** Try to start writing to the file in <b>fname</b>, passing the flags
- * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
- * access value <b>mode</b>.  If the O_APPEND flag is set, we append to the
- * original file.  Otherwise, we open a new temporary file in the same
- * directory, and either replace the original or remove the temporary file
- * when we're done.
- *
- * Return the fd for the newly opened file, and store working data in
- * *<b>data_out</b>.  The caller should not close the fd manually:
- * instead, call finish_writing_to_file() or abort_writing_to_file().
- * Returns -1 on failure.
- *
- * NOTE: When not appending, the flags O_CREAT and O_TRUNC are treated
- * as true and the flag O_EXCL is treated as false.
- *
- * NOTE: Ordinarily, O_APPEND means "seek to the end of the file before each
- * write()".  We don't do that.
- */
-int
-start_writing_to_file(const char *fname, int open_flags, int mode,
-                      open_file_t **data_out)
-{
-  open_file_t *new_file = tor_malloc_zero(sizeof(open_file_t));
-  const char *open_name;
-  int append = 0;
-
-  tor_assert(fname);
-  tor_assert(data_out);
-#if (O_BINARY != 0 && O_TEXT != 0)
-  tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
-#endif
-  new_file->fd = -1;
-  new_file->filename = tor_strdup(fname);
-  if (open_flags & O_APPEND) {
-    open_name = fname;
-    new_file->rename_on_close = 0;
-    append = 1;
-    open_flags &= ~O_APPEND;
-  } else {
-    tor_asprintf(&new_file->tempname, "%s.tmp", fname);
-    open_name = new_file->tempname;
-    /* We always replace an existing temporary file if there is one. */
-    open_flags |= O_CREAT|O_TRUNC;
-    open_flags &= ~O_EXCL;
-    new_file->rename_on_close = 1;
-  }
-#if O_BINARY != 0
-  if (open_flags & O_BINARY)
-    new_file->binary = 1;
-#endif
-
-  new_file->fd = tor_open_cloexec(open_name, open_flags, mode);
-  if (new_file->fd < 0) {
-    log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
-        open_name, fname, strerror(errno));
-    goto err;
-  }
-  if (append) {
-    if (tor_fd_seekend(new_file->fd) < 0) {
-      log_warn(LD_FS, "Couldn't seek to end of file \"%s\": %s", open_name,
-               strerror(errno));
-      goto err;
-    }
-  }
-
-  *data_out = new_file;
-
-  return new_file->fd;
-
- err:
-  if (new_file->fd >= 0)
-    close(new_file->fd);
-  *data_out = NULL;
-  tor_free(new_file->filename);
-  tor_free(new_file->tempname);
-  tor_free(new_file);
-  return -1;
-}
-
-/** Given <b>file_data</b> from start_writing_to_file(), return a stdio FILE*
- * that can be used to write to the same file.  The caller should not mix
- * stdio calls with non-stdio calls. */
-FILE *
-fdopen_file(open_file_t *file_data)
-{
-  tor_assert(file_data);
-  if (file_data->stdio_file)
-    return file_data->stdio_file;
-  tor_assert(file_data->fd >= 0);
-  if (!(file_data->stdio_file = fdopen(file_data->fd,
-                                       file_data->binary?"ab":"a"))) {
-    log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
-             file_data->fd, strerror(errno));
-  }
-  return file_data->stdio_file;
-}
-
-/** Combines start_writing_to_file with fdopen_file(): arguments are as
- * for start_writing_to_file, but  */
-FILE *
-start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
-                            open_file_t **data_out)
-{
-  FILE *res;
-  if (start_writing_to_file(fname, open_flags, mode, data_out)<0)
-    return NULL;
-  if (!(res = fdopen_file(*data_out))) {
-    abort_writing_to_file(*data_out);
-    *data_out = NULL;
-  }
-  return res;
-}
-
-/** Helper function: close and free the underlying file and memory in
- * <b>file_data</b>.  If we were writing into a temporary file, then delete
- * that file (if abort_write is true) or replaces the target file with
- * the temporary file (if abort_write is false). */
-static int
-finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
-{
-  int r = 0;
-
-  tor_assert(file_data && file_data->filename);
-  if (file_data->stdio_file) {
-    if (fclose(file_data->stdio_file)) {
-      log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
-               strerror(errno));
-      abort_write = r = -1;
-    }
-  } else if (file_data->fd >= 0 && close(file_data->fd) < 0) {
-    log_warn(LD_FS, "Error flushing \"%s\": %s", file_data->filename,
-             strerror(errno));
-    abort_write = r = -1;
-  }
-
-  if (file_data->rename_on_close) {
-    tor_assert(file_data->tempname && file_data->filename);
-    if (!abort_write) {
-      tor_assert(strcmp(file_data->filename, file_data->tempname));
-      if (replace_file(file_data->tempname, file_data->filename)) {
-        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
-                 strerror(errno));
-        abort_write = r = -1;
-      }
-    }
-    if (abort_write) {
-      int res = unlink(file_data->tempname);
-      if (res != 0) {
-        /* We couldn't unlink and we'll leave a mess behind */
-        log_warn(LD_FS, "Failed to unlink %s: %s",
-                 file_data->tempname, strerror(errno));
-        r = -1;
-      }
-    }
-  }
-
-  tor_free(file_data->filename);
-  tor_free(file_data->tempname);
-  tor_free(file_data);
-
-  return r;
-}
-
-/** Finish writing to <b>file_data</b>: close the file handle, free memory as
- * needed, and if using a temporary file, replace the original file with
- * the temporary file. */
-int
-finish_writing_to_file(open_file_t *file_data)
-{
-  return finish_writing_to_file_impl(file_data, 0);
-}
-
-/** Finish writing to <b>file_data</b>: close the file handle, free memory as
- * needed, and if using a temporary file, delete it. */
-int
-abort_writing_to_file(open_file_t *file_data)
-{
-  return finish_writing_to_file_impl(file_data, 1);
-}
-
-/** Helper: given a set of flags as passed to open(2), open the file
- * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
- * the file.  Do so as atomically as possible e.g. by opening temp files and
- * renaming. */
-static int
-write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
-                          int open_flags)
-{
-  open_file_t *file = NULL;
-  int fd;
-  ssize_t result;
-  fd = start_writing_to_file(fname, open_flags, 0600, &file);
-  if (fd<0)
-    return -1;
-  SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
-  {
-    result = write_all(fd, chunk->bytes, chunk->len, 0);
-    if (result < 0) {
-      log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
-          strerror(errno));
-      goto err;
-    }
-    tor_assert((size_t)result == chunk->len);
-  });
-
-  return finish_writing_to_file(file);
- err:
-  abort_writing_to_file(file);
-  return -1;
-}
-
-/** Given a smartlist of sized_chunk_t, write them to a file
- * <b>fname</b>, overwriting or creating the file as necessary.
- * If <b>no_tempfile</b> is 0 then the file will be written
- * atomically. */
-int
-write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin,
-                     int no_tempfile)
-{
-  int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
-
-  if (no_tempfile) {
-    /* O_APPEND stops write_chunks_to_file from using tempfiles */
-    flags |= O_APPEND;
-  }
-  return write_chunks_to_file_impl(fname, chunks, flags);
-}
-
-/** Write <b>len</b> bytes, starting at <b>str</b>, to <b>fname</b>
-    using the open() flags passed in <b>flags</b>. */
-static int
-write_bytes_to_file_impl(const char *fname, const char *str, size_t len,
-                         int flags)
-{
-  int r;
-  sized_chunk_t c = { str, len };
-  smartlist_t *chunks = smartlist_new();
-  smartlist_add(chunks, &c);
-  r = write_chunks_to_file_impl(fname, chunks, flags);
-  smartlist_free(chunks);
-  return r;
-}
-
-/** As write_str_to_file, but does not assume a NUL-terminated
- * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */
-MOCK_IMPL(int,
-write_bytes_to_file,(const char *fname, const char *str, size_t len,
-                     int bin))
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT));
-}
-
-/** As write_bytes_to_file, but if the file already exists, append the bytes
- * to the end of the file instead of overwriting it. */
-int
-append_bytes_to_file(const char *fname, const char *str, size_t len,
-                     int bin)
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
-}
-
-/** Like write_str_to_file(), but also return -1 if there was a file
-    already residing in <b>fname</b>. */
-int
-write_bytes_to_new_file(const char *fname, const char *str, size_t len,
-                        int bin)
-{
-  return write_bytes_to_file_impl(fname, str, len,
-                                  OPEN_FLAGS_DONT_REPLACE|
-                                  (bin?O_BINARY:O_TEXT));
-}
-
-/**
- * Read the contents of the open file <b>fd</b> presuming it is a FIFO
- * (or similar) file descriptor for which the size of the file isn't
- * known ahead of time. Return NULL on failure, and a NUL-terminated
- * string on success.  On success, set <b>sz_out</b> to the number of
- * bytes read.
- */
-char *
-read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out)
-{
-  ssize_t r;
-  size_t pos = 0;
-  char *string = NULL;
-  size_t string_max = 0;
-
-  if (max_bytes_to_read+1 >= SIZE_T_CEILING) {
-    errno = EINVAL;
-    return NULL;
-  }
-
-  do {
-    /* XXXX This "add 1K" approach is a little goofy; if we care about
-     * performance here, we should be doubling.  But in practice we shouldn't
-     * be using this function on big files anyway. */
-    string_max = pos + 1024;
-    if (string_max > max_bytes_to_read)
-      string_max = max_bytes_to_read + 1;
-    string = tor_realloc(string, string_max);
-    r = read(fd, string + pos, string_max - pos - 1);
-    if (r < 0) {
-      int save_errno = errno;
-      tor_free(string);
-      errno = save_errno;
-      return NULL;
-    }
-
-    pos += r;
-  } while (r > 0 && pos < max_bytes_to_read);
-
-  tor_assert(pos < string_max);
-  *sz_out = pos;
-  string[pos] = '\0';
-  return string;
-}
-
-/** Read the contents of <b>filename</b> into a newly allocated
- * string; return the string on success or NULL on failure.
- *
- * If <b>stat_out</b> is provided, store the result of stat()ing the
- * file into <b>stat_out</b>.
- *
- * If <b>flags</b> &amp; RFTS_BIN, open the file in binary mode.
- * If <b>flags</b> &amp; RFTS_IGNORE_MISSING, don't warn if the file
- * doesn't exist.
- */
-/*
- * This function <em>may</em> return an erroneous result if the file
- * is modified while it is running, but must not crash or overflow.
- * Right now, the error case occurs when the file length grows between
- * the call to stat and the call to read_all: the resulting string will
- * be truncated.
- */
-MOCK_IMPL(char *,
-read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
-{
-  int fd; /* router file */
-  struct stat statbuf;
-  char *string;
-  ssize_t r;
-  int bin = flags & RFTS_BIN;
-
-  tor_assert(filename);
-
-  fd = tor_open_cloexec(filename,O_RDONLY|(bin?O_BINARY:O_TEXT),0);
-  if (fd<0) {
-    int severity = LOG_WARN;
-    int save_errno = errno;
-    if (errno == ENOENT && (flags & RFTS_IGNORE_MISSING))
-      severity = LOG_INFO;
-    log_fn(severity, LD_FS,"Could not open \"%s\": %s",filename,
-           strerror(errno));
-    errno = save_errno;
-    return NULL;
-  }
-
-  if (fstat(fd, &statbuf)<0) {
-    int save_errno = errno;
-    close(fd);
-    log_warn(LD_FS,"Could not fstat \"%s\".",filename);
-    errno = save_errno;
-    return NULL;
-  }
-
-#ifndef _WIN32
-/** When we detect that we're reading from a FIFO, don't read more than
- * this many bytes.  It's insane overkill for most uses. */
-#define FIFO_READ_MAX (1024*1024)
-  if (S_ISFIFO(statbuf.st_mode)) {
-    size_t sz = 0;
-    string = read_file_to_str_until_eof(fd, FIFO_READ_MAX, &sz);
-    int save_errno = errno;
-    if (string && stat_out) {
-      statbuf.st_size = sz;
-      memcpy(stat_out, &statbuf, sizeof(struct stat));
-    }
-    close(fd);
-    if (!string)
-      errno = save_errno;
-    return string;
-  }
-#endif /* !defined(_WIN32) */
-
-  if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
-    close(fd);
-    errno = EINVAL;
-    return NULL;
-  }
-
-  string = tor_malloc((size_t)(statbuf.st_size+1));
-
-  r = read_all(fd,string,(size_t)statbuf.st_size,0);
-  if (r<0) {
-    int save_errno = errno;
-    log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
-             strerror(errno));
-    tor_free(string);
-    close(fd);
-    errno = save_errno;
-    return NULL;
-  }
-  string[r] = '\0'; /* NUL-terminate the result. */
-
-#if defined(_WIN32) || defined(__CYGWIN__)
-  if (!bin && strchr(string, '\r')) {
-    log_debug(LD_FS, "We didn't convert CRLF to LF as well as we hoped "
-              "when reading %s. Coping.",
-              filename);
-    tor_strstrip(string, "\r");
-    r = strlen(string);
-  }
-  if (!bin) {
-    statbuf.st_size = (size_t) r;
-  } else
-#endif /* defined(_WIN32) || defined(__CYGWIN__) */
-    if (r != statbuf.st_size) {
-      /* Unless we're using text mode on win32, we'd better have an exact
-       * match for size. */
-      int save_errno = errno;
-      log_warn(LD_FS,"Could read only %d of %ld bytes of file \"%s\".",
-               (int)r, (long)statbuf.st_size,filename);
-      tor_free(string);
-      close(fd);
-      errno = save_errno;
-      return NULL;
-    }
-  close(fd);
-  if (stat_out) {
-    memcpy(stat_out, &statbuf, sizeof(struct stat));
-  }
-
-  return string;
-}
-
 #define TOR_ISODIGIT(c) ('0' <= (c) && (c) <= '7')
 
 /** Given a c-style double-quoted escaped string in <b>s</b>, extract and
@@ -2050,185 +1199,6 @@ unescape_string(const char *s, char **result, size_t *size_out)
   }
 }
 
-/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
- * enclosing quotes. Backslashes are not unescaped. Return the unquoted
- * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
-char *
-get_unquoted_path(const char *path)
-{
-  size_t len = strlen(path);
-
-  if (len == 0) {
-    return tor_strdup("");
-  }
-
-  int has_start_quote = (path[0] == '\"');
-  int has_end_quote = (len > 0 && path[len-1] == '\"');
-  if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
-    return NULL;
-  }
-
-  char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
-  char *s = unquoted_path;
-  size_t i;
-  for (i = has_start_quote; i < len - has_end_quote; i++) {
-    if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
-      *(s-1) = path[i];
-    } else if (path[i] != '\"') {
-      *s++ = path[i];
-    } else {  /* unescaped quote */
-      tor_free(unquoted_path);
-      return NULL;
-    }
-  }
-  *s = '\0';
-  return unquoted_path;
-}
-
-/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
- * string. */
-char *
-expand_filename(const char *filename)
-{
-  tor_assert(filename);
-#ifdef _WIN32
-  /* Might consider using GetFullPathName() as described here:
-   * http://etutorials.org/Programming/secure+programming/
-   *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
-   */
-  return tor_strdup(filename);
-#else /* !(defined(_WIN32)) */
-  if (*filename == '~') {
-    char *home, *result=NULL;
-    const char *rest;
-
-    if (filename[1] == '/' || filename[1] == '\0') {
-      home = getenv("HOME");
-      if (!home) {
-        log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while "
-                 "expanding \"%s\"; defaulting to \"\".", filename);
-        home = tor_strdup("");
-      } else {
-        home = tor_strdup(home);
-      }
-      rest = strlen(filename)>=2?(filename+2):"";
-    } else {
-#ifdef HAVE_PWD_H
-      char *username, *slash;
-      slash = strchr(filename, '/');
-      if (slash)
-        username = tor_strndup(filename+1,slash-filename-1);
-      else
-        username = tor_strdup(filename+1);
-      if (!(home = get_user_homedir(username))) {
-        log_warn(LD_CONFIG,"Couldn't get homedir for \"%s\"",username);
-        tor_free(username);
-        return NULL;
-      }
-      tor_free(username);
-      rest = slash ? (slash+1) : "";
-#else /* !(defined(HAVE_PWD_H)) */
-      log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
-      return tor_strdup(filename);
-#endif /* defined(HAVE_PWD_H) */
-    }
-    tor_assert(home);
-    /* Remove trailing slash. */
-    if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) {
-      home[strlen(home)-1] = '\0';
-    }
-    tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest);
-    tor_free(home);
-    return result;
-  } else {
-    return tor_strdup(filename);
-  }
-#endif /* defined(_WIN32) */
-}
-
-/** Return a new list containing the filenames in the directory <b>dirname</b>.
- * Return NULL on error or if <b>dirname</b> is not a directory.
- */
-MOCK_IMPL(smartlist_t *,
-tor_listdir, (const char *dirname))
-{
-  smartlist_t *result;
-#ifdef _WIN32
-  char *pattern=NULL;
-  TCHAR tpattern[MAX_PATH] = {0};
-  char name[MAX_PATH*2+1] = {0};
-  HANDLE handle;
-  WIN32_FIND_DATA findData;
-  tor_asprintf(&pattern, "%s\\*", dirname);
-#ifdef UNICODE
-  mbstowcs(tpattern,pattern,MAX_PATH);
-#else
-  strlcpy(tpattern, pattern, MAX_PATH);
-#endif
-  if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) {
-    tor_free(pattern);
-    return NULL;
-  }
-  result = smartlist_new();
-  while (1) {
-#ifdef UNICODE
-    wcstombs(name,findData.cFileName,MAX_PATH);
-    name[sizeof(name)-1] = '\0';
-#else
-    strlcpy(name,findData.cFileName,sizeof(name));
-#endif /* defined(UNICODE) */
-    if (strcmp(name, ".") &&
-        strcmp(name, "..")) {
-      smartlist_add_strdup(result, name);
-    }
-    if (!FindNextFile(handle, &findData)) {
-      DWORD err;
-      if ((err = GetLastError()) != ERROR_NO_MORE_FILES) {
-        char *errstr = format_win32_error(err);
-        log_warn(LD_FS, "Error reading directory '%s': %s", dirname, errstr);
-        tor_free(errstr);
-      }
-      break;
-    }
-  }
-  FindClose(handle);
-  tor_free(pattern);
-#else /* !(defined(_WIN32)) */
-  const char *prot_dname = sandbox_intern_string(dirname);
-  DIR *d;
-  struct dirent *de;
-  if (!(d = opendir(prot_dname)))
-    return NULL;
-
-  result = smartlist_new();
-  while ((de = readdir(d))) {
-    if (!strcmp(de->d_name, ".") ||
-        !strcmp(de->d_name, ".."))
-      continue;
-    smartlist_add_strdup(result, de->d_name);
-  }
-  closedir(d);
-#endif /* defined(_WIN32) */
-  return result;
-}
-
-/** Return true iff <b>filename</b> is a relative path. */
-int
-path_is_relative(const char *filename)
-{
-  if (filename && filename[0] == '/')
-    return 0;
-#ifdef _WIN32
-  else if (filename && filename[0] == '\\')
-    return 0;
-  else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
-           filename[1] == ':' && filename[2] == '\\')
-    return 0;
-#endif /* defined(_WIN32) */
-  else
-    return 1;
-}
-
 /* =====
  * Process helpers
  * ===== */

src/common/util.h

@@ -35,16 +35,9 @@
 #include "lib/log/ratelim.h"
 #include "lib/log/util_bug.h"
 #include "lib/log/escape.h"
-
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
-#ifndef O_TEXT
-#define O_TEXT 0
-#endif
-#ifndef O_NOFOLLOW
-#define O_NOFOLLOW 0
-#endif
+#include "lib/fs/dir.h"
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
 
 uint64_t tor_htonll(uint64_t a);
 uint64_t tor_ntohll(uint64_t a);
@@ -117,8 +110,13 @@ int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
 /* File helpers */
-ssize_t write_all(tor_socket_t fd, const char *buf, size_t count,int isSocket);
-ssize_t read_all(tor_socket_t fd, char *buf, size_t count, int isSocket);
+
+#define write_all(fd, buf, count, isSock) \
+  ((isSock) ? write_all_to_socket((fd), (buf), (count)) \
+            : write_all_to_fd((int)(fd), (buf), (count)))
+#define read_all(fd, buf, count, isSock) \
+  ((isSock) ? read_all_from_socket((fd), (buf), (count)) \
+            : read_all_from_fd((int)(fd), (buf), (count)))
 
 /** Status of an I/O stream. */
 enum stream_status {
@@ -132,76 +130,7 @@ const char *stream_status_to_string(enum stream_status stream_status);
 
 enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
 
-MOCK_DECL(int,tor_unlink,(const char *pathname));
-
-/** Return values from file_status(); see that function's documentation
- * for details. */
-typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
-file_status_t file_status(const char *filename);
-
-/** Possible behaviors for check_private_dir() on encountering a nonexistent
- * directory; see that function's documentation for details. */
-typedef unsigned int cpd_check_t;
-#define CPD_NONE                 0
-#define CPD_CREATE               (1u << 0)
-#define CPD_CHECK                (1u << 1)
-#define CPD_GROUP_OK             (1u << 2)
-#define CPD_GROUP_READ           (1u << 3)
-#define CPD_CHECK_MODE_ONLY      (1u << 4)
-#define CPD_RELAX_DIRMODE_CHECK  (1u << 5)
-MOCK_DECL(int, check_private_dir,
-    (const char *dirname, cpd_check_t check,
-     const char *effective_user));
-
-#define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC)
-#define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND)
-#define OPEN_FLAGS_DONT_REPLACE (O_CREAT|O_EXCL|O_APPEND|O_WRONLY)
-typedef struct open_file_t open_file_t;
-int start_writing_to_file(const char *fname, int open_flags, int mode,
-                          open_file_t **data_out);
-FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
-                                  open_file_t **data_out);
-FILE *fdopen_file(open_file_t *file_data);
-int finish_writing_to_file(open_file_t *file_data);
-int abort_writing_to_file(open_file_t *file_data);
-MOCK_DECL(int,
-write_str_to_file,(const char *fname, const char *str, int bin));
-MOCK_DECL(int,
-write_bytes_to_file,(const char *fname, const char *str, size_t len,
-                     int bin));
-/** An ad-hoc type to hold a string of characters and a count; used by
- * write_chunks_to_file. */
-typedef struct sized_chunk_t {
-  const char *bytes;
-  size_t len;
-} sized_chunk_t;
-struct smartlist_t;
-int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
-                         int bin, int no_tempfile);
-int append_bytes_to_file(const char *fname, const char *str, size_t len,
-                         int bin);
-int write_bytes_to_new_file(const char *fname, const char *str, size_t len,
-                            int bin);
-
-/** Flag for read_file_to_str: open the file in binary mode. */
-#define RFTS_BIN            1
-/** Flag for read_file_to_str: it's okay if the file doesn't exist. */
-#define RFTS_IGNORE_MISSING 2
-
-#ifndef _WIN32
-struct stat;
-#endif
-MOCK_DECL_ATTR(char *, read_file_to_str,
-               (const char *filename, int flags, struct stat *stat_out),
-               ATTR_MALLOC);
-char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
-                                 size_t *sz_out)
-  ATTR_MALLOC;
 const char *unescape_string(const char *s, char **result, size_t *size_out);
-char *get_unquoted_path(const char *path);
-char *expand_filename(const char *filename);
-MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
-int path_is_relative(const char *filename);
 
 /* Process helpers */
 void start_daemon(void);

src/include.am

@@ -7,6 +7,7 @@ include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
 include src/lib/fdio/include.am
+include src/lib/fs/include.am
 include src/lib/include.libdonna.am
 include src/lib/intmath/include.am
 include src/lib/lock/include.am

src/lib/fdio/fdio.c

@@ -94,7 +94,7 @@ tor_ftruncate(int fd)
 
 /** Minimal version of write_all, for use by logging. */
 int
-write_all_to_fd(int fd, const char *buf, size_t count)
+write_all_to_fd_minimal(int fd, const char *buf, size_t count)
 {
   size_t written = 0;
   raw_assert(count < SSIZE_MAX);

src/lib/fdio/fdio.h

@@ -12,6 +12,6 @@ off_t tor_fd_getpos(int fd);
 int tor_fd_setpos(int fd, off_t pos);
 int tor_fd_seekend(int fd);
 int tor_ftruncate(int fd);
-int write_all_to_fd(int fd, const char *buf, size_t count);
+int write_all_to_fd_minimal(int fd, const char *buf, size_t count);
 
 #endif /* !defined(TOR_FDIO_H) */

src/lib/fs/.may_include

@@ -0,0 +1,11 @@
+orconfig.h
+lib/cc/*.h
+lib/container/*.h
+lib/err/*.h
+lib/fdio/*.h
+lib/malloc/*.h
+lib/fs/*.h
+lib/sandbox/*.h
+lib/string/*.h
+lib/testsupport/testsupport.h
+lib/log/*.h

src/lib/fs/dir.c

@@ -0,0 +1,360 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/fs/dir.h"
+#include "lib/fs/path.h"
+#include "lib/fs/userdb.h"
+
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/container/smartlist.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/compat_string.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <io.h>
+#include <direct.h>
+#include <windows.h>
+#else /* !(defined(_WIN32)) */
+#include <dirent.h>
+#include <pwd.h>
+#include <grp.h>
+#endif /* defined(_WIN32) */
+
+#include <errno.h>
+#include <string.h>
+
+/** Check whether <b>dirname</b> exists and is private.  If yes return 0.
+ * If <b>dirname</b> does not exist:
+ *  - if <b>check</b>&CPD_CREATE, try to create it and return 0 on success.
+ *  - if <b>check</b>&CPD_CHECK, and we think we can create it, return 0.
+ *  - if <b>check</b>&CPD_CHECK is false, and the directory exists, return 0.
+ *  - otherwise, return -1.
+ * If CPD_GROUP_OK is set, then it's okay if the directory
+ * is group-readable, but in all cases we create the directory mode 0700.
+ * If CPD_GROUP_READ is set, existing directory behaves as CPD_GROUP_OK and
+ * if the directory is created it will use mode 0750 with group read
+ * permission. Group read privileges also assume execute permission
+ * as norm for directories. If CPD_CHECK_MODE_ONLY is set, then we don't
+ * alter the directory permissions if they are too permissive:
+ * we just return -1.
+ * When effective_user is not NULL, check permissions against the given user
+ * and its primary group.
+ */
+MOCK_IMPL(int,
+check_private_dir,(const char *dirname, cpd_check_t check,
+                   const char *effective_user))
+{
+  int r;
+  struct stat st;
+
+  tor_assert(dirname);
+
+#ifndef _WIN32
+  int fd;
+  const struct passwd *pw = NULL;
+  uid_t running_uid;
+  gid_t running_gid;
+
+  /*
+   * Goal is to harden the implementation by removing any
+   * potential for race between stat() and chmod().
+   * chmod() accepts filename as argument. If an attacker can move
+   * the file between stat() and chmod(), a potential race exists.
+   *
+   * Several suggestions taken from:
+   * https://developer.apple.com/library/mac/documentation/
+   *     Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html
+   */
+
+  /* Open directory.
+   * O_NOFOLLOW to ensure that it does not follow symbolic links */
+  fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+  /* Was there an error? Maybe the directory does not exist? */
+  if (fd == -1) {
+
+    if (errno != ENOENT) {
+      /* Other directory error */
+      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+               strerror(errno));
+      return -1;
+    }
+
+    /* Received ENOENT: Directory does not exist */
+
+    /* Should we create the directory? */
+    if (check & CPD_CREATE) {
+      log_info(LD_GENERAL, "Creating directory %s", dirname);
+      if (check & CPD_GROUP_READ) {
+        r = mkdir(dirname, 0750);
+      } else {
+        r = mkdir(dirname, 0700);
+      }
+
+      /* check for mkdir() error */
+      if (r) {
+        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+            strerror(errno));
+        return -1;
+      }
+
+      /* we just created the directory. try to open it again.
+       * permissions on the directory will be checked again below.*/
+      fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
+
+      if (fd == -1) {
+        log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
+                 dirname,
+                 strerror(errno));
+        return -1;
+      } else {
+        close(fd);
+      }
+
+    } else if (!(check & CPD_CHECK)) {
+      log_warn(LD_FS, "Directory %s does not exist.", dirname);
+      return -1;
+    }
+
+    /* XXXX In the case where check==CPD_CHECK, we should look at the
+     * parent directory a little harder. */
+    return 0;
+  }
+
+  tor_assert(fd >= 0);
+
+  //f = tor_strdup(dirname);
+  //clean_name_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", dirname);
+  //r = stat(sandbox_intern_string(f), &st);
+  r = fstat(fd, &st);
+  if (r == -1) {
+      log_warn(LD_FS, "fstat() on directory %s failed.", dirname);
+      close(fd);
+      return -1;
+  }
+  //tor_free(f);
+
+  /* check that dirname is a directory */
+  if (!(st.st_mode & S_IFDIR)) {
+    log_warn(LD_FS, "%s is not a directory", dirname);
+    close(fd);
+    return -1;
+  }
+
+  if (effective_user) {
+    /* Look up the user and group information.
+     * If we have a problem, bail out. */
+    pw = tor_getpwnam(effective_user);
+    if (pw == NULL) {
+      log_warn(LD_CONFIG, "Error setting configured user: %s not found",
+               effective_user);
+      close(fd);
+      return -1;
+    }
+    running_uid = pw->pw_uid;
+    running_gid = pw->pw_gid;
+  } else {
+    running_uid = getuid();
+    running_gid = getgid();
+  }
+  if (st.st_uid != running_uid) {
+    char *process_ownername = NULL, *file_ownername = NULL;
+
+    {
+      const struct passwd *pw_running = tor_getpwuid(running_uid);
+      process_ownername = pw_running ? tor_strdup(pw_running->pw_name) :
+        tor_strdup("<unknown>");
+    }
+
+    {
+      const struct passwd *pw_stat = tor_getpwuid(st.st_uid);
+      file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) :
+        tor_strdup("<unknown>");
+    }
+
+    log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
+        "%s (%d). Perhaps you are running Tor as the wrong user?",
+             dirname, process_ownername, (int)running_uid,
+             file_ownername, (int)st.st_uid);
+
+    tor_free(process_ownername);
+    tor_free(file_ownername);
+    close(fd);
+    return -1;
+  }
+  if ( (check & (CPD_GROUP_OK|CPD_GROUP_READ))
+       && (st.st_gid != running_gid) && (st.st_gid != 0)) {
+    struct group *gr;
+    char *process_groupname = NULL;
+    gr = getgrgid(running_gid);
+    process_groupname = gr ? tor_strdup(gr->gr_name) : tor_strdup("<unknown>");
+    gr = getgrgid(st.st_gid);
+
+    log_warn(LD_FS, "%s is not owned by this group (%s, %d) but by group "
+             "%s (%d).  Are you running Tor as the wrong user?",
+             dirname, process_groupname, (int)running_gid,
+             gr ?  gr->gr_name : "<unknown>", (int)st.st_gid);
+
+    tor_free(process_groupname);
+    close(fd);
+    return -1;
+  }
+  unsigned unwanted_bits = 0;
+  if (check & (CPD_GROUP_OK|CPD_GROUP_READ)) {
+    unwanted_bits = 0027;
+  } else {
+    unwanted_bits = 0077;
+  }
+  unsigned check_bits_filter = ~0;
+  if (check & CPD_RELAX_DIRMODE_CHECK) {
+    check_bits_filter = 0022;
+  }
+  if ((st.st_mode & unwanted_bits & check_bits_filter) != 0) {
+    unsigned new_mode;
+    if (check & CPD_CHECK_MODE_ONLY) {
+      log_warn(LD_FS, "Permissions on directory %s are too permissive.",
+               dirname);
+      close(fd);
+      return -1;
+    }
+    log_warn(LD_FS, "Fixing permissions on directory %s", dirname);
+    new_mode = st.st_mode;
+    new_mode |= 0700; /* Owner should have rwx */
+    if (check & CPD_GROUP_READ) {
+      new_mode |= 0050; /* Group should have rx */
+    }
+    new_mode &= ~unwanted_bits; /* Clear the bits that we didn't want set...*/
+    if (fchmod(fd, new_mode)) {
+      log_warn(LD_FS, "Could not chmod directory %s: %s", dirname,
+               strerror(errno));
+      close(fd);
+      return -1;
+    } else {
+      close(fd);
+      return 0;
+    }
+  }
+  close(fd);
+#else /* !(!defined(_WIN32)) */
+  /* Win32 case: we can't open() a directory. */
+  (void)effective_user;
+
+  char *f = tor_strdup(dirname);
+  clean_fname_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", f);
+  r = stat(sandbox_intern_string(f), &st);
+  tor_free(f);
+  if (r) {
+    if (errno != ENOENT) {
+      log_warn(LD_FS, "Directory %s cannot be read: %s", dirname,
+               strerror(errno));
+      return -1;
+    }
+    if (check & CPD_CREATE) {
+      log_info(LD_GENERAL, "Creating directory %s", dirname);
+      r = mkdir(dirname);
+      if (r) {
+        log_warn(LD_FS, "Error creating directory %s: %s", dirname,
+                 strerror(errno));
+        return -1;
+      }
+    } else if (!(check & CPD_CHECK)) {
+      log_warn(LD_FS, "Directory %s does not exist.", dirname);
+      return -1;
+    }
+    return 0;
+  }
+  if (!(st.st_mode & S_IFDIR)) {
+    log_warn(LD_FS, "%s is not a directory", dirname);
+    return -1;
+  }
+
+#endif /* !defined(_WIN32) */
+  return 0;
+}
+
+/** Return a new list containing the filenames in the directory <b>dirname</b>.
+ * Return NULL on error or if <b>dirname</b> is not a directory.
+ */
+MOCK_IMPL(smartlist_t *,
+tor_listdir, (const char *dirname))
+{
+  smartlist_t *result;
+#ifdef _WIN32
+  char *pattern=NULL;
+  TCHAR tpattern[MAX_PATH] = {0};
+  char name[MAX_PATH*2+1] = {0};
+  HANDLE handle;
+  WIN32_FIND_DATA findData;
+  tor_asprintf(&pattern, "%s\\*", dirname);
+#ifdef UNICODE
+  mbstowcs(tpattern,pattern,MAX_PATH);
+#else
+  strlcpy(tpattern, pattern, MAX_PATH);
+#endif
+  if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) {
+    tor_free(pattern);
+    return NULL;
+  }
+  result = smartlist_new();
+  while (1) {
+#ifdef UNICODE
+    wcstombs(name,findData.cFileName,MAX_PATH);
+    name[sizeof(name)-1] = '\0';
+#else
+    strlcpy(name,findData.cFileName,sizeof(name));
+#endif /* defined(UNICODE) */
+    if (strcmp(name, ".") &&
+        strcmp(name, "..")) {
+      smartlist_add_strdup(result, name);
+    }
+    if (!FindNextFile(handle, &findData)) {
+      DWORD err;
+      if ((err = GetLastError()) != ERROR_NO_MORE_FILES) {
+        char *errstr = format_win32_error(err);
+        log_warn(LD_FS, "Error reading directory '%s': %s", dirname, errstr);
+        tor_free(errstr);
+      }
+      break;
+    }
+  }
+  FindClose(handle);
+  tor_free(pattern);
+#else /* !(defined(_WIN32)) */
+  const char *prot_dname = sandbox_intern_string(dirname);
+  DIR *d;
+  struct dirent *de;
+  if (!(d = opendir(prot_dname)))
+    return NULL;
+
+  result = smartlist_new();
+  while ((de = readdir(d))) {
+    if (!strcmp(de->d_name, ".") ||
+        !strcmp(de->d_name, ".."))
+      continue;
+    smartlist_add_strdup(result, de->d_name);
+  }
+  closedir(d);
+#endif /* defined(_WIN32) */
+  return result;
+}

src/lib/fs/dir.h

@@ -0,0 +1,27 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIR_H
+#define TOR_DIR_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
+
+/** Possible behaviors for check_private_dir() on encountering a nonexistent
+ * directory; see that function's documentation for details. */
+typedef unsigned int cpd_check_t;
+#define CPD_NONE                 0
+#define CPD_CREATE               (1u << 0)
+#define CPD_CHECK                (1u << 1)
+#define CPD_GROUP_OK             (1u << 2)
+#define CPD_GROUP_READ           (1u << 3)
+#define CPD_CHECK_MODE_ONLY      (1u << 4)
+#define CPD_RELAX_DIRMODE_CHECK  (1u << 5)
+MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
+                                   const char *effective_user));
+
+MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
+
+#endif

src/lib/fs/files.c

@@ -0,0 +1,711 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include "lib/fs/files.h"
+#include "lib/fs/path.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/escape.h"
+#include "lib/err/torerr.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/sandbox/sandbox.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/fdio/fdio.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+/** As open(path, flags, mode), but return an fd with the close-on-exec mode
+ * set. */
+int
+tor_open_cloexec(const char *path, int flags, unsigned mode)
+{
+  int fd;
+  const char *p = sandbox_intern_string(path);
+#ifdef O_CLOEXEC
+  fd = open(p, flags|O_CLOEXEC, mode);
+  if (fd >= 0)
+    return fd;
+  /* If we got an error, see if it is EINVAL. EINVAL might indicate that,
+   * even though we were built on a system with O_CLOEXEC support, we
+   * are running on one without. */
+  if (errno != EINVAL)
+    return -1;
+#endif /* defined(O_CLOEXEC) */
+
+  log_debug(LD_FS, "Opening %s with flags %x", p, flags);
+  fd = open(p, flags, mode);
+#ifdef FD_CLOEXEC
+  if (fd >= 0) {
+    if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      close(fd);
+      return -1;
+    }
+  }
+#endif /* defined(FD_CLOEXEC) */
+  return fd;
+}
+
+/** As fopen(path,mode), but ensures that the O_CLOEXEC bit is set on the
+ * underlying file handle. */
+FILE *
+tor_fopen_cloexec(const char *path, const char *mode)
+{
+  FILE *result = fopen(path, mode);
+#ifdef FD_CLOEXEC
+  if (result != NULL) {
+    if (fcntl(fileno(result), F_SETFD, FD_CLOEXEC) == -1) {
+      log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno));
+      fclose(result);
+      return NULL;
+    }
+  }
+#endif /* defined(FD_CLOEXEC) */
+  return result;
+}
+
+/** As rename(), but work correctly with the sandbox. */
+int
+tor_rename(const char *path_old, const char *path_new)
+{
+  log_debug(LD_FS, "Renaming %s to %s", path_old, path_new);
+  return rename(sandbox_intern_string(path_old),
+                sandbox_intern_string(path_new));
+}
+
+/**
+ * Rename the file <b>from</b> to the file <b>to</b>.  On Unix, this is
+ * the same as rename(2).  On windows, this removes <b>to</b> first if
+ * it already exists.
+ * Returns 0 on success.  Returns -1 and sets errno on failure.
+ */
+int
+replace_file(const char *from, const char *to)
+{
+#ifndef _WIN32
+  return tor_rename(from, to);
+#else
+  switch (file_status(to))
+    {
+    case FN_NOENT:
+      break;
+    case FN_FILE:
+    case FN_EMPTY:
+      if (unlink(to)) return -1;
+      break;
+    case FN_ERROR:
+      return -1;
+    case FN_DIR:
+      errno = EISDIR;
+      return -1;
+    }
+  return tor_rename(from,to);
+#endif /* !defined(_WIN32) */
+}
+
+/** Change <b>fname</b>'s modification time to now. */
+int
+touch_file(const char *fname)
+{
+  if (utime(fname, NULL)!=0)
+    return -1;
+  return 0;
+}
+
+/** Wrapper for unlink() to make it mockable for the test suite; returns 0
+ * if unlinking the file succeeded, -1 and sets errno if unlinking fails.
+ */
+
+MOCK_IMPL(int,
+tor_unlink,(const char *pathname))
+{
+  return unlink(pathname);
+}
+
+/** Write <b>count</b> bytes from <b>buf</b> to <b>fd</b>. Return the number
+ * of bytes written, or -1 on error.  Only use if fd is a blocking fd.  */
+ssize_t
+write_all_to_fd(int fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  ssize_t result;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written != count) {
+    result = write(fd, buf+written, count-written);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return (ssize_t)count;
+}
+
+/** Read from <b>fd</b> to <b>buf</b>, until we get <b>count</b> bytes or
+ * reach the end of the file.  Return the number of bytes read, or -1 on
+ * error. Only use if fd is a blocking fd. */
+ssize_t
+read_all_from_fd(int fd, char *buf, size_t count)
+{
+  size_t numread = 0;
+  ssize_t result;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
+    errno = EINVAL;
+    return -1;
+  }
+
+  while (numread < count) {
+    result = read(fd, buf+numread, count-numread);
+    if (result<0)
+      return -1;
+    else if (result == 0)
+      break;
+    numread += result;
+  }
+  return (ssize_t)numread;
+}
+
+/** Return:
+ * FN_ERROR if filename can't be read, is NULL, or is zero-length,
+ * FN_NOENT if it doesn't exist,
+ * FN_FILE if it is a non-empty regular file, or a FIFO on unix-like systems,
+ * FN_EMPTY for zero-byte regular files,
+ * FN_DIR if it's a directory, and
+ * FN_ERROR for any other file type.
+ * On FN_ERROR and FN_NOENT, sets errno.  (errno is not set when FN_ERROR
+ * is returned due to an unhandled file type.) */
+file_status_t
+file_status(const char *fname)
+{
+  struct stat st;
+  char *f;
+  int r;
+  if (!fname || strlen(fname) == 0) {
+    return FN_ERROR;
+  }
+  f = tor_strdup(fname);
+  clean_fname_for_stat(f);
+  log_debug(LD_FS, "stat()ing %s", f);
+  r = stat(sandbox_intern_string(f), &st);
+  tor_free(f);
+  if (r) {
+    if (errno == ENOENT) {
+      return FN_NOENT;
+    }
+    return FN_ERROR;
+  }
+  if (st.st_mode & S_IFDIR) {
+    return FN_DIR;
+  } else if (st.st_mode & S_IFREG) {
+    if (st.st_size > 0) {
+      return FN_FILE;
+    } else if (st.st_size == 0) {
+      return FN_EMPTY;
+    } else {
+      return FN_ERROR;
+    }
+#ifndef _WIN32
+  } else if (st.st_mode & S_IFIFO) {
+    return FN_FILE;
+#endif
+  } else {
+    return FN_ERROR;
+  }
+}
+
+/** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
+ * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
+ *
+ * This function replaces the old file atomically, if possible.  This
+ * function, and all other functions in util.c that create files, create them
+ * with mode 0600.
+ */
+MOCK_IMPL(int,
+write_str_to_file,(const char *fname, const char *str, int bin))
+{
+#ifdef _WIN32
+  if (!bin && strchr(str, '\r')) {
+    log_warn(LD_BUG,
+             "We're writing a text string that already contains a CR to %s",
+             escaped(fname));
+  }
+#endif /* defined(_WIN32) */
+  return write_bytes_to_file(fname, str, strlen(str), bin);
+}
+
+/** Represents a file that we're writing to, with support for atomic commit:
+ * we can write into a temporary file, and either remove the file on
+ * failure, or replace the original file on success. */
+struct open_file_t {
+  char *tempname; /**< Name of the temporary file. */
+  char *filename; /**< Name of the original file. */
+  unsigned rename_on_close:1; /**< Are we using the temporary file or not? */
+  unsigned binary:1; /**< Did we open in binary mode? */
+  int fd; /**< fd for the open file. */
+  FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */
+};
+
+/** Try to start writing to the file in <b>fname</b>, passing the flags
+ * <b>open_flags</b> to the open() syscall, creating the file (if needed) with
+ * access value <b>mode</b>.  If the O_APPEND flag is set, we append to the
+ * original file.  Otherwise, we open a new temporary file in the same
+ * directory, and either replace the original or remove the temporary file
+ * when we're done.
+ *
+ * Return the fd for the newly opened file, and store working data in
+ * *<b>data_out</b>.  The caller should not close the fd manually:
+ * instead, call finish_writing_to_file() or abort_writing_to_file().
+ * Returns -1 on failure.
+ *
+ * NOTE: When not appending, the flags O_CREAT and O_TRUNC are treated
+ * as true and the flag O_EXCL is treated as false.
+ *
+ * NOTE: Ordinarily, O_APPEND means "seek to the end of the file before each
+ * write()".  We don't do that.
+ */
+int
+start_writing_to_file(const char *fname, int open_flags, int mode,
+                      open_file_t **data_out)
+{
+  open_file_t *new_file = tor_malloc_zero(sizeof(open_file_t));
+  const char *open_name;
+  int append = 0;
+
+  tor_assert(fname);
+  tor_assert(data_out);
+#if (O_BINARY != 0 && O_TEXT != 0)
+  tor_assert((open_flags & (O_BINARY|O_TEXT)) != 0);
+#endif
+  new_file->fd = -1;
+  new_file->filename = tor_strdup(fname);
+  if (open_flags & O_APPEND) {
+    open_name = fname;
+    new_file->rename_on_close = 0;
+    append = 1;
+    open_flags &= ~O_APPEND;
+  } else {
+    tor_asprintf(&new_file->tempname, "%s.tmp", fname);
+    open_name = new_file->tempname;
+    /* We always replace an existing temporary file if there is one. */
+    open_flags |= O_CREAT|O_TRUNC;
+    open_flags &= ~O_EXCL;
+    new_file->rename_on_close = 1;
+  }
+#if O_BINARY != 0
+  if (open_flags & O_BINARY)
+    new_file->binary = 1;
+#endif
+
+  new_file->fd = tor_open_cloexec(open_name, open_flags, mode);
+  if (new_file->fd < 0) {
+    log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s",
+        open_name, fname, strerror(errno));
+    goto err;
+  }
+  if (append) {
+    if (tor_fd_seekend(new_file->fd) < 0) {
+      log_warn(LD_FS, "Couldn't seek to end of file \"%s\": %s", open_name,
+               strerror(errno));
+      goto err;
+    }
+  }
+
+  *data_out = new_file;
+
+  return new_file->fd;
+
+ err:
+  if (new_file->fd >= 0)
+    close(new_file->fd);
+  *data_out = NULL;
+  tor_free(new_file->filename);
+  tor_free(new_file->tempname);
+  tor_free(new_file);
+  return -1;
+}
+
+/** Given <b>file_data</b> from start_writing_to_file(), return a stdio FILE*
+ * that can be used to write to the same file.  The caller should not mix
+ * stdio calls with non-stdio calls. */
+FILE *
+fdopen_file(open_file_t *file_data)
+{
+  tor_assert(file_data);
+  if (file_data->stdio_file)
+    return file_data->stdio_file;
+  tor_assert(file_data->fd >= 0);
+  if (!(file_data->stdio_file = fdopen(file_data->fd,
+                                       file_data->binary?"ab":"a"))) {
+    log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename,
+             file_data->fd, strerror(errno));
+  }
+  return file_data->stdio_file;
+}
+
+/** Combines start_writing_to_file with fdopen_file(): arguments are as
+ * for start_writing_to_file, but  */
+FILE *
+start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+                            open_file_t **data_out)
+{
+  FILE *res;
+  if (start_writing_to_file(fname, open_flags, mode, data_out)<0)
+    return NULL;
+  if (!(res = fdopen_file(*data_out))) {
+    abort_writing_to_file(*data_out);
+    *data_out = NULL;
+  }
+  return res;
+}
+
+/** Helper function: close and free the underlying file and memory in
+ * <b>file_data</b>.  If we were writing into a temporary file, then delete
+ * that file (if abort_write is true) or replaces the target file with
+ * the temporary file (if abort_write is false). */
+static int
+finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
+{
+  int r = 0;
+
+  tor_assert(file_data && file_data->filename);
+  if (file_data->stdio_file) {
+    if (fclose(file_data->stdio_file)) {
+      log_warn(LD_FS, "Error closing \"%s\": %s", file_data->filename,
+               strerror(errno));
+      abort_write = r = -1;
+    }
+  } else if (file_data->fd >= 0 && close(file_data->fd) < 0) {
+    log_warn(LD_FS, "Error flushing \"%s\": %s", file_data->filename,
+             strerror(errno));
+    abort_write = r = -1;
+  }
+
+  if (file_data->rename_on_close) {
+    tor_assert(file_data->tempname && file_data->filename);
+    if (!abort_write) {
+      tor_assert(strcmp(file_data->filename, file_data->tempname));
+      if (replace_file(file_data->tempname, file_data->filename)) {
+        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+                 strerror(errno));
+        abort_write = r = -1;
+      }
+    }
+    if (abort_write) {
+      int res = unlink(file_data->tempname);
+      if (res != 0) {
+        /* We couldn't unlink and we'll leave a mess behind */
+        log_warn(LD_FS, "Failed to unlink %s: %s",
+                 file_data->tempname, strerror(errno));
+        r = -1;
+      }
+    }
+  }
+
+  tor_free(file_data->filename);
+  tor_free(file_data->tempname);
+  tor_free(file_data);
+
+  return r;
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, replace the original file with
+ * the temporary file. */
+int
+finish_writing_to_file(open_file_t *file_data)
+{
+  return finish_writing_to_file_impl(file_data, 0);
+}
+
+/** Finish writing to <b>file_data</b>: close the file handle, free memory as
+ * needed, and if using a temporary file, delete it. */
+int
+abort_writing_to_file(open_file_t *file_data)
+{
+  return finish_writing_to_file_impl(file_data, 1);
+}
+
+/** Helper: given a set of flags as passed to open(2), open the file
+ * <b>fname</b> and write all the sized_chunk_t structs in <b>chunks</b> to
+ * the file.  Do so as atomically as possible e.g. by opening temp files and
+ * renaming. */
+static int
+write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks,
+                          int open_flags)
+{
+  open_file_t *file = NULL;
+  int fd;
+  ssize_t result;
+  fd = start_writing_to_file(fname, open_flags, 0600, &file);
+  if (fd<0)
+    return -1;
+  SMARTLIST_FOREACH(chunks, sized_chunk_t *, chunk,
+  {
+    result = write_all_to_fd(fd, chunk->bytes, chunk->len);
+    if (result < 0) {
+      log_warn(LD_FS, "Error writing to \"%s\": %s", fname,
+          strerror(errno));
+      goto err;
+    }
+    tor_assert((size_t)result == chunk->len);
+  });
+
+  return finish_writing_to_file(file);
+ err:
+  abort_writing_to_file(file);
+  return -1;
+}
+
+/** Given a smartlist of sized_chunk_t, write them to a file
+ * <b>fname</b>, overwriting or creating the file as necessary.
+ * If <b>no_tempfile</b> is 0 then the file will be written
+ * atomically. */
+int
+write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin,
+                     int no_tempfile)
+{
+  int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT);
+
+  if (no_tempfile) {
+    /* O_APPEND stops write_chunks_to_file from using tempfiles */
+    flags |= O_APPEND;
+  }
+  return write_chunks_to_file_impl(fname, chunks, flags);
+}
+
+/** Write <b>len</b> bytes, starting at <b>str</b>, to <b>fname</b>
+    using the open() flags passed in <b>flags</b>. */
+static int
+write_bytes_to_file_impl(const char *fname, const char *str, size_t len,
+                         int flags)
+{
+  int r;
+  sized_chunk_t c = { str, len };
+  smartlist_t *chunks = smartlist_new();
+  smartlist_add(chunks, &c);
+  r = write_chunks_to_file_impl(fname, chunks, flags);
+  smartlist_free(chunks);
+  return r;
+}
+
+/** As write_str_to_file, but does not assume a NUL-terminated
+ * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */
+MOCK_IMPL(int,
+write_bytes_to_file,(const char *fname, const char *str, size_t len,
+                     int bin))
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT));
+}
+
+/** As write_bytes_to_file, but if the file already exists, append the bytes
+ * to the end of the file instead of overwriting it. */
+int
+append_bytes_to_file(const char *fname, const char *str, size_t len,
+                     int bin)
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_APPEND|(bin?O_BINARY:O_TEXT));
+}
+
+/** Like write_str_to_file(), but also return -1 if there was a file
+    already residing in <b>fname</b>. */
+int
+write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+                        int bin)
+{
+  return write_bytes_to_file_impl(fname, str, len,
+                                  OPEN_FLAGS_DONT_REPLACE|
+                                  (bin?O_BINARY:O_TEXT));
+}
+
+/**
+ * Read the contents of the open file <b>fd</b> presuming it is a FIFO
+ * (or similar) file descriptor for which the size of the file isn't
+ * known ahead of time. Return NULL on failure, and a NUL-terminated
+ * string on success.  On success, set <b>sz_out</b> to the number of
+ * bytes read.
+ */
+char *
+read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out)
+{
+  ssize_t r;
+  size_t pos = 0;
+  char *string = NULL;
+  size_t string_max = 0;
+
+  if (max_bytes_to_read+1 >= SIZE_T_CEILING) {
+    errno = EINVAL;
+    return NULL;
+  }
+
+  do {
+    /* XXXX This "add 1K" approach is a little goofy; if we care about
+     * performance here, we should be doubling.  But in practice we shouldn't
+     * be using this function on big files anyway. */
+    string_max = pos + 1024;
+    if (string_max > max_bytes_to_read)
+      string_max = max_bytes_to_read + 1;
+    string = tor_realloc(string, string_max);
+    r = read(fd, string + pos, string_max - pos - 1);
+    if (r < 0) {
+      int save_errno = errno;
+      tor_free(string);
+      errno = save_errno;
+      return NULL;
+    }
+
+    pos += r;
+  } while (r > 0 && pos < max_bytes_to_read);
+
+  tor_assert(pos < string_max);
+  *sz_out = pos;
+  string[pos] = '\0';
+  return string;
+}
+
+/** Read the contents of <b>filename</b> into a newly allocated
+ * string; return the string on success or NULL on failure.
+ *
+ * If <b>stat_out</b> is provided, store the result of stat()ing the
+ * file into <b>stat_out</b>.
+ *
+ * If <b>flags</b> &amp; RFTS_BIN, open the file in binary mode.
+ * If <b>flags</b> &amp; RFTS_IGNORE_MISSING, don't warn if the file
+ * doesn't exist.
+ */
+/*
+ * This function <em>may</em> return an erroneous result if the file
+ * is modified while it is running, but must not crash or overflow.
+ * Right now, the error case occurs when the file length grows between
+ * the call to stat and the call to read_all: the resulting string will
+ * be truncated.
+ */
+MOCK_IMPL(char *,
+read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
+{
+  int fd; /* router file */
+  struct stat statbuf;
+  char *string;
+  ssize_t r;
+  int bin = flags & RFTS_BIN;
+
+  tor_assert(filename);
+
+  fd = tor_open_cloexec(filename,O_RDONLY|(bin?O_BINARY:O_TEXT),0);
+  if (fd<0) {
+    int severity = LOG_WARN;
+    int save_errno = errno;
+    if (errno == ENOENT && (flags & RFTS_IGNORE_MISSING))
+      severity = LOG_INFO;
+    log_fn(severity, LD_FS,"Could not open \"%s\": %s",filename,
+           strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  if (fstat(fd, &statbuf)<0) {
+    int save_errno = errno;
+    close(fd);
+    log_warn(LD_FS,"Could not fstat \"%s\".",filename);
+    errno = save_errno;
+    return NULL;
+  }
+
+#ifndef _WIN32
+/** When we detect that we're reading from a FIFO, don't read more than
+ * this many bytes.  It's insane overkill for most uses. */
+#define FIFO_READ_MAX (1024*1024)
+  if (S_ISFIFO(statbuf.st_mode)) {
+    size_t sz = 0;
+    string = read_file_to_str_until_eof(fd, FIFO_READ_MAX, &sz);
+    int save_errno = errno;
+    if (string && stat_out) {
+      statbuf.st_size = sz;
+      memcpy(stat_out, &statbuf, sizeof(struct stat));
+    }
+    close(fd);
+    if (!string)
+      errno = save_errno;
+    return string;
+  }
+#endif /* !defined(_WIN32) */
+
+  if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
+    close(fd);
+    errno = EINVAL;
+    return NULL;
+  }
+
+  string = tor_malloc((size_t)(statbuf.st_size+1));
+
+  r = read_all_from_fd(fd,string,(size_t)statbuf.st_size);
+  if (r<0) {
+    int save_errno = errno;
+    log_warn(LD_FS,"Error reading from file \"%s\": %s", filename,
+             strerror(errno));
+    tor_free(string);
+    close(fd);
+    errno = save_errno;
+    return NULL;
+  }
+  string[r] = '\0'; /* NUL-terminate the result. */
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+  if (!bin && strchr(string, '\r')) {
+    log_debug(LD_FS, "We didn't convert CRLF to LF as well as we hoped "
+              "when reading %s. Coping.",
+              filename);
+    tor_strstrip(string, "\r");
+    r = strlen(string);
+  }
+  if (!bin) {
+    statbuf.st_size = (size_t) r;
+  } else
+#endif /* defined(_WIN32) || defined(__CYGWIN__) */
+    if (r != statbuf.st_size) {
+      /* Unless we're using text mode on win32, we'd better have an exact
+       * match for size. */
+      int save_errno = errno;
+      log_warn(LD_FS,"Could read only %d of %ld bytes of file \"%s\".",
+               (int)r, (long)statbuf.st_size,filename);
+      tor_free(string);
+      close(fd);
+      errno = save_errno;
+      return NULL;
+    }
+  close(fd);
+  if (stat_out) {
+    memcpy(stat_out, &statbuf, sizeof(struct stat));
+  }
+
+  return string;
+}

src/lib/fs/files.h

@@ -0,0 +1,100 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_FS_H
+#define TOR_FS_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+/* We need these for struct stat to work */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#endif
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_TEXT
+#define O_TEXT 0
+#endif
+#ifndef O_NOFOLLOW
+#define O_NOFOLLOW 0
+#endif
+
+struct stat;
+
+int tor_open_cloexec(const char *path, int flags, unsigned mode);
+FILE *tor_fopen_cloexec(const char *path, const char *mode);
+int tor_rename(const char *path_old, const char *path_new);
+
+int replace_file(const char *from, const char *to);
+int touch_file(const char *fname);
+
+MOCK_DECL(int,tor_unlink,(const char *pathname));
+
+/** Return values from file_status(); see that function's documentation
+ * for details. */
+typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
+
+file_status_t file_status(const char *filename);
+
+int64_t tor_get_avail_disk_space(const char *path);
+
+ssize_t write_all_to_fd(int fd, const char *buf, size_t count);
+ssize_t read_all_from_fd(int fd, char *buf, size_t count);
+
+#define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC)
+#define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND)
+#define OPEN_FLAGS_DONT_REPLACE (O_CREAT|O_EXCL|O_APPEND|O_WRONLY)
+typedef struct open_file_t open_file_t;
+int start_writing_to_file(const char *fname, int open_flags, int mode,
+                          open_file_t **data_out);
+FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode,
+                                  open_file_t **data_out);
+FILE *fdopen_file(open_file_t *file_data);
+int finish_writing_to_file(open_file_t *file_data);
+int abort_writing_to_file(open_file_t *file_data);
+MOCK_DECL(int, write_str_to_file,(const char *fname, const char *str,
+                                  int bin));
+MOCK_DECL(int, write_bytes_to_file,(const char *fname, const char *str,
+                                    size_t len,int bin));
+
+/** An ad-hoc type to hold a string of characters and a count; used by
+ * write_chunks_to_file. */
+typedef struct sized_chunk_t {
+  const char *bytes;
+  size_t len;
+} sized_chunk_t;
+struct smartlist_t;
+int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks,
+                         int bin, int no_tempfile);
+int append_bytes_to_file(const char *fname, const char *str, size_t len,
+                         int bin);
+int write_bytes_to_new_file(const char *fname, const char *str, size_t len,
+                            int bin);
+
+/** Flag for read_file_to_str: open the file in binary mode. */
+#define RFTS_BIN            1
+/** Flag for read_file_to_str: it's okay if the file doesn't exist. */
+#define RFTS_IGNORE_MISSING 2
+
+MOCK_DECL_ATTR(char *, read_file_to_str,(const char *filename, int flags,
+                                         struct stat *stat_out),
+               ATTR_MALLOC);
+char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
+                                 size_t *sz_out)
+  ATTR_MALLOC;
+
+#endif

src/lib/fs/include.am

@@ -0,0 +1,25 @@
+
+noinst_LIBRARIES += src/lib/libtor-fs.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-fs-testing.a
+endif
+
+src_lib_libtor_fs_a_SOURCES =			\
+	src/lib/fs/dir.c			\
+	src/lib/fs/files.c			\
+	src/lib/fs/mmap.c			\
+	src/lib/fs/path.c			\
+	src/lib/fs/userdb.c
+
+src_lib_libtor_fs_testing_a_SOURCES = \
+	$(src_lib_libtor_fs_a_SOURCES)
+src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=					\
+	src/lib/fs/dir.h				\
+	src/lib/fs/files.h				\
+	src/lib/fs/mmap.h				\
+	src/lib/fs/path.h				\
+	src/lib/fs/userdb.h

src/lib/fs/mmap.c

@@ -0,0 +1,234 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/fs/mmap.h"
+#include "lib/fs/files.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/string/compat_string.h"
+#include "lib/malloc/util_malloc.h"
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
+/** Try to create a memory mapping for <b>filename</b> and return it.  On
+ * failure, return NULL. Sets errno properly, using ERANGE to mean
+ * "empty file". Must only be called on trusted Tor-owned files, as changing
+ * the underlying file's size causes unspecified behavior. */
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+  int fd; /* router file */
+  char *string;
+  int result;
+  tor_mmap_t *res;
+  size_t size, filesize;
+  struct stat st;
+
+  tor_assert(filename);
+
+  fd = tor_open_cloexec(filename, O_RDONLY, 0);
+  if (fd<0) {
+    int save_errno = errno;
+    int severity = (errno == ENOENT) ? LOG_INFO : LOG_WARN;
+    log_fn(severity, LD_FS,"Could not open \"%s\" for mmap(): %s",filename,
+           strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  /* Get the size of the file */
+  result = fstat(fd, &st);
+  if (result != 0) {
+    int save_errno = errno;
+    log_warn(LD_FS,
+             "Couldn't fstat opened descriptor for \"%s\" during mmap: %s",
+             filename, strerror(errno));
+    close(fd);
+    errno = save_errno;
+    return NULL;
+  }
+  size = filesize = (size_t)(st.st_size);
+
+  if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
+    log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
+    errno = EFBIG;
+    close(fd);
+    return NULL;
+  }
+  if (!size) {
+    /* Zero-length file. If we call mmap on it, it will succeed but
+     * return NULL, and bad things will happen. So just fail. */
+    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+    errno = ERANGE;
+    close(fd);
+    return NULL;
+  }
+
+  string = mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
+  close(fd);
+  if (string == MAP_FAILED) {
+    int save_errno = errno;
+    log_warn(LD_FS,"Could not mmap file \"%s\": %s", filename,
+             strerror(errno));
+    errno = save_errno;
+    return NULL;
+  }
+
+  res = tor_malloc_zero(sizeof(tor_mmap_t));
+  res->data = string;
+  res->size = filesize;
+  res->mapping_size = size;
+
+  return res;
+}
+/** Release storage held for a memory mapping; returns 0 on success,
+ * or -1 on failure (and logs a warning). */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+  int res;
+
+  if (handle == NULL)
+    return 0;
+
+  res = munmap((char*)handle->data, handle->mapping_size);
+  if (res == 0) {
+    /* munmap() succeeded */
+    tor_free(handle);
+  } else {
+    log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s",
+             strerror(errno));
+    res = -1;
+  }
+
+  return res;
+}
+#elif defined(_WIN32)
+tor_mmap_t *
+tor_mmap_file(const char *filename)
+{
+  TCHAR tfilename[MAX_PATH]= {0};
+  tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
+  int empty = 0;
+  HANDLE file_handle = INVALID_HANDLE_VALUE;
+  DWORD size_low, size_high;
+  uint64_t real_size;
+  res->mmap_handle = NULL;
+#ifdef UNICODE
+  mbstowcs(tfilename,filename,MAX_PATH);
+#else
+  strlcpy(tfilename,filename,MAX_PATH);
+#endif
+  file_handle = CreateFile(tfilename,
+                           GENERIC_READ, FILE_SHARE_READ,
+                           NULL,
+                           OPEN_EXISTING,
+                           FILE_ATTRIBUTE_NORMAL,
+                           0);
+
+  if (file_handle == INVALID_HANDLE_VALUE)
+    goto win_err;
+
+  size_low = GetFileSize(file_handle, &size_high);
+
+  if (size_low == INVALID_FILE_SIZE && GetLastError() != NO_ERROR) {
+    log_warn(LD_FS,"Error getting size of \"%s\".",filename);
+    goto win_err;
+  }
+  if (size_low == 0 && size_high == 0) {
+    log_info(LD_FS,"File \"%s\" is empty. Ignoring.",filename);
+    empty = 1;
+    goto err;
+  }
+  real_size = (((uint64_t)size_high)<<32) | size_low;
+  if (real_size > SIZE_MAX) {
+    log_warn(LD_FS,"File \"%s\" is too big to map; not trying.",filename);
+    goto err;
+  }
+  res->size = real_size;
+
+  res->mmap_handle = CreateFileMapping(file_handle,
+                                       NULL,
+                                       PAGE_READONLY,
+                                       size_high,
+                                       size_low,
+                                       NULL);
+  if (res->mmap_handle == NULL)
+    goto win_err;
+  res->data = (char*) MapViewOfFile(res->mmap_handle,
+                                    FILE_MAP_READ,
+                                    0, 0, 0);
+  if (!res->data)
+    goto win_err;
+
+  CloseHandle(file_handle);
+  return res;
+ win_err: {
+    DWORD e = GetLastError();
+    int severity = (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) ?
+      LOG_INFO : LOG_WARN;
+    char *msg = format_win32_error(e);
+    log_fn(severity, LD_FS, "Couldn't mmap file \"%s\": %s", filename, msg);
+    tor_free(msg);
+    if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND)
+      errno = ENOENT;
+    else
+      errno = EINVAL;
+  }
+ err:
+  if (empty)
+    errno = ERANGE;
+  if (file_handle != INVALID_HANDLE_VALUE)
+    CloseHandle(file_handle);
+  tor_munmap_file(res);
+  return NULL;
+}
+
+/* Unmap the file, and return 0 for success or -1 for failure */
+int
+tor_munmap_file(tor_mmap_t *handle)
+{
+  if (handle == NULL)
+    return 0;
+
+  if (handle->data) {
+    /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would
+       have to be redefined as non-const. */
+    BOOL ok = UnmapViewOfFile( (LPVOID) handle->data);
+    if (!ok) {
+      log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d",
+               (int)GetLastError());
+    }
+  }
+
+  if (handle->mmap_handle != NULL)
+    CloseHandle(handle->mmap_handle);
+  tor_free(handle);
+
+  return 0;
+}
+#else
+#error "cannot implement tor_mmap_file"
+#endif /* defined(HAVE_MMAP) || ... || ... */

src/lib/fs/mmap.h

@@ -0,0 +1,35 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MMAP_H
+#define TOR_MMAP_H
+
+#include "lib/cc/compat_compiler.h"
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windef.h>
+#endif
+
+/** Represents an mmaped file. Allocated via tor_mmap_file; freed with
+ * tor_munmap_file. */
+typedef struct tor_mmap_t {
+  const char *data; /**< Mapping of the file's contents. */
+  size_t size; /**< Size of the file. */
+
+  /* None of the fields below should be accessed from outside compat.c */
+#ifdef HAVE_MMAP
+  size_t mapping_size; /**< Size of the actual mapping. (This is this file
+                        * size, rounded up to the nearest page.) */
+#elif defined _WIN32
+  HANDLE mmap_handle;
+#endif /* defined(HAVE_MMAP) || ... */
+
+} tor_mmap_t;
+
+tor_mmap_t *tor_mmap_file(const char *filename) ATTR_NONNULL((1));
+int tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1));
+
+#endif

src/lib/fs/path.c

@@ -0,0 +1,289 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/fs/path.h"
+#include "lib/malloc/util_malloc.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/fs/userdb.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <errno.h>
+#include <string.h>
+
+/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
+ * enclosing quotes. Backslashes are not unescaped. Return the unquoted
+ * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
+char *
+get_unquoted_path(const char *path)
+{
+  size_t len = strlen(path);
+
+  if (len == 0) {
+    return tor_strdup("");
+  }
+
+  int has_start_quote = (path[0] == '\"');
+  int has_end_quote = (len > 0 && path[len-1] == '\"');
+  if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
+    return NULL;
+  }
+
+  char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
+  char *s = unquoted_path;
+  size_t i;
+  for (i = has_start_quote; i < len - has_end_quote; i++) {
+    if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
+      *(s-1) = path[i];
+    } else if (path[i] != '\"') {
+      *s++ = path[i];
+    } else {  /* unescaped quote */
+      tor_free(unquoted_path);
+      return NULL;
+    }
+  }
+  *s = '\0';
+  return unquoted_path;
+}
+
+/** Expand any homedir prefix on <b>filename</b>; return a newly allocated
+ * string. */
+char *
+expand_filename(const char *filename)
+{
+  tor_assert(filename);
+#ifdef _WIN32
+  /* Might consider using GetFullPathName() as described here:
+   * http://etutorials.org/Programming/secure+programming/
+   *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
+   */
+  return tor_strdup(filename);
+#else /* !(defined(_WIN32)) */
+  if (*filename == '~') {
+    char *home, *result=NULL;
+    const char *rest;
+
+    if (filename[1] == '/' || filename[1] == '\0') {
+      home = getenv("HOME");
+      if (!home) {
+        log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while "
+                 "expanding \"%s\"; defaulting to \"\".", filename);
+        home = tor_strdup("");
+      } else {
+        home = tor_strdup(home);
+      }
+      rest = strlen(filename)>=2?(filename+2):"";
+    } else {
+#ifdef HAVE_PWD_H
+      char *username, *slash;
+      slash = strchr(filename, '/');
+      if (slash)
+        username = tor_strndup(filename+1,slash-filename-1);
+      else
+        username = tor_strdup(filename+1);
+      if (!(home = get_user_homedir(username))) {
+        log_warn(LD_CONFIG,"Couldn't get homedir for \"%s\"",username);
+        tor_free(username);
+        return NULL;
+      }
+      tor_free(username);
+      rest = slash ? (slash+1) : "";
+#else /* !(defined(HAVE_PWD_H)) */
+      log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
+      return tor_strdup(filename);
+#endif /* defined(HAVE_PWD_H) */
+    }
+    tor_assert(home);
+    /* Remove trailing slash. */
+    if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) {
+      home[strlen(home)-1] = '\0';
+    }
+    tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest);
+    tor_free(home);
+    return result;
+  } else {
+    return tor_strdup(filename);
+  }
+#endif /* defined(_WIN32) */
+}
+
+/** Return true iff <b>filename</b> is a relative path. */
+int
+path_is_relative(const char *filename)
+{
+  if (filename && filename[0] == '/')
+    return 0;
+#ifdef _WIN32
+  else if (filename && filename[0] == '\\')
+    return 0;
+  else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
+           filename[1] == ':' && filename[2] == '\\')
+    return 0;
+#endif /* defined(_WIN32) */
+  else
+    return 1;
+}
+
+/** Clean up <b>name</b> so that we can use it in a call to "stat".  On Unix,
+ * we do nothing.  On Windows, we remove a trailing slash, unless the path is
+ * the root of a disk. */
+void
+clean_fname_for_stat(char *name)
+{
+#ifdef _WIN32
+  size_t len = strlen(name);
+  if (!len)
+    return;
+  if (name[len-1]=='\\' || name[len-1]=='/') {
+    if (len == 1 || (len==3 && name[1]==':'))
+      return;
+    name[len-1]='\0';
+  }
+#else /* !(defined(_WIN32)) */
+  (void)name;
+#endif /* defined(_WIN32) */
+}
+
+/** Modify <b>fname</b> to contain the name of its parent directory.  Doesn't
+ * actually examine the filesystem; does a purely syntactic modification.
+ *
+ * The parent of the root director is considered to be iteself.
+ *
+ * Path separators are the forward slash (/) everywhere and additionally
+ * the backslash (\) on Win32.
+ *
+ * Cuts off any number of trailing path separators but otherwise ignores
+ * them for purposes of finding the parent directory.
+ *
+ * Returns 0 if a parent directory was successfully found, -1 otherwise (fname
+ * did not have any path separators or only had them at the end).
+ * */
+int
+get_parent_directory(char *fname)
+{
+  char *cp;
+  int at_end = 1;
+  tor_assert(fname);
+#ifdef _WIN32
+  /* If we start with, say, c:, then don't consider that the start of the path
+   */
+  if (fname[0] && fname[1] == ':') {
+    fname += 2;
+  }
+#endif /* defined(_WIN32) */
+  /* Now we want to remove all path-separators at the end of the string,
+   * and to remove the end of the string starting with the path separator
+   * before the last non-path-separator.  In perl, this would be
+   *   s#[/]*$##; s#/[^/]*$##;
+   * on a unixy platform.
+   */
+  cp = fname + strlen(fname);
+  at_end = 1;
+  while (--cp >= fname) {
+    int is_sep = (*cp == '/'
+#ifdef _WIN32
+                  || *cp == '\\'
+#endif
+                  );
+    if (is_sep) {
+      if (cp == fname) {
+        /* This is the first separator in the file name; don't remove it! */
+        cp[1] = '\0';
+        return 0;
+      }
+      *cp = '\0';
+      if (! at_end)
+        return 0;
+    } else {
+      at_end = 0;
+    }
+  }
+  return -1;
+}
+
+#ifndef _WIN32
+/** Return a newly allocated string containing the output of getcwd(). Return
+ * NULL on failure. (We can't just use getcwd() into a PATH_MAX buffer, since
+ * Hurd hasn't got a PATH_MAX.)
+ */
+static char *
+alloc_getcwd(void)
+{
+#ifdef HAVE_GET_CURRENT_DIR_NAME
+  /* Glibc makes this nice and simple for us. */
+  char *cwd = get_current_dir_name();
+  char *result = NULL;
+  if (cwd) {
+    /* We make a copy here, in case tor_malloc() is not malloc(). */
+    result = tor_strdup(cwd);
+    raw_free(cwd); // alias for free to avoid tripping check-spaces.
+  }
+  return result;
+#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+  size_t size = 1024;
+  char *buf = NULL;
+  char *ptr = NULL;
+
+  while (ptr == NULL) {
+    buf = tor_realloc(buf, size);
+    ptr = getcwd(buf, size);
+
+    if (ptr == NULL && errno != ERANGE) {
+      tor_free(buf);
+      return NULL;
+    }
+
+    size *= 2;
+  }
+  return buf;
+#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */
+}
+#endif /* !defined(_WIN32) */
+
+/** Expand possibly relative path <b>fname</b> to an absolute path.
+ * Return a newly allocated string, possibly equal to <b>fname</b>. */
+char *
+make_path_absolute(char *fname)
+{
+#ifdef _WIN32
+  char *absfname_malloced = _fullpath(NULL, fname, 1);
+
+  /* We don't want to assume that tor_free can free a string allocated
+   * with malloc.  On failure, return fname (it's better than nothing). */
+  char *absfname = tor_strdup(absfname_malloced ? absfname_malloced : fname);
+  if (absfname_malloced) raw_free(absfname_malloced);
+
+  return absfname;
+#else /* !(defined(_WIN32)) */
+  char *absfname = NULL, *path = NULL;
+
+  tor_assert(fname);
+
+  if (fname[0] == '/') {
+    absfname = tor_strdup(fname);
+  } else {
+    path = alloc_getcwd();
+    if (path) {
+      tor_asprintf(&absfname, "%s/%s", path, fname);
+      tor_free(path);
+    } else {
+      /* LCOV_EXCL_START Can't make getcwd fail. */
+      /* If getcwd failed, the best we can do here is keep using the
+       * relative path.  (Perhaps / isn't readable by this UID/GID.) */
+      log_warn(LD_GENERAL, "Unable to find current working directory: %s",
+               strerror(errno));
+      absfname = tor_strdup(fname);
+      /* LCOV_EXCL_STOP */
+    }
+  }
+  return absfname;
+#endif /* defined(_WIN32) */
+}

src/lib/fs/path.h

@@ -0,0 +1,24 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PATH_H
+#define TOR_PATH_H
+
+#include "lib/cc/compat_compiler.h"
+
+#ifdef _WIN32
+#define PATH_SEPARATOR "\\"
+#else
+#define PATH_SEPARATOR "/"
+#endif
+
+char *get_unquoted_path(const char *path);
+char *expand_filename(const char *filename);
+int path_is_relative(const char *filename);
+void clean_fname_for_stat(char *name);
+int get_parent_directory(char *fname);
+char *make_path_absolute(char *fname);
+
+#endif

src/lib/fs/userdb.c

@@ -0,0 +1,132 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "lib/fs/userdb.h"
+
+#ifndef _WIN32
+#include "lib/malloc/util_malloc.h"
+#include "lib/log/torlog.h"
+#include "lib/log/util_bug.h"
+
+#include <pwd.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Cached struct from the last getpwname() call we did successfully. */
+static struct passwd *passwd_cached = NULL;
+
+/** Helper: copy a struct passwd object.
+ *
+ * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir.  Tor doesn't use
+ * any others, and I don't want to run into incompatibilities.
+ */
+static struct passwd *
+tor_passwd_dup(const struct passwd *pw)
+{
+  struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd));
+  if (pw->pw_name)
+    new_pw->pw_name = tor_strdup(pw->pw_name);
+  if (pw->pw_dir)
+    new_pw->pw_dir = tor_strdup(pw->pw_dir);
+  new_pw->pw_uid = pw->pw_uid;
+  new_pw->pw_gid = pw->pw_gid;
+
+  return new_pw;
+}
+
+#define tor_passwd_free(pw) \
+  FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw))
+
+/** Helper: free one of our cached 'struct passwd' values. */
+static void
+tor_passwd_free_(struct passwd *pw)
+{
+  if (!pw)
+    return;
+
+  tor_free(pw->pw_name);
+  tor_free(pw->pw_dir);
+  tor_free(pw);
+}
+
+/** Wrapper around getpwnam() that caches result. Used so that we don't need
+ * to give the sandbox access to /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
+ *
+ * When called with a NULL argument, this function clears storage associated
+ * with static variables it uses.
+ **/
+const struct passwd *
+tor_getpwnam(const char *username)
+{
+  struct passwd *pw;
+
+  if (username == NULL) {
+    tor_passwd_free(passwd_cached);
+    passwd_cached = NULL;
+    return NULL;
+  }
+
+  if ((pw = getpwnam(username))) {
+    tor_passwd_free(passwd_cached);
+    passwd_cached = tor_passwd_dup(pw);
+    log_info(LD_GENERAL, "Caching new entry %s for %s",
+             passwd_cached->pw_name, username);
+    return pw;
+  }
+
+  /* Lookup failed */
+  if (! passwd_cached || ! passwd_cached->pw_name)
+    return NULL;
+
+  if (! strcmp(username, passwd_cached->pw_name))
+    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+  return NULL;
+}
+
+/** Wrapper around getpwnam() that can use cached result from
+ * tor_getpwnam(). Used so that we don't need to give the sandbox access to
+ * /etc/passwd.
+ *
+ * The following fields alone will definitely be copied in the output: pw_uid,
+ * pw_gid, pw_name, pw_dir.  Other fields are not present in cached values.
+ */
+const struct passwd *
+tor_getpwuid(uid_t uid)
+{
+  struct passwd *pw;
+
+  if ((pw = getpwuid(uid))) {
+    return pw;
+  }
+
+  /* Lookup failed */
+  if (! passwd_cached)
+    return NULL;
+
+  if (uid == passwd_cached->pw_uid)
+    return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky
+
+  return NULL;
+}
+
+/** Allocate and return a string containing the home directory for the
+ * user <b>username</b>. Only works on posix-like systems. */
+char *
+get_user_homedir(const char *username)
+{
+  const struct passwd *pw;
+  tor_assert(username);
+
+  if (!(pw = tor_getpwnam(username))) {
+    log_err(LD_CONFIG,"User \"%s\" not found.", username);
+    return NULL;
+  }
+  return tor_strdup(pw->pw_dir);
+}
+#endif /* !defined(_WIN32) */

src/lib/fs/userdb.h

@@ -0,0 +1,20 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_USERDB_H
+#define TOR_USERDB_H
+
+#include "orconfig.h"
+
+#ifndef _WIN32
+#include <sys/types.h>
+
+struct passwd;
+const struct passwd *tor_getpwnam(const char *username);
+const struct passwd *tor_getpwuid(uid_t uid);
+char *get_user_homedir(const char *username);
+#endif
+
+#endif

src/lib/log/include.am

@@ -11,6 +11,10 @@ src_lib_libtor_log_a_SOURCES =			\
 	src/lib/log/torlog.c			\
 	src/lib/log/util_bug.c
 
+if WIN32
+src_lib_libtor_log_a_SOURCES += src/lib/log/win32err.c
+endif
+
 src_lib_libtor_log_testing_a_SOURCES = \
 	$(src_lib_libtor_log_a_SOURCES)
 src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
@@ -23,4 +27,5 @@ noinst_HEADERS +=					\
 	src/lib/log/escape.h				\
 	src/lib/log/ratelim.h				\
 	src/lib/log/torlog.h				\
-	src/lib/log/util_bug.h
+	src/lib/log/util_bug.h				\
+	src/lib/log/win32err.h

src/lib/log/torlog.c

@@ -348,7 +348,7 @@ log_tor_version(logfile_t *lf, int reset)
     tor_snprintf(buf+n, sizeof(buf)-n,
                  "Tor %s opening %slog file.\n", VERSION, is_new?"new ":"");
   }
-  if (write_all_to_fd(lf->fd, buf, strlen(buf)) < 0) /* error */
+  if (write_all_to_fd_minimal(lf->fd, buf, strlen(buf)) < 0) /* error */
     return -1; /* failed */
   return 0;
 }
@@ -562,7 +562,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
       lf->callback(severity, domain, msg_after_prefix);
     }
   } else {
-    if (write_all_to_fd(lf->fd, buf, msg_len) < 0) { /* error */
+    if (write_all_to_fd_minimal(lf->fd, buf, msg_len) < 0) { /* error */
       /* don't log the error! mark this log entry to be blown away, and
        * continue. */
       lf->seems_dead = 1;

src/lib/log/win32err.c

@@ -0,0 +1,56 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifdef _WIN32
+#include "orconfig.h"
+#include "lib/log/win32err.h"
+#include "lib/malloc/util_malloc.h"
+
+#include <tchar.h>
+#include <windows.h>
+
+/** Return a newly allocated string describing the windows system error code
+ * <b>err</b>.  Note that error codes are different from errno.  Error codes
+ * come from GetLastError() when a winapi call fails.  errno is set only when
+ * ANSI functions fail.  Whee. */
+char *
+format_win32_error(DWORD err)
+{
+  TCHAR *str = NULL;
+  char *result;
+  DWORD n;
+
+  /* Somebody once decided that this interface was better than strerror(). */
+  n = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                 FORMAT_MESSAGE_FROM_SYSTEM |
+                 FORMAT_MESSAGE_IGNORE_INSERTS,
+                 NULL, err,
+                 MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
+                 (LPVOID)&str,
+                 0, NULL);
+
+  if (str && n) {
+#ifdef UNICODE
+    size_t len;
+    if (n > 128*1024)
+      len = (128 * 1024) * 2 + 1; /* This shouldn't be possible, but let's
+                                   * make sure. */
+    else
+      len = n * 2 + 1;
+    result = tor_malloc(len);
+    wcstombs(result,str,len);
+    result[len-1] = '\0';
+#else /* !(defined(UNICODE)) */
+    result = tor_strdup(str);
+#endif /* defined(UNICODE) */
+  } else {
+    result = tor_strdup("<unformattable error>");
+  }
+  if (str) {
+    LocalFree(str); /* LocalFree != free() */
+  }
+  return result;
+}
+#endif /* defined(_WIN32) */

src/lib/log/win32err.h

@@ -0,0 +1,17 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_WIN32ERR_H
+#define TOR_WIN32ERR_H
+
+#include "orconfig.h"
+
+/* Platform-specific helpers. */
+#ifdef _WIN32
+#include <windef.h>
+char *format_win32_error(DWORD err);
+#endif
+
+#endif

src/lib/net/.may_include

@@ -5,6 +5,7 @@ ht.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
+lib/err/*.h
 lib/lock/*.h
 lib/log/*.h
 lib/net/*.h

src/lib/net/socket.c

@@ -7,6 +7,7 @@
 #include "lib/net/socket.h"
 #include "lib/net/address.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/err/torerr.h"
 #include "lib/lock/compat_mutex.h"
 #include "lib/log/torlog.h"
 #include "lib/log/util_bug.h"
@@ -647,3 +648,46 @@ set_socket_nonblocking(tor_socket_t sock)
 
   return 0;
 }
+
+/** Read from <b>sock</b> to <b>buf</b>, until we get <b>count</b> bytes or
+ * reach the end of the file.  Return the number of bytes read, or -1 on
+ * error. Only use if fd is a blocking fd. */
+ssize_t
+read_all_from_socket(tor_socket_t sock, char *buf, size_t count)
+{
+  size_t numread = 0;
+  ssize_t result;
+
+  if (count > SIZE_T_CEILING || count > SSIZE_MAX) {
+    errno = EINVAL;
+    return -1;
+  }
+
+  while (numread < count) {
+    result = tor_socket_recv(sock, buf+numread, count-numread, 0);
+    if (result<0)
+      return -1;
+    else if (result == 0)
+      break;
+    numread += result;
+  }
+  return (ssize_t)numread;
+}
+
+/** Write <b>count</b> bytes from <b>buf</b> to <b>sock</b>. Return the number
+ * of bytes written, or -1 on error.  Only use if fd is a blocking fd.  */
+ssize_t
+write_all_to_socket(tor_socket_t fd, const char *buf, size_t count)
+{
+  size_t written = 0;
+  ssize_t result;
+  raw_assert(count < SSIZE_MAX);
+
+  while (written != count) {
+    result = tor_socket_send(fd, buf+written, count-written, 0);
+    if (result<0)
+      return -1;
+    written += result;
+  }
+  return (ssize_t)count;
+}

src/lib/net/socket.h

@@ -52,6 +52,9 @@ int network_init(void);
 int get_max_sockets(void);
 void set_max_sockets(int);
 
+ssize_t write_all_to_socket(tor_socket_t fd, const char *buf, size_t count);
+ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
+
 /* For stupid historical reasons, windows sockets have an independent
  * set of errnos, and an independent way to get them.  Also, you can't
  * always believe WSAEWOULDBLOCK.  Use the macros below to compare