Merge branch 'storage_labeled_squashed'

src/common/confline.c

@@ -0,0 +1,320 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "compat.h"
+#include "confline.h"
+#include "torlog.h"
+#include "util.h"
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * append it to *<b>lst</b>. */
+void
+config_line_append(config_line_t **lst,
+                   const char *key,
+                   const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = NULL;
+  while (*lst)
+    lst = &((*lst)->next);
+
+  (*lst) = newline;
+}
+
+/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or
+ * NULL if no such key exists.
+ *
+ * (In options parsing, this is for handling commandline-only options only;
+ * other options should be looked up in the appropriate data structure.) */
+const config_line_t *
+config_line_find(const config_line_t *lines,
+                 const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed.  Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If <b>extended</b> is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines(const char *string, config_line_t **result, int extended)
+{
+  config_line_t *list = NULL, **next;
+  char *k, *v;
+  const char *parse_err;
+
+  next = &list;
+  do {
+    k = v = NULL;
+    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
+    if (!string) {
+      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
+               parse_err?parse_err:"<unknown>");
+      config_free_lines(list);
+      tor_free(k);
+      tor_free(v);
+      return -1;
+    }
+    if (k && v) {
+      unsigned command = CONFIG_LINE_NORMAL;
+      if (extended) {
+        if (k[0] == '+') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          command = CONFIG_LINE_APPEND;
+        } else if (k[0] == '/') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          tor_free(v);
+          v = tor_strdup("");
+          command = CONFIG_LINE_CLEAR;
+        }
+      }
+      /* This list can get long, so we keep a pointer to the end of it
+       * rather than using config_line_append over and over and getting
+       * n^2 performance. */
+      *next = tor_malloc_zero(sizeof(config_line_t));
+      (*next)->key = k;
+      (*next)->value = v;
+      (*next)->next = NULL;
+      (*next)->command = command;
+      next = &((*next)->next);
+    } else {
+      tor_free(k);
+      tor_free(v);
+    }
+  } while (*string);
+
+  *result = list;
+  return 0;
+}
+
+/**
+ * Free all the configuration lines on the linked list <b>front</b>.
+ */
+void
+config_free_lines(config_line_t *front)
+{
+  config_line_t *tmp;
+
+  while (front) {
+    tmp = front;
+    front = tmp->next;
+
+    tor_free(tmp->key);
+    tor_free(tmp->value);
+    tor_free(tmp);
+  }
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
+config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+  return config_lines_dup_and_filter(inp, NULL);
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>,
+ * but only the ones that match <b>key</b>. */
+config_line_t *
+config_lines_dup_and_filter(const config_line_t *inp,
+                            const char *key)
+{
+  config_line_t *result = NULL;
+  config_line_t **next_out = &result;
+  while (inp) {
+    if (key && strcasecmpstart(inp->key, key)) {
+      inp = inp->next;
+      continue;
+    }
+    *next_out = tor_malloc_zero(sizeof(config_line_t));
+    (*next_out)->key = tor_strdup(inp->key);
+    (*next_out)->value = tor_strdup(inp->value);
+    inp = inp->next;
+    next_out = &((*next_out)->next);
+  }
+  (*next_out) = NULL;
+  return result;
+}
+
+/** Return true iff a and b contain identical keys and values in identical
+ * order. */
+int
+config_lines_eq(config_line_t *a, config_line_t *b)
+{
+  while (a && b) {
+    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
+      return 0;
+    a = a->next;
+    b = b->next;
+  }
+  if (a || b)
+    return 0;
+  return 1;
+}
+
+/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
+int
+config_count_key(const config_line_t *a, const char *key)
+{
+  int n = 0;
+  while (a) {
+    if (!strcasecmp(a->key, key)) {
+      ++n;
+    }
+    a = a->next;
+  }
+  return n;
+}
+
+/** Given a string containing part of a configuration file or similar format,
+ * advance past comments and whitespace and try to parse a single line.  If we
+ * parse a line successfully, set *<b>key_out</b> to a new string holding the
+ * key portion and *<b>value_out</b> to a new string holding the value portion
+ * of the line, and return a pointer to the start of the next line.  If we run
+ * out of data, return a pointer to the end of the string.  If we encounter an
+ * error, return NULL and set *<b>err_out</b> (if provided) to an error
+ * message.
+ */
+const char *
+parse_config_line_from_str_verbose(const char *line, char **key_out,
+                                   char **value_out,
+                                   const char **err_out)
+{
+  /*
+    See torrc_format.txt for a description of the (silly) format this parses.
+   */
+  const char *key, *val, *cp;
+  int continuation = 0;
+
+  tor_assert(key_out);
+  tor_assert(value_out);
+
+  *key_out = *value_out = NULL;
+  key = val = NULL;
+  /* Skip until the first keyword. */
+  while (1) {
+    while (TOR_ISSPACE(*line))
+      ++line;
+    if (*line == '#') {
+      while (*line && *line != '\n')
+        ++line;
+    } else {
+      break;
+    }
+  }
+
+  if (!*line) { /* End of string? */
+    *key_out = *value_out = NULL;
+    return line;
+  }
+
+  /* Skip until the next space or \ followed by newline. */
+  key = line;
+  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
+         ! (line[0] == '\\' && line[1] == '\n'))
+    ++line;
+  *key_out = tor_strndup(key, line-key);
+
+  /* Skip until the value. */
+  while (*line == ' ' || *line == '\t')
+    ++line;
+
+  val = line;
+
+  /* Find the end of the line. */
+  if (*line == '\"') { // XXX No continuation handling is done here
+    if (!(line = unescape_string(line, value_out, NULL))) {
+      if (err_out)
+        *err_out = "Invalid escape sequence in quoted string";
+      return NULL;
+    }
+    while (*line == ' ' || *line == '\t')
+      ++line;
+    if (*line == '\r' && *(++line) == '\n')
+      ++line;
+    if (*line && *line != '#' && *line != '\n') {
+      if (err_out)
+        *err_out = "Excess data after quoted string";
+      return NULL;
+    }
+  } else {
+    /* Look for the end of the line. */
+    while (*line && *line != '\n' && (*line != '#' || continuation)) {
+      if (*line == '\\' && line[1] == '\n') {
+        continuation = 1;
+        line += 2;
+      } else if (*line == '#') {
+        do {
+          ++line;
+        } while (*line && *line != '\n');
+        if (*line == '\n')
+          ++line;
+      } else {
+        ++line;
+      }
+    }
+
+    if (*line == '\n') {
+      cp = line++;
+    } else {
+      cp = line;
+    }
+    /* Now back cp up to be the last nonspace character */
+    while (cp>val && TOR_ISSPACE(*(cp-1)))
+      --cp;
+
+    tor_assert(cp >= val);
+
+    /* Now copy out and decode the value. */
+    *value_out = tor_strndup(val, cp-val);
+    if (continuation) {
+      char *v_out, *v_in;
+      v_out = v_in = *value_out;
+      while (*v_in) {
+        if (*v_in == '#') {
+          do {
+            ++v_in;
+          } while (*v_in && *v_in != '\n');
+          if (*v_in == '\n')
+            ++v_in;
+        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
+          v_in += 2;
+        } else {
+          *v_out++ = *v_in++;
+        }
+      }
+      *v_out = '\0';
+    }
+  }
+
+  if (*line == '#') {
+    do {
+      ++line;
+    } while (*line && *line != '\n');
+  }
+  while (TOR_ISSPACE(*line)) ++line;
+
+  return line;
+}
+

src/common/confline.h

@@ -0,0 +1,47 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFLINE_H
+#define TOR_CONFLINE_H
+
+/** Ordinary configuration line. */
+#define CONFIG_LINE_NORMAL 0
+/** Appends to previous configuration for the same option, even if we
+ * would ordinary replace it. */
+#define CONFIG_LINE_APPEND 1
+/* Removes all previous configuration for an option. */
+#define CONFIG_LINE_CLEAR 2
+
+/** A linked list of lines in a config file, or elsewhere */
+typedef struct config_line_t {
+  char *key;
+  char *value;
+  struct config_line_t *next;
+
+  /** What special treatment (if any) does this line require? */
+  unsigned int command:2;
+  /** If true, subsequent assignments to this linelist should replace
+   * it, not extend it.  Set only on the first item in a linelist in an
+   * or_options_t. */
+  unsigned int fragile:1;
+} config_line_t;
+
+void config_line_append(config_line_t **lst,
+                        const char *key, const char *val);
+config_line_t *config_lines_dup(const config_line_t *inp);
+config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
+                                           const char *key);
+const config_line_t *config_line_find(const config_line_t *lines,
+                                      const char *key);
+int config_lines_eq(config_line_t *a, config_line_t *b);
+int config_count_key(const config_line_t *a, const char *key);
+int config_get_lines(const char *string, config_line_t **result, int extended);
+void config_free_lines(config_line_t *front);
+const char *parse_config_line_from_str_verbose(const char *line,
+                                       char **key_out, char **value_out,
+                                       const char **err_out);
+#endif
+

src/common/include.am

@@ -84,6 +84,7 @@ LIBOR_A_SRC = \
   src/common/compat.c					\
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
+  src/common/confline.c					\
   src/common/container.c				\
   src/common/log.c					\
   src/common/memarea.c					\
@@ -144,6 +145,7 @@ COMMONHEADERS = \
   src/common/compat_openssl.h			\
   src/common/compat_threads.h			\
   src/common/compat_time.h			\
+  src/common/confline.h				\
   src/common/container.h			\
   src/common/crypto.h				\
   src/common/crypto_curve25519.h		\

src/common/storagedir.c

@@ -3,6 +3,8 @@
 
 #include "container.h"
 #include "compat.h"
+#include "confline.h"
+#include "memarea.h"
 #include "sandbox.h"
 #include "storagedir.h"
 #include "torlog.h"
@@ -237,28 +239,29 @@ find_unused_fname(storage_dir_t *d)
   return NULL;
 }
 
-/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
- * in <b>d</b>.  On success, return 0 and set *<b>fname_out</b> to a
- * newly allocated string containing the filename.  On failure, return
- * -1. */
-int
-storage_dir_save_bytes_to_file(storage_dir_t *d,
-                               const uint8_t *data,
-                               size_t length,
-                               int binary,
-                               char **fname_out)
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+                                const smartlist_t *chunks,
+                                int binary,
+                                char **fname_out)
 {
+  uint64_t total_length = 0;
   char *fname = find_unused_fname(d);
   if (!fname)
     return -1;
 
+  SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+                    total_length += ch->len);
+
   char *path = NULL;
   tor_asprintf(&path, "%s/%s", d->directory, fname);
 
-  int r = write_bytes_to_file(path, (const char *)data, length, binary);
+  int r = write_chunks_to_file(path, chunks, binary, 0);
   if (r == 0) {
     if (d->usage_known)
-      d->usage += length;
+      d->usage += total_length;
     if (fname_out) {
       *fname_out = tor_strdup(fname);
     }
@@ -270,6 +273,25 @@ storage_dir_save_bytes_to_file(storage_dir_t *d,
   return r;
 }
 
+/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
+ * in <b>d</b>.  On success, return 0 and set *<b>fname_out</b> to a
+ * newly allocated string containing the filename.  On failure, return
+ * -1. */
+int
+storage_dir_save_bytes_to_file(storage_dir_t *d,
+                               const uint8_t *data,
+                               size_t length,
+                               int binary,
+                               char **fname_out)
+{
+  smartlist_t *chunks = smartlist_new();
+  sized_chunk_t chunk = { (const char *)data, length };
+  smartlist_add(chunks, &chunk);
+  int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+  smartlist_free(chunks);
+  return r;
+}
+
 /**
  * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
  * <b>str</b>.
@@ -284,6 +306,106 @@ storage_dir_save_string_to_file(storage_dir_t *d,
                 (const uint8_t*)str, strlen(str), binary, fname_out);
 }
 
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in <b>labels</b>. Files
+ * stored in this format can be recovered with storage_dir_map_labeled
+ * or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                  const config_line_t *labels,
+                                  const uint8_t *data,
+                                  size_t length,
+                                  char **fname_out)
+{
+  /*
+   * The storage format is to prefix the data with the key-value pairs in
+   * <b>labels</b>, and a single NUL separator.  But code outside this module
+   * MUST NOT rely on that format.
+   */
+
+  smartlist_t *chunks = smartlist_new();
+  memarea_t *area = memarea_new();
+  const config_line_t *line;
+  for (line = labels; line; line = line->next) {
+    sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+    sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+    const size_t allocated = sz->len + 1;
+    char *bytes = memarea_alloc(area, allocated);
+    tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+    sz->bytes = bytes;
+    smartlist_add(chunks, sz);
+  }
+
+  sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+  nul->len = 1;
+  nul->bytes = "\0";
+  smartlist_add(chunks, nul);
+
+  sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+  datachunk->bytes = (const char *)data;
+  datachunk->len = length;
+  smartlist_add(chunks, datachunk);
+
+  int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+  smartlist_free(chunks);
+  memarea_drop_all(area);
+  return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled().  On failure,
+ * return NULL.  On success, write a set of newly allocated labels into to
+ * *<b>labels_out</b>, a pointer to the into *<b>data_out</b>, and the data's
+ * into *<b>sz_out</b>. On success, also return a tor_mmap_t object whose
+ * contents should not be used -- it needs to be kept around, though, for as
+ * long as <b>data_out</b> is going to be valid.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+                         const char *fname,
+                         config_line_t **labels_out,
+                         const uint8_t **data_out,
+                         size_t *sz_out)
+{
+  tor_mmap_t *m = storage_dir_map(dir, fname);
+  if (! m)
+    goto err;
+  const char *nulp = memchr(m->data, '\0', m->size);
+  if (! nulp)
+    goto err;
+  if (labels_out && config_get_lines(m->data, labels_out, 0) < 0)
+    goto err;
+  size_t offset = nulp - m->data + 1;
+  tor_assert(offset <= m->size);
+  *data_out = (const uint8_t *)(m->data + offset);
+  *sz_out = m->size - offset;
+
+  return m;
+ err:
+  tor_munmap_file(m);
+  return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+                          const char *fname,
+                          config_line_t **labels_out,
+                          size_t *sz_out)
+{
+  const uint8_t *data = NULL;
+  tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+                                           &data, sz_out);
+  if (m == NULL)
+    return NULL;
+  uint8_t *result = tor_memdup(data, *sz_out);
+  tor_munmap_file(m);
+  return result;
+}
+
 /**
  * Remove the file called <b>fname</b> from <b>d</b>.
  */

src/common/storagedir.h

@@ -5,7 +5,7 @@
 #define TOR_STORAGEDIR_H
 
 typedef struct storage_dir_t storage_dir_t;
-
+struct config_line_t;
 struct sandbox_cfg_elem;
 
 storage_dir_t * storage_dir_new(const char *dirname, int n_files);
@@ -26,6 +26,19 @@ int storage_dir_save_string_to_file(storage_dir_t *d,
                                     const char *data,
                                     int binary,
                                     char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                      const struct config_line_t *labels,
+                                      const uint8_t *data,
+                                      size_t length,
+                                      char **fname_out);
+tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+                                     const char *fname,
+                                     struct config_line_t **labels_out,
+                                     const uint8_t **data_out,
+                                     size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+                                   struct config_line_t **labels_out,
+                                   size_t *sz_out);
 void storage_dir_remove_file(storage_dir_t *d,
                              const char *fname);
 int storage_dir_shrink(storage_dir_t *d,

src/common/util.c

@@ -3045,137 +3045,6 @@ unescape_string(const char *s, char **result, size_t *size_out)
   }
 }
 
-/** Given a string containing part of a configuration file or similar format,
- * advance past comments and whitespace and try to parse a single line.  If we
- * parse a line successfully, set *<b>key_out</b> to a new string holding the
- * key portion and *<b>value_out</b> to a new string holding the value portion
- * of the line, and return a pointer to the start of the next line.  If we run
- * out of data, return a pointer to the end of the string.  If we encounter an
- * error, return NULL and set *<b>err_out</b> (if provided) to an error
- * message.
- */
-const char *
-parse_config_line_from_str_verbose(const char *line, char **key_out,
-                                   char **value_out,
-                                   const char **err_out)
-{
-  /*
-    See torrc_format.txt for a description of the (silly) format this parses.
-   */
-  const char *key, *val, *cp;
-  int continuation = 0;
-
-  tor_assert(key_out);
-  tor_assert(value_out);
-
-  *key_out = *value_out = NULL;
-  key = val = NULL;
-  /* Skip until the first keyword. */
-  while (1) {
-    while (TOR_ISSPACE(*line))
-      ++line;
-    if (*line == '#') {
-      while (*line && *line != '\n')
-        ++line;
-    } else {
-      break;
-    }
-  }
-
-  if (!*line) { /* End of string? */
-    *key_out = *value_out = NULL;
-    return line;
-  }
-
-  /* Skip until the next space or \ followed by newline. */
-  key = line;
-  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
-         ! (line[0] == '\\' && line[1] == '\n'))
-    ++line;
-  *key_out = tor_strndup(key, line-key);
-
-  /* Skip until the value. */
-  while (*line == ' ' || *line == '\t')
-    ++line;
-
-  val = line;
-
-  /* Find the end of the line. */
-  if (*line == '\"') { // XXX No continuation handling is done here
-    if (!(line = unescape_string(line, value_out, NULL))) {
-      if (err_out)
-        *err_out = "Invalid escape sequence in quoted string";
-      return NULL;
-    }
-    while (*line == ' ' || *line == '\t')
-      ++line;
-    if (*line == '\r' && *(++line) == '\n')
-      ++line;
-    if (*line && *line != '#' && *line != '\n') {
-      if (err_out)
-        *err_out = "Excess data after quoted string";
-      return NULL;
-    }
-  } else {
-    /* Look for the end of the line. */
-    while (*line && *line != '\n' && (*line != '#' || continuation)) {
-      if (*line == '\\' && line[1] == '\n') {
-        continuation = 1;
-        line += 2;
-      } else if (*line == '#') {
-        do {
-          ++line;
-        } while (*line && *line != '\n');
-        if (*line == '\n')
-          ++line;
-      } else {
-        ++line;
-      }
-    }
-
-    if (*line == '\n') {
-      cp = line++;
-    } else {
-      cp = line;
-    }
-    /* Now back cp up to be the last nonspace character */
-    while (cp>val && TOR_ISSPACE(*(cp-1)))
-      --cp;
-
-    tor_assert(cp >= val);
-
-    /* Now copy out and decode the value. */
-    *value_out = tor_strndup(val, cp-val);
-    if (continuation) {
-      char *v_out, *v_in;
-      v_out = v_in = *value_out;
-      while (*v_in) {
-        if (*v_in == '#') {
-          do {
-            ++v_in;
-          } while (*v_in && *v_in != '\n');
-          if (*v_in == '\n')
-            ++v_in;
-        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
-          v_in += 2;
-        } else {
-          *v_out++ = *v_in++;
-        }
-      }
-      *v_out = '\0';
-    }
-  }
-
-  if (*line == '#') {
-    do {
-      ++line;
-    } while (*line && *line != '\n');
-  }
-  while (TOR_ISSPACE(*line)) ++line;
-
-  return line;
-}
-
 /** Expand any homedir prefix on <b>filename</b>; return a newly allocated
  * string. */
 char *

src/common/util.h

@@ -389,9 +389,6 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
                                  size_t *sz_out)
   ATTR_MALLOC;
 const char *unescape_string(const char *s, char **result, size_t *size_out);
-const char *parse_config_line_from_str_verbose(const char *line,
-                                       char **key_out, char **value_out,
-                                       const char **err_out);
 char *expand_filename(const char *filename);
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 int path_is_relative(const char *filename);

src/or/confparse.c

@@ -31,8 +31,6 @@ static int config_parse_msec_interval(const char *s, int *ok);
 static int config_parse_interval(const char *s, int *ok);
 static void config_reset(const config_format_t *fmt, void *options,
                          const config_var_t *var, int use_defaults);
-static config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
-                                                  const char *key);
 
 /** Allocate an empty configuration object of a given format type. */
 void *
@@ -80,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
   return option;
 }
 
-/** Helper: allocate a new configuration option mapping 'key' to 'val',
- * append it to *<b>lst</b>. */
-void
-config_line_append(config_line_t **lst,
-                   const char *key,
-                   const char *val)
-{
-  config_line_t *newline;
-
-  newline = tor_malloc_zero(sizeof(config_line_t));
-  newline->key = tor_strdup(key);
-  newline->value = tor_strdup(val);
-  newline->next = NULL;
-  while (*lst)
-    lst = &((*lst)->next);
-
-  (*lst) = newline;
-}
-
-/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL
- * if no such key exists. For handling commandline-only options only; other
- * options should be looked up in the appropriate data structure. */
-const config_line_t *
-config_line_find(const config_line_t *lines,
-                 const char *key)
-{
-  const config_line_t *cl;
-  for (cl = lines; cl; cl = cl->next) {
-    if (!strcmp(cl->key, key))
-      return cl;
-  }
-  return NULL;
-}
-
-/** Helper: parse the config string and strdup into key/value
- * strings. Set *result to the list, or NULL if parsing the string
- * failed.  Return 0 on success, -1 on failure. Warn and ignore any
- * misformatted lines.
- *
- * If <b>extended</b> is set, then treat keys beginning with / and with + as
- * indicating "clear" and "append" respectively. */
-int
-config_get_lines(const char *string, config_line_t **result, int extended)
-{
-  config_line_t *list = NULL, **next;
-  char *k, *v;
-  const char *parse_err;
-
-  next = &list;
-  do {
-    k = v = NULL;
-    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
-    if (!string) {
-      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
-               parse_err?parse_err:"<unknown>");
-      config_free_lines(list);
-      tor_free(k);
-      tor_free(v);
-      return -1;
-    }
-    if (k && v) {
-      unsigned command = CONFIG_LINE_NORMAL;
-      if (extended) {
-        if (k[0] == '+') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          command = CONFIG_LINE_APPEND;
-        } else if (k[0] == '/') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          tor_free(v);
-          v = tor_strdup("");
-          command = CONFIG_LINE_CLEAR;
-        }
-      }
-      /* This list can get long, so we keep a pointer to the end of it
-       * rather than using config_line_append over and over and getting
-       * n^2 performance. */
-      *next = tor_malloc_zero(sizeof(config_line_t));
-      (*next)->key = k;
-      (*next)->value = v;
-      (*next)->next = NULL;
-      (*next)->command = command;
-      next = &((*next)->next);
-    } else {
-      tor_free(k);
-      tor_free(v);
-    }
-  } while (*string);
-
-  *result = list;
-  return 0;
-}
-
-/**
- * Free all the configuration lines on the linked list <b>front</b>.
- */
-void
-config_free_lines(config_line_t *front)
-{
-  config_line_t *tmp;
-
-  while (front) {
-    tmp = front;
-    front = tmp->next;
-
-    tor_free(tmp->key);
-    tor_free(tmp->value);
-    tor_free(tmp);
-  }
-}
-
 /** If <b>key</b> is a deprecated configuration option, return the message
  * explaining why it is deprecated (which may be an empty string). Return NULL
  * if it is not deprecated. The <b>key</b> field must be fully expanded. */
@@ -633,36 +517,6 @@ config_value_needs_escape(const char *value)
   return 0;
 }
 
-/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
-config_line_t *
-config_lines_dup(const config_line_t *inp)
-{
-  return config_lines_dup_and_filter(inp, NULL);
-}
-
-/** Return a newly allocated deep copy of the lines in <b>inp</b>,
- * but only the ones that match <b>key</b>. */
-static config_line_t *
-config_lines_dup_and_filter(const config_line_t *inp,
-                            const char *key)
-{
-  config_line_t *result = NULL;
-  config_line_t **next_out = &result;
-  while (inp) {
-    if (key && strcasecmpstart(inp->key, key)) {
-      inp = inp->next;
-      continue;
-    }
-    *next_out = tor_malloc_zero(sizeof(config_line_t));
-    (*next_out)->key = tor_strdup(inp->key);
-    (*next_out)->value = tor_strdup(inp->value);
-    inp = inp->next;
-    next_out = &((*next_out)->next);
-  }
-  (*next_out) = NULL;
-  return result;
-}
-
 /** Return newly allocated line or lines corresponding to <b>key</b> in the
  * configuration <b>options</b>.  If <b>escape_val</b> is true and a
  * value needs to be quoted before it's put in a config file, quote and
@@ -1028,36 +882,6 @@ config_free(const config_format_t *fmt, void *options)
   tor_free(options);
 }
 
-/** Return true iff a and b contain identical keys and values in identical
- * order. */
-int
-config_lines_eq(config_line_t *a, config_line_t *b)
-{
-  while (a && b) {
-    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
-      return 0;
-    a = a->next;
-    b = b->next;
-  }
-  if (a || b)
-    return 0;
-  return 1;
-}
-
-/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
-int
-config_count_key(const config_line_t *a, const char *key)
-{
-  int n = 0;
-  while (a) {
-    if (!strcasecmp(a->key, key)) {
-      ++n;
-    }
-    a = a->next;
-  }
-  return n;
-}
-
 /** Return true iff the option <b>name</b> has the same value in <b>o1</b>
  * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
  */

src/or/confparse.h

@@ -103,14 +103,7 @@ typedef struct config_format_t {
 #define CAL_WARN_DEPRECATIONS (1u<<2)
 
 void *config_new(const config_format_t *fmt);
-void config_line_append(config_line_t **lst,
-                        const char *key, const char *val);
-config_line_t *config_lines_dup(const config_line_t *inp);
-const config_line_t *config_line_find(const config_line_t *lines,
-                                      const char *key);
 void config_free(const config_format_t *fmt, void *options);
-int config_lines_eq(config_line_t *a, config_line_t *b);
-int config_count_key(const config_line_t *a, const char *key);
 config_line_t *config_get_assigned_option(const config_format_t *fmt,
                                           const void *options, const char *key,
                                           int escape_val);
@@ -132,8 +125,6 @@ const char *config_find_deprecation(const config_format_t *fmt,
 const config_var_t *config_find_option(const config_format_t *fmt,
                                        const char *key);
 
-int config_get_lines(const char *string, config_line_t **result, int extended);
-void config_free_lines(config_line_t *front);
 const char *config_expand_abbrev(const config_format_t *fmt,
                                  const char *option,
                                  int command_line, int warn_obsolete);

src/or/conscache.c

@@ -0,0 +1,506 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+#include "config.h"
+#include "conscache.h"
+#include "storagedir.h"
+
+#define CCE_MAGIC 0x17162253
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t.  It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+  uint32_t magic; /**< Must be set to CCE_MAGIC */
+  int32_t refcnt; /**< Reference count. */
+  unsigned can_remove : 1; /**< If true, we want to delete this file. */
+  /** If true, we intend to unmap this file as soon as we're done with it. */
+  unsigned release_aggressively : 1;
+
+  /** Filename for this object within the storage_dir_t */
+  char *fname;
+  /** Labels associated with this object. Immutable once the object
+   * is created. */
+  config_line_t *labels;
+  /** Pointer to the cache that includes this entry (if any). */
+  consensus_cache_t *in_cache;
+
+  /** Since what time has this object been mapped into RAM, but with the cache
+   * being the only having a reference to it? */
+  time_t unused_since;
+  /** mmaped contents of the underlying file.  May be NULL */
+  tor_mmap_t *map;
+  /** Length of the body within <b>map</b>. */
+  size_t bodylen;
+  /** Pointer to the body within <b>map</b>. */
+  const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+  /** Underling storage_dir_t to handle persistence */
+  storage_dir_t *dir;
+  /** List of all the entries in the directory. */
+  smartlist_t *entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+                                      consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
+ * data directory, to hold up to <b>max_entries</b> of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+  char *directory = get_datadir_fname(subdir);
+  cache->dir = storage_dir_new(directory, max_entries);
+  tor_free(directory);
+  if (!cache->dir) {
+    tor_free(cache);
+    return NULL;
+  }
+
+  consensus_cache_rescan(cache);
+  return cache;
+}
+
+/**
+ * Helper: clear all entries from <b>cache</b> (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+  consensus_cache_delete_pending(cache);
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    ent->in_cache = NULL;
+    consensus_cache_entry_decref(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(cache->entries);
+  cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by <b>cache</b>.
+ */
+void
+consensus_cache_free(consensus_cache_t *cache)
+{
+  if (! cache)
+    return;
+
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+  storage_dir_free(cache->dir);
+  tor_free(cache);
+}
+
+/**
+ * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
+ * labeling that data with <b>labels</b>.  On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it.  Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided <b>labels</b> MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+                    const config_line_t *labels,
+                    const uint8_t *data,
+                    size_t datalen)
+{
+  char *fname = NULL;
+  int r = storage_dir_save_labeled_to_file(cache->dir,
+                                            labels, data, datalen, &fname);
+  if (r < 0 || fname == NULL) {
+    return NULL;
+  }
+  consensus_cache_entry_t *ent =
+    tor_malloc_zero(sizeof(consensus_cache_entry_t));
+  ent->magic = CCE_MAGIC;
+  ent->fname = fname;
+  ent->labels = config_lines_dup(labels);
+  ent->in_cache = cache;
+  ent->unused_since = TIME_MAX;
+  smartlist_add(cache->entries, ent);
+  /* Start the reference count at 2: the caller owns one copy, and the
+   * cache owns another.
+   */
+  ent->refcnt = 2;
+
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+                           const char *key,
+                           const char *value)
+{
+  smartlist_t *tmp = smartlist_new();
+  consensus_cache_find_all(tmp, cache, key, value);
+  consensus_cache_entry_t *ent = NULL;
+  if (smartlist_len(tmp))
+    ent = smartlist_get(tmp, 0);
+  smartlist_free(tmp);
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+                         consensus_cache_t *cache,
+                         const char *key,
+                         const char *value)
+{
+  if (! key) {
+    smartlist_add_all(out, cache->entries);
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (found_val && !strcmp(value, found_val)) {
+      smartlist_add(out, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have <b>key</b>=<b>value</b> in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+                            const char *key,
+                            const char *value)
+{
+  if (BUG(lst == NULL))
+    return; // LCOV_EXCL_LINE
+  if (key == NULL)
+    return;
+  SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (! found_val || strcmp(value, found_val)) {
+      SMARTLIST_DEL_CURRENT(lst, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If <b>ent</b> has a label with the given <b>key</b>, return its
+ * value.  Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                const char *key)
+{
+  const config_line_t *match = config_line_find(ent->labels, key);
+  if (match)
+    return match->value;
+  else
+    return NULL;
+}
+
+/**
+ * Return a pointer to the labels in <b>ent</b>.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+  return ent->labels;
+}
+
+/**
+ * Increase the reference count of <b>ent</b>.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+  ++ent->refcnt;
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to <b>ent</b>.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use <b>ent</b> after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+  if (! ent)
+    return;
+  if (BUG(ent->refcnt <= 0))
+    return; // LCOV_EXCL_LINE
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+
+  --ent->refcnt;
+
+  if (ent->refcnt == 1 && ent->in_cache) {
+    /* Only the cache has a reference: we don't need to keep the file
+     * mapped */
+    if (ent->map) {
+      if (ent->release_aggressively) {
+        consensus_cache_entry_unmap(ent);
+      } else {
+        ent->unused_since = approx_time();
+      }
+    }
+    return;
+  }
+
+  if (ent->refcnt > 0)
+    return;
+
+  /* Refcount is zero; we can free it. */
+  if (ent->map) {
+    consensus_cache_entry_unmap(ent);
+  }
+  tor_free(ent->fname);
+  config_free_lines(ent->labels);
+  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+  tor_free(ent);
+}
+
+/**
+ * Mark <b>ent</b> for deletion from the cache.  Deletion will not occur
+ * until the cache is the only place that holds a reference to <b>ent</b>.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+  ent->can_remove = 1;
+}
+
+/**
+ * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+  ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of <b>ent</b> into memory if it isn't already
+ * loaded.  On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
+ * to its size, and return 0.  On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                               const uint8_t **body_out,
+                               size_t *sz_out)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return -1; // LCOV_EXCL_LINE
+
+  if (! ent->map) {
+    if (! ent->in_cache)
+      return -1;
+
+    consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+                              (consensus_cache_entry_t *)ent);
+    if (! ent->map) {
+      return -1;
+    }
+  }
+
+  *body_out = ent->body;
+  *sz_out = ent->bodylen;
+  return 0;
+}
+
+/**
+ * Unmap every mmap'd element of <b>cache</b> that has been unused
+ * since <b>cutoff</b>.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->unused_since > cutoff) {
+      /* Has been unused only for a little while */
+      continue;
+    }
+    if (ent->map == NULL) {
+      /* Not actually mapped. */
+      continue;
+    }
+    consensus_cache_entry_unmap(ent);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Delete every element of <b>cache</b> has been marked with
+ * consensus_cache_entry_mark_for_removal, and which is not in use except by
+ * the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->can_remove == 0) {
+      /* Don't want to delete this. */
+      continue;
+    }
+    if (BUG(ent->refcnt <= 0)) {
+      continue; // LCOV_EXCL_LINE
+    }
+
+    SMARTLIST_DEL_CURRENT(cache->entries, ent);
+    ent->in_cache = NULL;
+    char *fname = tor_strdup(ent->fname); /* save a copy */
+    consensus_cache_entry_decref(ent);
+    storage_dir_remove_file(cache->dir, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+
+  cache->entries = smartlist_new();
+  const smartlist_t *fnames = storage_dir_list(cache->dir);
+  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+    tor_mmap_t *map = NULL;
+    config_line_t *labels = NULL;
+    const uint8_t *body;
+    size_t bodylen;
+    map = storage_dir_map_labeled(cache->dir, fname,
+                                  &labels, &body, &bodylen);
+    if (! map) {
+      /* Can't load this; continue */
+      log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+               escaped(fname), strerror(errno));
+      continue;
+    }
+    consensus_cache_entry_t *ent =
+      tor_malloc_zero(sizeof(consensus_cache_entry_t));
+    ent->magic = CCE_MAGIC;
+    ent->fname = tor_strdup(fname);
+    ent->labels = labels;
+    ent->refcnt = 1;
+    ent->in_cache = cache;
+    ent->unused_since = TIME_MAX;
+    smartlist_add(cache->entries, ent);
+    tor_munmap_file(map); /* don't actually need to keep this around */
+  } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that <b>ent</b> is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+                          consensus_cache_entry_t *ent)
+{
+  if (ent->map)
+    return;
+
+  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+                                     NULL, &ent->body, &ent->bodylen);
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap <b>ent</b> from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to <b>ent</b>
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+  ent->unused_since = TIME_MAX;
+  if (!ent->map)
+    return;
+
+  tor_munmap_file(ent->map);
+  ent->map = NULL;
+  ent->body = NULL;
+  ent->bodylen = 0;
+  ent->unused_since = TIME_MAX;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff <b>ent</b> is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+  if (ent->map) {
+    tor_assert(ent->body);
+    return 1;
+  } else {
+    tor_assert(!ent->body);
+    return 0;
+  }
+}
+#endif
+

src/or/conscache.h

@@ -0,0 +1,52 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free(consensus_cache_t *cache);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+                                             const config_line_t *labels,
+                                             const uint8_t *data,
+                                             size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+                                             consensus_cache_t *cache,
+                                             const char *key,
+                                             const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+                              consensus_cache_t *cache,
+                              const char *key,
+                              const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+                                 const char *key,
+                                 const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                            const char *key);
+const config_line_t *consensus_cache_entry_get_labels(
+                                          const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+                                            consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                                   const uint8_t **body_out,
+                                   size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif
+

src/or/include.am

@@ -36,6 +36,7 @@ LIBTOR_A_SOURCES = \
 	src/or/connection.c				\
 	src/or/connection_edge.c			\
 	src/or/connection_or.c				\
+	src/or/conscache.c				\
 	src/or/consdiff.c				\
 	src/or/control.c				\
 	src/or/cpuworker.c				\
@@ -152,6 +153,7 @@ ORHEADERS = \
 	src/or/connection.h				\
 	src/or/connection_edge.h			\
 	src/or/connection_or.h				\
+	src/or/conscache.h				\
 	src/or/consdiff.h				\
 	src/or/control.h				\
 	src/or/cpuworker.h				\

src/or/or.h

@@ -75,6 +75,7 @@
 #include "address.h"
 #include "compat_libevent.h"
 #include "ht.h"
+#include "confline.h"
 #include "replaycache.h"
 #include "crypto_curve25519.h"
 #include "crypto_ed25519.h"
@@ -3529,27 +3530,6 @@ typedef struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 } port_cfg_t;
 
-/** Ordinary configuration line. */
-#define CONFIG_LINE_NORMAL 0
-/** Appends to previous configuration for the same option, even if we
- * would ordinary replace it. */
-#define CONFIG_LINE_APPEND 1
-/* Removes all previous configuration for an option. */
-#define CONFIG_LINE_CLEAR 2
-
-/** A linked list of lines in a config file. */
-typedef struct config_line_t {
-  char *key;
-  char *value;
-  struct config_line_t *next;
-  /** What special treatment (if any) does this line require? */
-  unsigned int command:2;
-  /** If true, subsequent assignments to this linelist should replace
-   * it, not extend it.  Set only on the first item in a linelist in an
-   * or_options_t. */
-  unsigned int fragile:1;
-} config_line_t;
-
 typedef struct routerset_t routerset_t;
 
 /** A magic value for the (Socks|OR|...)Port options below, telling Tor

src/test/include.am

@@ -87,6 +87,7 @@ src_test_test_SOURCES = \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
 	src/test/test_connection.c \
+	src/test/test_conscache.c \
 	src/test/test_consdiff.c \
 	src/test/test_containers.c \
 	src/test/test_controller.c \

src/test/test.c

@@ -1195,6 +1195,7 @@ struct testgroup_t testgroups[] = {
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
   { "connection/", connection_tests },
+  { "conscache/", conscache_tests },
   { "consdiff/", consdiff_tests },
   { "container/", container_tests },
   { "control/", controller_tests },

src/test/test.h

@@ -190,6 +190,7 @@ extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
 extern struct testcase_t connection_tests[];
+extern struct testcase_t conscache_tests[];
 extern struct testcase_t consdiff_tests[];
 extern struct testcase_t container_tests[];
 extern struct testcase_t controller_tests[];

src/test/test_conscache.c

@@ -0,0 +1,341 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_conscache_open_failure(void *arg)
+{
+  (void) arg;
+  /* Try opening a directory that doesn't exist and which we shouldn't be
+   * able to create. */
+  consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
+  tt_ptr_op(cache, OP_EQ, NULL);
+
+ done:
+  ;
+}
+
+static void
+test_conscache_simple_usage(void *arg)
+{
+  (void)arg;
+  consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create object; make sure it exists. */
+  config_line_t *labels = NULL;
+  config_line_append(&labels, "Hello", "world");
+  config_line_append(&labels, "Adios", "planetas");
+  ent = consensus_cache_add(cache,
+                            labels, (const uint8_t *)"A\0B\0C", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent);
+
+  /* Make a second object */
+  config_line_append(&labels, "Hello", "mundo");
+  config_line_append(&labels, "Adios", "planets");
+  ent2 = consensus_cache_add(cache,
+                             labels, (const uint8_t *)"xyzzy", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent2);
+  tt_assert(! consensus_cache_entry_is_mapped(ent2));
+  consensus_cache_entry_decref(ent2);
+  ent2 = NULL;
+
+  /* Check get_value */
+  tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
+  tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
+
+  /* Check find_first */
+  ent2 = consensus_cache_find_first(cache, "Hello", "world!");
+  tt_ptr_op(ent2, OP_EQ, NULL);
+  ent2 = consensus_cache_find_first(cache, "Hello", "world");
+  tt_ptr_op(ent2, OP_EQ, ent);
+  ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_ptr_op(ent2, OP_NE, ent);
+
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+
+  /* Check get_body */
+  const uint8_t *bp = NULL;
+  size_t sz = 0;
+  int r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* Free and re-create the cache, to rescan the directory. */
+  consensus_cache_free(cache);
+  consensus_cache_entry_decref(ent);
+  cache = consensus_cache_open("cons", 128);
+
+  /* Make sure the entry is still there */
+  ent = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_assert(ent);
+  ent2 = consensus_cache_find_first(cache, "Adios", "planets");
+  tt_ptr_op(ent, OP_EQ, ent2);
+  consensus_cache_entry_incref(ent);
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+  r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "xyzzy", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* There should be two entries total. */
+  smartlist_t *entries = smartlist_new();
+  consensus_cache_find_all(entries, cache, NULL, NULL);
+  int n = smartlist_len(entries);
+  smartlist_free(entries);
+  tt_int_op(n, OP_EQ, 2);
+
+ done:
+  consensus_cache_entry_decref(ent);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_cleanup(void *arg)
+{
+  (void)arg;
+  const int N = 20;
+  consensus_cache_entry_t **ents =
+    tor_calloc(N, sizeof(consensus_cache_entry_t*));
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries. */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "cleanup");
+    config_line_append(&labels, "index", num);
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    ents[i] = consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ents[i]);
+    /* We're still holding a reference to each entry at this point. */
+  }
+
+  /* Page all of the entries into RAM */
+  for (i = 0; i < N; ++i) {
+    const uint8_t *bp;
+    size_t sz;
+    tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
+    consensus_cache_entry_get_body(ents[i], &bp, &sz);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as deletable. */
+  for (i = 7; i < N; i += 7) {
+    consensus_cache_entry_mark_for_removal(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as aggressively unpaged. */
+  for (i = 3; i < N; i += 3) {
+    consensus_cache_entry_mark_for_aggressive_release(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Incref some of the entries again */
+  for (i = 0; i < N; i += 2) {
+    consensus_cache_entry_incref(ents[i]);
+  }
+
+  /* Now we're going to decref everything. We do so at a specific time.  I'm
+   * picking the moment when I was writing this test, at 2017-04-05 12:16:48
+   * UTC. */
+  const time_t example_time = 1491394608;
+  update_approx_time(example_time);
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    if (i % 2) {
+      ents[i] = NULL; /* We're no longer holding any reference here. */
+    }
+  }
+
+  /* At this point, the aggressively-released items with refcount 1 should
+   * be unmapped. Nothing should be deleted. */
+  consensus_cache_entry_t *e_tmp;
+  e_tmp = consensus_cache_find_first(cache, "index", "3");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "5");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "6");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "7");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+  /* Delete the pending-deletion items. */
+  consensus_cache_delete_pending(cache);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 20 - 1); /* 1 entry was deleted */
+  }
+  e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
+  tt_assert(e_tmp == NULL); // so deleted.
+  e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
+  tt_assert(e_tmp); // so, not deleted.
+
+  /* Now do lazy unmapping. */
+  // should do nothing.
+  consensus_cache_unmap_lazy(cache, example_time - 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  // should actually unmap
+  consensus_cache_unmap_lazy(cache, example_time + 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  // This one will still be mapped, since it has a reference.
+  e_tmp = consensus_cache_find_first(cache, "index", "16");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    ents[i] = NULL;
+  }
+
+  /* Free and re-create the cache, to rescan the directory. Make sure the
+   * deleted thing is still deleted, along with the other deleted thing. */
+  consensus_cache_free(cache);
+  cache = consensus_cache_open("cons", 128);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 18);
+  }
+
+ done:
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+  }
+  tor_free(ents);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_filter(void *arg)
+{
+  (void)arg;
+  const int N = 30;
+  smartlist_t *lst = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries with different labels */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "filter");
+    config_line_append(&labels, "index", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 3);
+    config_line_append(&labels, "mod3", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 5);
+    config_line_append(&labels, "mod5", num);
+
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    consensus_cache_entry_t *ent =
+      consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ent);
+    consensus_cache_entry_decref(ent);
+  }
+
+  lst = smartlist_new();
+  /* Find nothing. */
+  consensus_cache_find_all(lst, cache, "mod5", "5");
+  tt_int_op(smartlist_len(lst), OP_EQ, 0);
+  /* Find everything. */
+  consensus_cache_find_all(lst, cache, "test-id", "filter");
+  tt_int_op(smartlist_len(lst), OP_EQ, N);
+
+  /* Now filter to find the entries that have i%3 == 1 */
+  consensus_cache_filter_list(lst, "mod3", "1");
+  tt_int_op(smartlist_len(lst), OP_EQ, 10);
+  /* Now filter to find the entries that also have i%5 == 3 */
+  consensus_cache_filter_list(lst, "mod5", "3");
+  tt_int_op(smartlist_len(lst), OP_EQ, 2);
+  /* So now we have those entries for which i%15 == 13. */
+
+  consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
+  consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
+  const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
+  const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
+  tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
+             (!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
+
+ done:
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+  smartlist_free(lst);
+}
+
+#define ENT(name)                                               \
+  { #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t conscache_tests[] = {
+  ENT(open_failure),
+  ENT(simple_usage),
+  ENT(cleanup),
+  ENT(filter),
+  END_OF_TESTCASES
+};
+

src/test/test_storagedir.c

@@ -256,6 +256,109 @@ test_storagedir_cleaning(void *arg)
   }
 }
 
+static void
+test_storagedir_save_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL;
+  char *fname = NULL;
+  uint8_t *saved = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  crypto_rand((char *)inp, 8192);
+
+  config_line_append(&labels, "Foo", "bar baz");
+  config_line_append(&labels, "quux", "quuzXxz");
+  const char expected[] =
+    "Foo bar baz\n"
+    "quux quuzXxz\n";
+
+  int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  size_t n;
+  saved = storage_dir_read(d, fname, 1, &n);
+  tt_assert(memchr(saved, '\0', n));
+  tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
+  tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  tor_free(saved);
+}
+
+static void
+test_storagedir_read_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL, *labels2 = NULL;
+  char *fname = NULL;
+  tor_mmap_t *map = NULL;
+  uint8_t *as_read = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tor_snprintf((char*)inp, 8192,
+               "Hello world\n"
+               "This is a test\n"
+               "Yadda yadda.\n");
+  size_t bodylen = 8192 - strlen((char*)inp) - 1;
+  crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
+
+  int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* Try mapping */
+  const uint8_t *datap = NULL;
+  size_t sz = 0;
+  map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
+  tt_assert(map);
+  tt_assert(datap);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(labels);
+  tt_str_op(labels->key, OP_EQ, "Hello");
+  tt_str_op(labels->value, OP_EQ, "world");
+  tt_assert(labels->next);
+  tt_str_op(labels->next->key, OP_EQ, "This");
+  tt_str_op(labels->next->value, OP_EQ, "is a test");
+  tt_assert(labels->next->next);
+  tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
+  tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
+  tt_assert(labels->next->next->next == NULL);
+
+  /* Try reading this time. */
+  sz = 0;
+  as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
+  tt_assert(as_read);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(config_lines_eq(labels, labels2));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  config_free_lines(labels2);
+  tor_munmap_file(map);
+  tor_free(as_read);
+}
+
 #define ENT(name)                                               \
   { #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
 
@@ -265,6 +368,8 @@ struct testcase_t storagedir_tests[] = {
   ENT(deletion),
   ENT(full),
   ENT(cleaning),
+  ENT(save_labeled),
+  ENT(read_labeled),
   END_OF_TESTCASES
 };