Replace "read consensus from disk" with "map consensus from disk".

Implements 27244, and should save a bunch of RAM on clients.

src/feature/dirauth/dirvote.c

@@ -3648,7 +3648,9 @@ dirvote_publish_consensus(void)
       continue;
     }
 
-    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+    if (networkstatus_set_current_consensus(pending->body,
+                                            strlen(pending->body),
+                                            name, 0, NULL))
       log_warn(LD_DIR, "Error publishing %s consensus", name);
     else
       log_notice(LD_DIR, "Published %s consensus", name);

src/feature/dircache/directory.c

@@ -2606,17 +2606,17 @@ handle_response_fetch_consensus(dir_connection_t *conn,
   if (looks_like_a_consensus_diff(body, body_len)) {
     /* First find our previous consensus. Maybe it's in ram, maybe not. */
     cached_dir_t *cd = dirserv_get_consensus(flavname);
-    const char *consensus_body;
+    const char *consensus_body = NULL;
     size_t consensus_body_len;
-    char *owned_consensus = NULL;
+    tor_mmap_t *mapped_consensus = NULL;
     if (cd) {
       consensus_body = cd->dir;
       consensus_body_len = cd->dir_len;
     } else {
-      owned_consensus = networkstatus_read_cached_consensus(flavname);
-      if (owned_consensus) {
-        consensus_body = owned_consensus;
-        consensus_body_len = strlen(consensus_body);
+      mapped_consensus = networkstatus_map_cached_consensus(flavname);
+      if (mapped_consensus) {
+        consensus_body = mapped_consensus->data;
+        consensus_body_len = mapped_consensus->size;
       }
     }
     if (!consensus_body) {
@@ -2629,7 +2629,7 @@ handle_response_fetch_consensus(dir_connection_t *conn,
 
     new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
                                          body, body_len);
-    tor_free(owned_consensus);
+    tor_munmap_file(mapped_consensus);
     if (new_consensus == NULL) {
       log_warn(LD_DIR, "Could not apply consensus diff received from server "
                "'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2651,7 +2651,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
     sourcename = "downloaded";
   }
 
-  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+  if ((r=networkstatus_set_current_consensus(consensus,
+                                             strlen(consensus),
+                                             flavname, 0,
                                              conn->identity_digest))<0) {
     log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
            "Unable to load %s consensus directory %s from "

src/feature/nodelist/networkstatus.c

@@ -105,8 +105,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
 typedef struct consensus_waiting_for_certs_t {
   /** The consensus itself. */
   networkstatus_t *consensus;
-  /** The encoded version of the consensus, nul-terminated. */
-  char *body;
   /** When did we set the current value of consensus_waiting_for_certs?  If
    * this is too recent, we shouldn't try to fetch a new consensus for a
    * little while, to give ourselves time to get certificates for this one. */
@@ -199,14 +197,11 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>.  If
- * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
- * the file isn't there. */
+/** Return the filename used to cache the consensus of a given flavor */
 static char *
-networkstatus_read_cached_consensus_impl(int flav,
-                                         const char *flavorname,
-                                         int unverified_consensus)
+networkstatus_get_cache_fname(int flav,
+                              const char *flavorname,
+                              int unverified_consensus)
 {
   char buf[128];
   const char *prefix;
@@ -221,21 +216,35 @@ networkstatus_read_cached_consensus_impl(int flav,
     tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
   }
 
-  char *filename = get_cachedir_fname(buf);
-  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+                                        const char *flavorname,
+                                        int unverified_consensus)
+{
+  char *filename = networkstatus_get_cache_fname(flav,
+                                                 flavorname,
+                                                 unverified_consensus);
+  tor_mmap_t *result = tor_mmap_file(filename);
   tor_free(filename);
   return result;
 }
 
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
   int flav = networkstatus_parse_flavor_name(flavorname);
   if (flav < 0)
     return NULL;
-  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+  return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
 }
 
 /** Read every cached v3 consensus networkstatus from the disk. */
@@ -248,24 +257,26 @@ router_reload_consensus_networkstatus(void)
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
     const char *flavor = networkstatus_get_flavor_name(flav);
-    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
+    tor_mmap_t *m = networkstatus_map_cached_consensus_impl(flav, flavor, 0);
+    if (m) {
+      if (networkstatus_set_current_consensus(m->data, m->size,
+                                              flavor, flags, NULL) < -1) {
         log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
                  flavor);
       }
-      tor_free(s);
+      tor_munmap_file(m);
     }
 
-    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor,
+    m = networkstatus_map_cached_consensus_impl(flav, flavor, 1);
+    if (m) {
+      if (networkstatus_set_current_consensus(m->data, m->size,
+                                     flavor,
                                      flags | NSSET_WAS_WAITING_FOR_CERTS,
                                      NULL)) {
         log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
                  "from cache", flavor);
       }
-      tor_free(s);
+      tor_munmap_file(m);
     }
   }
 
@@ -1833,6 +1844,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
  */
 int
 networkstatus_set_current_consensus(const char *consensus,
+                                    size_t consensus_len,
                                     const char *flavor,
                                     unsigned flags,
                                     const char *source_dir)
@@ -1862,7 +1874,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
   /* Make sure it's parseable. */
   c = networkstatus_parse_vote_from_string(consensus,
-                                           strlen(consensus),
+                                           consensus_len,
                                            NULL, NS_TYPE_CONSENSUS);
   if (!c) {
     log_warn(LD_DIR, "Unable to parse networkstatus consensus");
@@ -1951,14 +1963,12 @@ networkstatus_set_current_consensus(const char *consensus,
           c->valid_after > current_valid_after) {
         waiting = &consensus_waiting_for_certs[flav];
         networkstatus_vote_free(waiting->consensus);
-        tor_free(waiting->body);
         waiting->consensus = c;
         free_consensus = 0;
-        waiting->body = tor_strdup(consensus);
         waiting->set_at = now;
         waiting->dl_failed = 0;
         if (!from_cache) {
-          write_str_to_file(unverified_fname, consensus, 0);
+          write_bytes_to_file(unverified_fname, consensus, consensus_len, 0);
         }
         if (dl_certs)
           authority_certs_fetch_missing(c, now, source_dir);
@@ -2049,10 +2059,6 @@ networkstatus_set_current_consensus(const char *consensus,
       waiting->consensus->valid_after <= c->valid_after) {
     networkstatus_vote_free(waiting->consensus);
     waiting->consensus = NULL;
-    if (consensus != waiting->body)
-      tor_free(waiting->body);
-    else
-      waiting->body = NULL;
     waiting->set_at = 0;
     waiting->dl_failed = 0;
     if (unlink(unverified_fname) != 0) {
@@ -2147,14 +2153,16 @@ networkstatus_note_certs_arrived(const char *source_dir)
     if (!waiting->consensus)
       continue;
     if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
-      char *waiting_body = waiting->body;
-      if (!networkstatus_set_current_consensus(
-                                 waiting_body,
-                                 flavor_name,
-                                 NSSET_WAS_WAITING_FOR_CERTS,
-                                 source_dir)) {
-        tor_free(waiting_body);
+      tor_mmap_t *mapping = networkstatus_map_cached_consensus_impl(
+                                                 i, flavor_name, 1);
+      if (mapping) {
+        networkstatus_set_current_consensus(mapping->data,
+                                            mapping->size,
+                                            flavor_name,
+                                            NSSET_WAS_WAITING_FOR_CERTS,
+                                            source_dir);
       }
+      tor_munmap_file(mapping);
     }
   }
 }
@@ -2730,6 +2738,5 @@ networkstatus_free_all(void)
       networkstatus_vote_free(waiting->consensus);
       waiting->consensus = NULL;
     }
-    tor_free(waiting->body);
   }
 }

src/feature/nodelist/networkstatus.h

@@ -16,7 +16,7 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free_(routerstatus_t *rs);
 #define routerstatus_free(rs) \
@@ -108,6 +108,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
 #define NSSET_ACCEPT_OBSOLETE 8
 #define NSSET_REQUIRE_FLAVOR 16
 int networkstatus_set_current_consensus(const char *consensus,
+                                        size_t consensus_len,
                                         const char *flavor,
                                         unsigned flags,
                                         const char *source_dir);
@@ -159,4 +160,3 @@ extern networkstatus_t *current_md_consensus;
 #endif /* defined(NETWORKSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_NETWORKSTATUS_H) */
-

src/test/test_routerlist.c

@@ -659,7 +659,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
   MOCK(clock_skew_warning, mock_clock_skew_warning);
   /* Caller will call teardown_capture_of_logs() */
   setup_capture_of_logs(LOG_WARN);
-  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+  retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+                                               "microdesc", 0,
                                                NULL);
 
  done: