Browse Source

Split directory.c code into several modules

Parts of this C file naturally belong in dircache, dirclient, and
dircommon: so, move them there.
Nick Mathewson 5 years ago
parent
commit
194acfb51d
45 changed files with 3390 additions and 2643 deletions
  1. 3 0
      changes/ticket26744
  2. 8 2
      src/core/include.am
  3. 1 1
      src/core/mainloop/connection.c
  4. 1 1
      src/core/mainloop/mainloop.c
  5. 1 1
      src/core/or/circuitbuild.c
  6. 1 1
      src/core/or/circuitlist.c
  7. 1 1
      src/core/or/circuituse.c
  8. 1 1
      src/core/or/connection_edge.c
  9. 1 1
      src/core/or/relay.c
  10. 3 1
      src/feature/client/bridges.c
  11. 1 1
      src/feature/client/entrynodes.c
  12. 3 1
      src/feature/control/control.c
  13. 2 1
      src/feature/dirauth/dirvote.c
  14. 2 1
      src/feature/dirauth/process_descs.c
  15. 1739 0
      src/feature/dircache/dircache.c
  16. 43 0
      src/feature/dircache/dircache.h
  17. 0 347
      src/feature/dircache/directory.h
  18. 1 1
      src/feature/dircache/dirserv.c
  19. 96 2255
      src/feature/dirclient/dirclient.c
  20. 172 0
      src/feature/dirclient/dirclient.h
  21. 422 0
      src/feature/dirclient/dlstatus.c
  22. 58 0
      src/feature/dirclient/dlstatus.h
  23. 651 0
      src/feature/dircommon/directory.c
  24. 129 0
      src/feature/dircommon/directory.h
  25. 2 1
      src/feature/hs/hs_client.c
  26. 2 1
      src/feature/hs/hs_service.c
  27. 3 1
      src/feature/nodelist/authcert.c
  28. 1 1
      src/feature/nodelist/dirlist.c
  29. 2 1
      src/feature/nodelist/microdesc.c
  30. 3 1
      src/feature/nodelist/networkstatus.c
  31. 2 1
      src/feature/nodelist/node_select.c
  32. 3 1
      src/feature/nodelist/routerlist.c
  33. 2 1
      src/feature/relay/router.c
  34. 2 1
      src/feature/rend/rendclient.c
  35. 2 1
      src/feature/rend/rendservice.c
  36. 2 3
      src/test/fuzz/fuzz_http.c
  37. 1 1
      src/test/test_connection.c
  38. 7 1
      src/test/test_dir.c
  39. 3 1
      src/test/test_dir_handle_get.c
  40. 3 2
      src/test/test_entrynodes.c
  41. 4 2
      src/test/test_hs_cache.c
  42. 1 1
      src/test/test_hs_client.c
  43. 1 1
      src/test/test_hs_common.c
  44. 1 1
      src/test/test_oos.c
  45. 3 2
      src/test/test_routerlist.c

+ 3 - 0
changes/ticket26744

@@ -0,0 +1,3 @@
+  o Code simplification and refactoring:
+    - Split directory.c into separate pieces for client, server, and
+      common functionality. Closes ticket 26744.

+ 8 - 2
src/core/include.am

@@ -62,9 +62,12 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/dirauth/keypin.c		\
 	src/feature/dircache/conscache.c	\
 	src/feature/dircache/consdiffmgr.c	\
-	src/feature/dircache/directory.c	\
+	src/feature/dircache/dircache.c		\
 	src/feature/dircache/dirserv.c		\
+	src/feature/dirclient/dirclient.c	\
+	src/feature/dirclient/dlstatus.c	\
 	src/feature/dircommon/consdiff.c	\
+	src/feature/dircommon/directory.c	\
 	src/feature/dircommon/fp_pair.c		\
 	src/feature/dircommon/voting_schedule.c	\
 	src/feature/hibernate/hibernate.c	\
@@ -257,12 +260,15 @@ noinst_HEADERS +=					\
 	src/feature/dircache/cached_dir_st.h		\
 	src/feature/dircache/conscache.h		\
 	src/feature/dircache/consdiffmgr.h		\
-	src/feature/dircache/directory.h		\
+	src/feature/dircache/dircache.h			\
 	src/feature/dircache/dirserv.h			\
 	src/feature/dirclient/dir_server_st.h		\
+	src/feature/dirclient/dirclient.h		\
+	src/feature/dirclient/dlstatus.h		\
 	src/feature/dirclient/download_status_st.h	\
 	src/feature/dircommon/consdiff.h		\
 	src/feature/dircommon/dir_connection_st.h	\
+	src/feature/dircommon/directory.h		\
 	src/feature/dircommon/fp_pair.h			\
 	src/feature/dircommon/vote_timing_st.h		\
 	src/feature/dircommon/voting_schedule.h		\

+ 1 - 1
src/core/mainloop/connection.c

@@ -77,7 +77,7 @@
 #include "core/or/connection_or.h"
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/relay/dns.h"
 #include "feature/client/dnsserv.h"

+ 1 - 1
src/core/mainloop/mainloop.c

@@ -68,7 +68,7 @@
 #include "feature/control/control.h"
 #include "core/mainloop/cpuworker.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/relay/dns.h"

+ 1 - 1
src/core/or/circuitbuild.c

@@ -44,7 +44,7 @@
 #include "core/or/connection_or.h"
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/client/entrynodes.h"
 #include "core/crypto/hs_ntor.h"
 #include "core/mainloop/mainloop.h"

+ 1 - 1
src/core/or/circuitlist.c

@@ -69,7 +69,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_dh.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/client/entrynodes.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/hs/hs_circuit.h"

+ 1 - 1
src/core/or/circuituse.c

@@ -40,7 +40,7 @@
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/control/control.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_client.h"

+ 1 - 1
src/core/or/connection_edge.c

@@ -73,7 +73,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/relay/dns.h"
 #include "feature/client/dnsserv.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_common.h"

+ 1 - 1
src/core/or/relay.c

@@ -63,7 +63,7 @@
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/relay/dns.h"
 #include "feature/stats/geoip.h"
 #include "feature/hs/hs_cache.h"

+ 3 - 1
src/feature/client/bridges.c

@@ -18,7 +18,9 @@
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/client/entrynodes.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/policies.h"

+ 1 - 1
src/feature/client/entrynodes.c

@@ -125,7 +125,7 @@
 #include "core/mainloop/connection.h"
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/client/entrynodes.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/microdesc.h"

+ 3 - 1
src/feature/control/control.c

@@ -56,7 +56,9 @@
 #include "feature/control/fmt_serverstatus.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"

+ 2 - 1
src/feature/dirauth/dirvote.c

@@ -9,7 +9,8 @@
 #include "feature/dirauth/dircollate.h"
 #include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/voteflags.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/guardfraction.h"

+ 2 - 1
src/feature/dirauth/process_descs.c

@@ -19,7 +19,8 @@
 #include "core/or/policies.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/reachability.h"
-#include "feature/dircache/directory.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"

+ 1739 - 0
src/feature/dircache/dircache.c

@@ -0,0 +1,1739 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRCACHE_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/relay.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/mode.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip.h"
+#include "feature/stats/rephist.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Maximum size, in bytes, for any directory object that we're accepting
+ * as an upload. */
+#define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */
+
+/** HTTP cache control: how long do we tell proxies they can cache each
+ * kind of document we serve? */
+#define FULL_DIR_CACHE_LIFETIME (60*60)
+#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
+#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
+#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
+#define ROUTERDESC_CACHE_LIFETIME (30*60)
+#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
+#define ROBOTS_CACHE_LIFETIME (24*60*60)
+#define MICRODESC_CACHE_LIFETIME (48*60*60)
+
+/** Parse an HTTP request string <b>headers</b> of the form
+ * \verbatim
+ * "\%s [http[s]://]\%s HTTP/1..."
+ * \endverbatim
+ * If it's well-formed, strdup the second \%s into *<b>url</b>, and
+ * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
+ * so it does. Return 0.
+ * Otherwise, return -1.
+ */
+STATIC int
+parse_http_url(const char *headers, char **url)
+{
+  char *command = NULL;
+  if (parse_http_command(headers, &command, url) < 0) {
+    return -1;
+  }
+  if (strcmpstart(*url, "/tor/")) {
+    char *new_url = NULL;
+    tor_asprintf(&new_url, "/tor%s%s",
+                 *url[0] == '/' ? "" : "/",
+                 *url);
+    tor_free(*url);
+    *url = new_url;
+  }
+  tor_free(command);
+  return 0;
+}
+
+/** Create an http response for the client <b>conn</b> out of
+ * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
+ */
+static void
+write_short_http_response(dir_connection_t *conn, int status,
+                       const char *reason_phrase)
+{
+  char *buf = NULL;
+  char *datestring = NULL;
+
+  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
+    reason_phrase = "unspecified";
+  }
+
+  if (server_mode(get_options())) {
+    /* include the Date: header, but only if we're a relay or bridge */
+    char datebuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(datebuf, time(NULL));
+    tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
+  }
+
+  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
+               status, reason_phrase, datestring?datestring:"");
+
+  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
+  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+
+  tor_free(datestring);
+  tor_free(buf);
+}
+
+/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
+ * with <b>type</b> as the Content-Type.
+ *
+ * If <b>length</b> is nonnegative, it is the Content-Length.
+ * If <b>encoding</b> is provided, it is the Content-Encoding.
+ * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
+ * up to cache_lifetime seconds.  Otherwise, the content may not be cached. */
+static void
+write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
+                           const char *type, const char *encoding,
+                           const char *extra_headers,
+                           long cache_lifetime)
+{
+  char date[RFC1123_TIME_LEN+1];
+  time_t now = time(NULL);
+  buf_t *buf = buf_new_with_capacity(1024);
+
+  tor_assert(conn);
+
+  format_rfc1123_time(date, now);
+
+  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
+  if (type) {
+    buf_add_printf(buf, "Content-Type: %s\r\n", type);
+  }
+  if (!is_local_addr(&conn->base_.addr)) {
+    /* Don't report the source address for a nearby/private connection.
+     * Otherwise we tend to mis-report in cases where incoming ports are
+     * being forwarded to a Tor server running behind the firewall. */
+    buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
+  }
+  if (encoding) {
+    buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
+  }
+  if (length >= 0) {
+    buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
+  }
+  if (cache_lifetime > 0) {
+    char expbuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
+    /* We could say 'Cache-control: max-age=%d' here if we start doing
+     * http/1.1 */
+    buf_add_printf(buf, "Expires: %s\r\n", expbuf);
+  } else if (cache_lifetime == 0) {
+    /* We could say 'Cache-control: no-cache' here if we start doing
+     * http/1.1 */
+    buf_add_string(buf, "Pragma: no-cache\r\n");
+  }
+  if (extra_headers) {
+    buf_add_string(buf, extra_headers);
+  }
+  buf_add_string(buf, "\r\n");
+
+  connection_buf_add_buf(TO_CONN(conn), buf);
+  buf_free(buf);
+}
+
+/** As write_http_response_header_impl, but sets encoding and content-typed
+ * based on whether the response will be <b>compressed</b> or not. */
+static void
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+                            compress_method_t method,
+                            const char *extra_headers, long cache_lifetime)
+{
+  const char *methodname = compression_method_get_name(method);
+  const char *doctype;
+  if (method == NO_METHOD)
+    doctype = "text/plain";
+  else
+    doctype = "application/octet-stream";
+  write_http_response_header_impl(conn, length,
+                                  doctype,
+                                  methodname,
+                                  extra_headers,
+                                  cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+                           compress_method_t method,
+                           long cache_lifetime)
+{
+  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 &lt;&lt; x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+  unsigned result = (1u << NO_METHOD);
+  smartlist_t *methods = smartlist_new();
+  smartlist_split_string(methods, h, ",",
+             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+    compress_method_t method = compression_method_get_by_name(m);
+    if (method != UNKNOWN_METHOD) {
+      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+      result |= (1u << method);
+    }
+  } SMARTLIST_FOREACH_END(m);
+  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+    tor_free(m);
+  } SMARTLIST_FOREACH_END(m);
+  smartlist_free(methods);
+  return result;
+}
+
+/** Decide whether a client would accept the consensus we have.
+ *
+ * Clients can say they only want a consensus if it's signed by more
+ * than half the authorities in a list.  They pass this list in
+ * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
+ *
+ * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
+ * of the full authority identity digest. (Only strings of even length,
+ * i.e. encodings of full bytes, are handled correctly.  In the case
+ * of an odd number of hex digits the last one is silently ignored.)
+ *
+ * Returns 1 if more than half of the requested authorities signed the
+ * consensus, 0 otherwise.
+ */
+static int
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                       const char *want_url)
+{
+  smartlist_t *voters = smartlist_new();
+  int need_at_least;
+  int have = 0;
+
+  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+    smartlist_free(voters);
+    return 1; // We don't know the voters; assume the client won't mind. */
+  }
+
+  smartlist_t *want_authorities = smartlist_new();
+  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
+  need_at_least = smartlist_len(want_authorities)/2+1;
+
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+      if (!strcasecmpstart(digest, want_digest)) {
+        have++;
+        break;
+      };
+    } SMARTLIST_FOREACH_END(digest);
+
+    /* early exit, if we already have enough */
+    if (have >= need_at_least)
+      break;
+  } SMARTLIST_FOREACH_END(want_digest);
+
+  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
+  smartlist_free(want_authorities);
+  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+  smartlist_free(voters);
+  return (have >= need_at_least);
+}
+
+/** Return the compression level we should use for sending a compressed
+ * response of size <b>n_bytes</b>. */
+STATIC compression_level_t
+choose_compression_level(ssize_t n_bytes)
+{
+  if (! have_been_under_memory_pressure()) {
+    return HIGH_COMPRESSION; /* we have plenty of RAM. */
+  } else if (n_bytes < 0) {
+    return HIGH_COMPRESSION; /* unknown; might be big. */
+  } else if (n_bytes < 1024) {
+    return LOW_COMPRESSION;
+  } else if (n_bytes < 2048) {
+    return MEDIUM_COMPRESSION;
+  } else {
+    return HIGH_COMPRESSION;
+  }
+}
+
+/** Information passed to handle a GET request. */
+typedef struct get_handler_args_t {
+  /** Bitmask of compression methods that the client said (or implied) it
+   * supported. */
+  unsigned compression_supported;
+  /** If nonzero, the time included an if-modified-since header with this
+   * value. */
+  time_t if_modified_since;
+  /** String containing the requested URL or resource. */
+  const char *url;
+  /** String containing the HTTP headers */
+  const char *headers;
+} get_handler_args_t;
+
+/** Entry for handling an HTTP GET request.
+ *
+ * This entry matches a request if "string" is equal to the requested
+ * resource, or if "is_prefix" is true and "string" is a prefix of the
+ * requested resource.
+ *
+ * The 'handler' function is called to handle the request.  It receives
+ * an arguments structure, and must return 0 on success or -1 if we should
+ * close the connection.
+ **/
+typedef struct url_table_ent_s {
+  const char *string;
+  int is_prefix;
+  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
+} url_table_ent_t;
+
+static int handle_get_frontpage(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_current_consensus(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_status_vote(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_microdesc(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_descriptor(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_keys(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                                       const get_handler_args_t *args);
+static int handle_get_robots(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+static int handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                const get_handler_args_t *args);
+
+/** Table for handling GET requests. */
+static const url_table_ent_t url_table[] = {
+  { "/tor/", 0, handle_get_frontpage },
+  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
+  { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/", 1, handle_get_status_vote },
+  { "/tor/micro/d/", 1, handle_get_microdesc },
+  { "/tor/server/", 1, handle_get_descriptor },
+  { "/tor/extra/", 1, handle_get_descriptor },
+  { "/tor/keys/", 1, handle_get_keys },
+  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
+  { "/tor/robots.txt", 0, handle_get_robots },
+  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
+  { NULL, 0, NULL },
+};
+
+/** Helper function: called when a dirserver gets a complete HTTP GET
+ * request.  Look for a request for a directory or for a rendezvous
+ * service descriptor.  On finding one, write a response into
+ * conn-\>outbuf.  If the request is unrecognized, send a 404.
+ * Return 0 if we handled this successfully, or -1 if we need to close
+ * the connection. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+                              const char *req_body, size_t req_body_len))
+{
+  char *url, *url_mem, *header;
+  time_t if_modified_since = 0;
+  int zlib_compressed_in_url;
+  unsigned compression_methods_supported;
+
+  /* We ignore the body of a GET request. */
+  (void)req_body;
+  (void)req_body_len;
+
+  log_debug(LD_DIRSERV,"Received GET command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
+    struct tm tm;
+    if (parse_http_time(header, &tm) == 0) {
+      if (tor_timegm(&tm, &if_modified_since)<0) {
+        if_modified_since = 0;
+      } else {
+        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
+      }
+    }
+    /* The correct behavior on a malformed If-Modified-Since header is to
+     * act as if no If-Modified-Since header had been given. */
+    tor_free(header);
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  url_mem = url;
+  {
+    size_t url_len = strlen(url);
+
+    zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+    if (zlib_compressed_in_url) {
+      url[url_len-2] = '\0';
+    }
+  }
+
+  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+    compression_methods_supported = parse_accept_encoding_header(header);
+    tor_free(header);
+  } else {
+    compression_methods_supported = (1u << NO_METHOD);
+  }
+  if (zlib_compressed_in_url) {
+    compression_methods_supported |= (1u << ZLIB_METHOD);
+  }
+
+  /* Remove all methods that we don't both support. */
+  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
+
+  get_handler_args_t args;
+  args.url = url;
+  args.headers = headers;
+  args.if_modified_since = if_modified_since;
+  args.compression_supported = compression_methods_supported;
+
+  int i, result = -1;
+  for (i = 0; url_table[i].string; ++i) {
+    int match;
+    if (url_table[i].is_prefix) {
+      match = !strcmpstart(url, url_table[i].string);
+    } else {
+      match = !strcmp(url, url_table[i].string);
+    }
+    if (match) {
+      result = url_table[i].handler(conn, &args);
+      goto done;
+    }
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+  result = 0;
+
+ done:
+  tor_free(url_mem);
+  return result;
+}
+
+/** Helper function for GET / or GET /tor/
+ */
+static int
+handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void) args; /* unused */
+  const char *frontpage = get_dirportfrontpage();
+
+  if (frontpage) {
+    size_t dlen;
+    dlen = strlen(frontpage);
+    /* Let's return a disclaimer page (users shouldn't use V1 anymore,
+       and caches don't fetch '/', so this is safe). */
+
+    /* [We don't check for write_bucket_low here, since we want to serve
+     *  this page no matter what.] */
+    write_http_response_header_impl(conn, dlen, "text/html", "identity",
+                                    NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
+    connection_buf_add(frontpage, dlen, TO_CONN(conn));
+  } else {
+    write_short_http_response(conn, 404, "Not found");
+  }
+  return 0;
+}
+
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
+ */
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now)
+{
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+  char timestamp[ISO_TIME_LEN+1];
+  time_t valid_until;
+  char *dupes;
+
+  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+    return;
+
+  if ((dupes = rate_limit_log(&warned, now))) {
+    format_local_iso_time(timestamp, valid_until);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+             "serve it to clients. It was valid until %s local time and we "
+             "continued to serve it for up to 24 hours after it expired.%s",
+             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+    tor_free(dupes);
+  }
+}
+
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+                    const char *action)
+{
+  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+      DIGEST256_LEN) {
+    return 0;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "%s contained bogus digest %s; %s.",
+           location, escaped(hex), action);
+    return -1;
+  }
+}
+
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0.  Otherwise return
+ * -1.  */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+  if (hdr == NULL) {
+    return -1;
+  }
+  smartlist_t *hex_digests = smartlist_new();
+  *digests_out = smartlist_new();
+  smartlist_split_string(hex_digests, hdr, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+    uint8_t digest[DIGEST256_LEN];
+    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+                             "ignoring")) {
+      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+    }
+  } SMARTLIST_FOREACH_END(hex);
+  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+  smartlist_free(hex_digests);
+  tor_free(hdr);
+  return 0;
+}
+
+/** Fallback compression method.  The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>.  The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+               unsigned compression_methods,
+               compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    unsigned u;
+    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+      compress_method_t method = srv_meth_pref_precompressed[u];
+      if (0 == (compression_methods & (1u<<method)))
+        continue; // client doesn't like this one, or we don't have it.
+      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+                                     diff_from, DIGEST256_LEN,
+                                     method) == CONSDIFF_AVAILABLE) {
+        tor_assert_nonfatal(result);
+        *compression_used_out = method;
+        return result;
+      }
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = FALLBACK_COMPRESS_METHOD;
+      return result;
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The preferred set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+                    unsigned compression_methods,
+                    compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+    compress_method_t method = srv_meth_pref_precompressed[u];
+
+    if (0 == (compression_methods & (1u<<method)))
+      continue;
+
+    if (consdiffmgr_find_consensus(&result, flav,
+                                   method) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = method;
+      return result;
+    }
+  }
+
+  if (consdiffmgr_find_consensus(&result, flav,
+        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+    tor_assert_nonfatal(result);
+    *compression_used_out = FALLBACK_COMPRESS_METHOD;
+    return result;
+  }
+
+  return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+  unsigned u;
+  compress_method_t *methods;
+  size_t length;
+
+  if (stream) {
+    methods = srv_meth_pref_streaming_compression;
+    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+  } else {
+    methods = srv_meth_pref_precompressed;
+    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+  }
+
+  for (u = 0; u < length; ++u) {
+    compress_method_t method = methods[u];
+    if (compression_methods & (1u<<method))
+      return method;
+  }
+
+  return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ *  flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+                                    const smartlist_t *digests)
+{
+  const networkstatus_t *ns = NULL;
+
+  if (digests == NULL)
+    return 0;
+
+  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+  if (ns == NULL)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+      return 1;
+  } SMARTLIST_FOREACH_END(digest);
+
+  return 0;
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+  /** name of the flavor to retrieve. */
+  char *flavor;
+  /** flavor to retrive, as enum. */
+  consensus_flavor_t flav;
+  /** plus-separated list of authority fingerprints; see
+   * client_likes_consensus(). Aliases the URL in the request passed to
+   * parse_consensus_request(). */
+  const char *want_fps;
+  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+   * to return a diff from. */
+  smartlist_t *diff_from_digests;
+  /** If true, never send a full consensus. If there is no diff, send
+   * a 404 instead. */
+  int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+  if (!req)
+    return;
+  tor_free(req->flavor);
+  if (req->diff_from_digests) {
+    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(req->diff_from_digests);
+  }
+  memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+                        const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  memset(out, 0, sizeof(parsed_consensus_request_t));
+  out->flav = FLAV_NS;
+
+  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+  const char CONSENSUS_FLAVORED_PREFIX[] =
+    "/tor/status-vote/current/consensus-";
+
+  /* figure out the flavor if any, and who we wanted to sign the thing */
+  const char *after_flavor = NULL;
+
+  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+    const char *f, *cp;
+    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+    cp = strchr(f, '/');
+    if (cp) {
+      after_flavor = cp+1;
+      out->flavor = tor_strndup(f, cp-f);
+    } else {
+      out->flavor = tor_strdup(f);
+    }
+    int flav = networkstatus_parse_flavor_name(out->flavor);
+    if (flav < 0)
+      flav = FLAV_NS;
+    out->flav = flav;
+  } else {
+    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+  }
+
+  /* see whether we've been asked explicitly for a diff from an older
+   * consensus. (The user might also have said that a diff would be okay,
+   * via X-Or-Diff-From-Consensus */
+  const char DIFF_COMPONENT[] = "diff/";
+  char *diff_hash_in_url = NULL;
+  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+    after_flavor += strlen(DIFF_COMPONENT);
+    const char *cp = strchr(after_flavor, '/');
+    if (cp) {
+      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+      out->want_fps = cp+1;
+    } else {
+      diff_hash_in_url = tor_strdup(after_flavor);
+      out->want_fps = NULL;
+    }
+  } else {
+    out->want_fps = after_flavor;
+  }
+
+  if (diff_hash_in_url) {
+    uint8_t diff_from[DIGEST256_LEN];
+    out->diff_from_digests = smartlist_new();
+    out->diff_only = 1;
+    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+                                  "rejecting");
+    tor_free(diff_hash_in_url);
+    if (ok) {
+      smartlist_add(out->diff_from_digests,
+                    tor_memdup(diff_from, DIGEST256_LEN));
+    } else {
+      return -1;
+    }
+  } else {
+    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+  }
+
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+                             const get_handler_args_t *args)
+{
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 0);
+  const time_t if_modified_since = args->if_modified_since;
+  int clear_spool = 0;
+
+  /* v3 network status fetch. */
+  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+  time_t now = time(NULL);
+  parsed_consensus_request_t req;
+
+  if (parse_consensus_request(&req, args) < 0) {
+    write_short_http_response(conn, 404, "Couldn't parse request");
+    goto done;
+  }
+
+  if (digest_list_contains_best_consensus(req.flav,
+                                          req.diff_from_digests)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  struct consensus_cache_entry_t *cached_consensus = NULL;
+
+  compress_method_t compression_used = NO_METHOD;
+  if (req.diff_from_digests) {
+    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+                                      args->compression_supported,
+                                      &compression_used);
+  }
+
+  if (req.diff_only && !cached_consensus) {
+    write_short_http_response(conn, 404, "No such diff available");
+    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (! cached_consensus) {
+    cached_consensus = find_best_consensus(req.flav,
+                                           args->compression_supported,
+                                           &compression_used);
+  }
+
+  time_t fresh_until, valid_until;
+  int have_fresh_until = 0, have_valid_until = 0;
+  if (cached_consensus) {
+    have_fresh_until =
+      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+    have_valid_until =
+      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+  }
+
+  if (cached_consensus && have_valid_until &&
+      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+    write_short_http_response(conn, 404, "Consensus is too old");
+    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (cached_consensus && req.want_fps &&
+      !client_likes_consensus(cached_consensus, req.want_fps)) {
+    write_short_http_response(conn, 404, "Consensus not signed by sufficient "
+                           "number of requested authorities");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+    goto done;
+  }
+
+  conn->spool = smartlist_new();
+  clear_spool = 1;
+  {
+    spooled_resource_t *spooled;
+    if (cached_consensus) {
+      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+      smartlist_add(conn->spool, spooled);
+    }
+  }
+
+  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
+
+  size_t size_guess = 0;
+  int n_expired = 0;
+  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+                                              compress_method != NO_METHOD,
+                                              &size_guess,
+                                              &n_expired);
+
+  if (!smartlist_len(conn->spool) && !n_expired) {
+    write_short_http_response(conn, 404, "Not found");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (!smartlist_len(conn->spool)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    log_debug(LD_DIRSERV,
+              "Client asked for network status lists, but we've been "
+              "writing too many bytes lately. Sending 503 Dir busy.");
+    write_short_http_response(conn, 503, "Directory busy, try again later");
+    geoip_note_ns_response(GEOIP_REJECT_BUSY);
+    goto done;
+  }
+
+  tor_addr_t addr;
+  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+                           &addr, NULL,
+                           time(NULL));
+    geoip_note_ns_response(GEOIP_SUCCESS);
+    /* Note that a request for a network status has started, so that we
+     * can measure the download time later on. */
+    if (conn->dirreq_id)
+      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+    else
+      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+                         DIRREQ_DIRECT);
+  }
+
+  /* Use this header to tell caches that the response depends on the
+   * X-Or-Diff-From-Consensus header (or lack thereof). */
+  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+  clear_spool = 0;
+
+  // The compress_method might have been NO_METHOD, but we store the data
+  // compressed. Decompress them using `compression_used`. See fallback code in
+  // find_best_consensus() and find_best_diff().
+  write_http_response_headers(conn, -1,
+                             compress_method == NO_METHOD ?
+                               NO_METHOD : compression_used,
+                             vary_header,
+                             smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+    conn->compress_state = tor_compress_new(0, compression_used,
+                                            HIGH_COMPRESSION);
+
+  /* Prime the connection with some data. */
+  const int initial_flush_result = connection_dirserv_flushed_some(conn);
+  tor_assert_nonfatal(initial_flush_result == 0);
+  goto done;
+
+ done:
+  parsed_consensus_request_clear(&req);
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/{current,next}/...
+ */
+static int
+handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  {
+    ssize_t body_len = 0;
+    ssize_t estimated_len = 0;
+    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
+    /* This smartlist holds strings that we can compress on the fly. */
+    smartlist_t *items = smartlist_new();
+    /* This smartlist holds cached_dir_t objects that have a precompressed
+     * deflated version. */
+    smartlist_t *dir_items = smartlist_new();
+    dirvote_dirreq_get_status_vote(url, items, dir_items);
+    if (!smartlist_len(dir_items) && !smartlist_len(items)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto vote_done;
+    }
+
+    /* We're sending items from at most one kind of source */
+    tor_assert_nonfatal(smartlist_len(items) == 0 ||
+                        smartlist_len(dir_items) == 0);
+
+    int streaming;
+    unsigned mask;
+    if (smartlist_len(items)) {
+      /* We're taking strings and compressing them on the fly. */
+      streaming = 1;
+      mask = ~0u;
+    } else {
+      /* We're taking cached_dir_t objects. We only have them uncompressed
+       * or deflated. */
+      streaming = 0;
+      mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+    }
+    const compress_method_t compress_method = find_best_compression_method(
+                              args->compression_supported&mask, streaming);
+
+    SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+                      body_len += compress_method != NO_METHOD ?
+                        d->dir_compressed_len : d->dir_len);
+    estimated_len += body_len;
+    SMARTLIST_FOREACH(items, const char *, item, {
+        size_t ln = strlen(item);
+        if (compress_method != NO_METHOD) {
+          estimated_len += ln/2;
+        } else {
+          body_len += ln; estimated_len += ln;
+        }
+      });
+
+    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto vote_done;
+    }
+    write_http_response_header(conn, body_len ? body_len : -1,
+                 compress_method,
+                 lifetime);
+
+    if (smartlist_len(items)) {
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                           choose_compression_level(estimated_len));
+        SMARTLIST_FOREACH(items, const char *, c,
+                 connection_buf_add_compress(c, strlen(c), conn, 0));
+        connection_buf_add_compress("", 0, conn, 1);
+      } else {
+        SMARTLIST_FOREACH(items, const char *, c,
+                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
+      }
+    } else {
+      SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+          connection_buf_add(compress_method != NO_METHOD ?
+                                    d->dir_compressed : d->dir,
+                                  compress_method != NO_METHOD ?
+                                    d->dir_compressed_len : d->dir_len,
+                                  TO_CONN(conn)));
+    }
+  vote_done:
+    smartlist_free(items);
+    smartlist_free(dir_items);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/micro/d/...
+ */
+static int
+handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  int clear_spool = 1;
+  {
+    conn->spool = smartlist_new();
+
+    dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+                                      DIR_SPOOL_MICRODESC,
+                                      conn->spool, NULL,
+                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
+
+    size_t size_guess = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, 0,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, NULL);
+    if (smartlist_len(conn->spool) == 0) {
+      write_short_http_response(conn, 404, "Not found");
+      goto done;
+    }
+    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      log_info(LD_DIRSERV,
+               "Client asked for server descriptors, but we've been "
+               "writing too many bytes lately. Sending 503 Dir busy.");
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto done;
+    }
+
+    clear_spool = 0;
+    write_http_response_header(conn, -1,
+                               compress_method,
+                               MICRODESC_CACHE_LIFETIME);
+
+    if (compress_method != NO_METHOD)
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                      choose_compression_level(size_guess));
+
+    const int initial_flush_result = connection_dirserv_flushed_some(conn);
+    tor_assert_nonfatal(initial_flush_result == 0);
+    goto done;
+  }
+
+ done:
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
+  return 0;
+}
+
+/** Helper function for GET /tor/{server,extra}/...
+ */
+static int
+handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const or_options_t *options = get_options();
+  int clear_spool = 1;
+  if (!strcmpstart(url,"/tor/server/") ||
+      (!options->BridgeAuthoritativeDir &&
+       !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
+    int res;
+    const char *msg = NULL;
+    int cache_lifetime = 0;
+    int is_extra = !strcmpstart(url,"/tor/extra/");
+    url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
+    dir_spool_source_t source;
+    time_t publish_cutoff = 0;
+    if (!strcmpstart(url, "d/")) {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
+    } else {
+      source =
+        is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+      /* We only want to apply a publish cutoff when we're requesting
+       * resources by fingerprint. */
+      publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    }
+
+    conn->spool = smartlist_new();
+    res = dirserv_get_routerdesc_spool(conn->spool, url,
+                                       source,
+                                       connection_dir_is_encrypted(conn),
+                                       &msg);
+
+    if (!strcmpstart(url, "all")) {
+      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+    } else if (smartlist_len(conn->spool) == 1) {
+      cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
+    }
+
+    size_t size_guess = 0;
+    int n_expired = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, &n_expired);
+
+    /* If we are the bridge authority and the descriptor is a bridge
+     * descriptor, remember that we served this descriptor for desc stats. */
+    /* XXXX it's a bit of a kludge to have this here. */
+    if (get_options()->BridgeAuthoritativeDir &&
+        source == DIR_SPOOL_SERVER_BY_FP) {
+      SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+        const routerinfo_t *router =
+          router_get_by_id_digest((const char *)spooled->digest);
+        /* router can be NULL here when the bridge auth is asked for its own
+         * descriptor. */
+        if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+          rep_hist_note_desc_served(router->cache_info.identity_digest);
+      } SMARTLIST_FOREACH_END(spooled);
+    }
+
+    if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+      if (msg == NULL)
+        msg = "Not found";
+      write_short_http_response(conn, 404, msg);
+    } else {
+      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+        log_info(LD_DIRSERV,
+                 "Client asked for server descriptors, but we've been "
+                 "writing too many bytes lately. Sending 503 Dir busy.");
+        write_short_http_response(conn, 503,
+                                  "Directory busy, try again later");
+        dir_conn_clear_spool(conn);
+        goto done;
+      }
+      write_http_response_header(conn, -1, compress_method, cache_lifetime);
+      if (compress_method != NO_METHOD)
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(size_guess));
+      clear_spool = 0;
+      /* Prime the connection with some data. */
+      int initial_flush_result = connection_dirserv_flushed_some(conn);
+      tor_assert_nonfatal(initial_flush_result == 0);
+    }
+    goto done;
+  }
+ done:
+  if (clear_spool)
+    dir_conn_clear_spool(conn);
+  return 0;
+}
+
+/** Helper function for GET /tor/keys/...
+ */
+static int
+handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  const time_t if_modified_since = args->if_modified_since;
+  {
+    smartlist_t *certs = smartlist_new();
+    ssize_t len = -1;
+    if (!strcmp(url, "/tor/keys/all")) {
+      authority_cert_get_all(certs);
+    } else if (!strcmp(url, "/tor/keys/authority")) {
+      authority_cert_t *cert = get_my_v3_authority_cert();
+      if (cert)
+        smartlist_add(certs, cert);
+    } else if (!strcmpstart(url, "/tor/keys/fp/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_newest_by_id(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/sk/")) {
+      smartlist_t *fps = smartlist_new();
+      dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
+                                           fps, NULL,
+                                           DSR_HEX|DSR_SORT_UNIQ);
+      SMARTLIST_FOREACH(fps, char *, d, {
+          authority_cert_t *c = authority_cert_get_by_sk_digest(d);
+          if (c) smartlist_add(certs, c);
+          tor_free(d);
+      });
+      smartlist_free(fps);
+    } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
+      smartlist_t *fp_sks = smartlist_new();
+      dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
+                                                fp_sks);
+      SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
+          authority_cert_t *c = authority_cert_get_by_digests(pair->first,
+                                                              pair->second);
+          if (c) smartlist_add(certs, c);
+          tor_free(pair);
+      });
+      smartlist_free(fp_sks);
+    } else {
+      write_short_http_response(conn, 400, "Bad request");
+      goto keys_done;
+    }
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 404, "Not found");
+      goto keys_done;
+    }
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+      if (c->cache_info.published_on < if_modified_since)
+        SMARTLIST_DEL_CURRENT(certs, c));
+    if (!smartlist_len(certs)) {
+      write_short_http_response(conn, 304, "Not modified");
+      goto keys_done;
+    }
+    len = 0;
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+                      len += c->cache_info.signed_descriptor_len);
+
+    if (global_write_bucket_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len,
+                                2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
+      goto keys_done;
+    }
+
+    write_http_response_header(conn,
+                               compress_method != NO_METHOD ? -1 : len,
+                               compress_method,
+                               60*60);
+    if (compress_method != NO_METHOD) {
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                              choose_compression_level(len));
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add_compress(
+                c->cache_info.signed_descriptor_body,
+                c->cache_info.signed_descriptor_len,
+                conn, 0));
+      connection_buf_add_compress("", 0, conn, 1);
+    } else {
+      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+            connection_buf_add(c->cache_info.signed_descriptor_body,
+                                    c->cache_info.signed_descriptor_len,
+                                    TO_CONN(conn)));
+    }
+  keys_done:
+    smartlist_free(certs);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/rendezvous2/
+ */
+static int
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  if (connection_dir_is_encrypted(conn)) {
+    /* Handle v2 rendezvous descriptor fetch request. */
+    const char *descp;
+    const char *query = url + strlen("/tor/rendezvous2/");
+    if (rend_valid_descriptor_id(query)) {
+      log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
+               safe_str(escaped(query)));
+      switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
+        case 1: /* valid */
+          write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
+          connection_buf_add(descp, strlen(descp), TO_CONN(conn));
+          break;
+        case 0: /* well-formed but not present */
+          write_short_http_response(conn, 404, "Not found");
+          break;
+        case -1: /* not well-formed */
+          write_short_http_response(conn, 400, "Bad request");
+          break;
+      }
+    } else { /* not well-formed */
+      write_short_http_response(conn, 400, "Bad request");
+    }
+    goto done;
+  } else {
+    /* Not encrypted! */
+    write_short_http_response(conn, 404, "Not found");
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  int retval;
+  const char *desc_str = NULL;
+  const char *pubkey_str = NULL;
+  const char *url = args->url;
+
+  /* Reject unencrypted dir connections */
+  if (!connection_dir_is_encrypted(conn)) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* After the path prefix follows the base64 encoded blinded pubkey which we
+   * use to get the descriptor from the cache. Skip the prefix and get the
+   * pubkey. */
+  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+  pubkey_str = url + strlen("/tor/hs/3/");
+  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+                                  pubkey_str, &desc_str);
+  if (retval <= 0 || desc_str == NULL) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* Found requested descriptor! Pass it to this nice client. */
+  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
+
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/networkstatus-bridges
+ */
+static int
+handle_get_networkstatus_bridges(dir_connection_t *conn,
+                                 const get_handler_args_t *args)
+{
+  const char *headers = args->headers;
+
+  const or_options_t *options = get_options();
+  if (options->BridgeAuthoritativeDir &&
+      options->BridgePassword_AuthDigest_ &&
+      connection_dir_is_encrypted(conn)) {
+    char *status;
+    char digest[DIGEST256_LEN];
+
+    char *header = http_get_header(headers, "Authorization: Basic ");
+    if (header)
+      crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
+
+    /* now make sure the password is there and right */
+    if (!header ||
+        tor_memneq(digest,
+                   options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
+      write_short_http_response(conn, 404, "Not found");
+      tor_free(header);
+      goto done;
+    }
+    tor_free(header);
+
+    /* all happy now. send an answer. */
+    status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
+    size_t dlen = strlen(status);
+    write_http_response_header(conn, dlen, NO_METHOD, 0);
+    connection_buf_add(status, dlen, TO_CONN(conn));
+    tor_free(status);
+    goto done;
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET robots.txt or /tor/robots.txt */
+static int
+handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
+{
+  (void)args;
+  {
+    const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
+    size_t len = strlen(robots);
+    write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
+    connection_buf_add(robots, len, TO_CONN(conn));
+  }
+  return 0;
+}
+
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the separator "/". For instance:
+ *      /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+                           const char **end_pos)
+{
+  int ok;
+  unsigned long version;
+  const char *start;
+  char *end = NULL;
+
+  tor_assert(url);
+  tor_assert(prefix);
+  tor_assert(end_pos);
+
+  /* Check if the prefix does start the url. */
+  if (strcmpstart(url, prefix)) {
+    goto err;
+  }
+  /* Move pointer to the end of the prefix string. */
+  start = url + strlen(prefix);
+  /* Try this to be the HS version and if we are still at the separator, next
+   * will be move to the right value. */
+  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+  if (!ok) {
+    goto err;
+  }
+
+  *end_pos = end;
+  return (int) version;
+ err:
+  *end_pos = NULL;
+  return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+  int version;
+  const char *end_pos;
+
+  tor_assert(url);
+  tor_assert(body);
+
+  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+  if (version < 0) {
+    goto err;
+  }
+
+  /* We have a valid version number, now make sure it's a publish request. Use
+   * the end position just after the version and check for the command. */
+  if (strcmpstart(end_pos, "/publish")) {
+    goto err;
+  }
+
+  switch (version) {
+  case HS_VERSION_THREE:
+    if (hs_cache_store_as_dir(body) < 0) {
+      goto err;
+    }
+    log_info(LD_REND, "Publish request for HS descriptor handled "
+                      "successfully.");
+    break;
+  default:
+    /* Unsupported version, return a bad request. */
+    goto err;
+  }
+
+  return 200;
+ err:
+  /* Bad request. */
+  return 400;
+}
+
+/** Helper function: called when a dirserver gets a complete HTTP POST
+ * request.  Look for an uploaded server descriptor or rendezvous
+ * service descriptor.  On finding one, process it and write a
+ * response into conn-\>outbuf.  If the request is unrecognized, send a
+ * 400.  Always return 0. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+                               const char *body, size_t body_len))
+{
+  char *url = NULL;
+  const or_options_t *options = get_options();
+
+  log_debug(LD_DIRSERV,"Received POST command.");
+
+  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+  if (!public_server_mode(options)) {
+    log_info(LD_DIR, "Rejected dir post request from %s "
+             "since we're not a public relay.", conn->base_.address);
+    write_short_http_response(conn, 503, "Not acting as a public relay");
+    goto done;
+  }
+
+  if (parse_http_url(headers, &url) < 0) {
+    write_short_http_response(conn, 400, "Bad request");
+    return 0;
+  }
+  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+  /* Handle v2 rendezvous service publish request. */
+  if (connection_dir_is_encrypted(conn) &&
+      !strcmpstart(url,"/tor/rendezvous2/publish")) {
+    if (rend_cache_store_v2_desc_as_dir(body) < 0) {
+      log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
+               (int)body_len, conn->base_.address);
+      write_short_http_response(conn, 400,
+                             "Invalid v2 service descriptor rejected");
+    } else {
+      write_short_http_response(conn, 200, "Service descriptor (v2) stored");
+      log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
+    }
+    goto done;
+  }
+
+  /* Handle HS descriptor publish request. */
+  /* XXX: This should be disabled with a consensus param until we want to
+   * the prop224 be deployed and thus use. */
+  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+    const char *msg = "HS descriptor stored successfully.";
+
+    /* We most probably have a publish request for an HS descriptor. */
+    int code = handle_post_hs_descriptor(url, body);
+    if (code != 200) {
+      msg = "Invalid HS descriptor. Rejected.";
+    }
+    write_short_http_response(conn, code, msg);
+    goto done;
+  }
+
+  if (!authdir_mode(options)) {
+    /* we just provide cached directories; we don't want to
+     * receive anything. */
+    write_short_http_response(conn, 400, "Nonauthoritative directory does not "
+                           "accept posted server descriptors");
+    goto done;
+  }
+
+  if (authdir_mode(options) &&
+      !strcmp(url,"/tor/")) { /* server descriptor post */
+    const char *msg = "[None]";
+    uint8_t purpose = authdir_mode_bridge(options) ?
+                      ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
+                                             conn->base_.address, &msg);
+    tor_assert(msg);
+
+    if (r == ROUTER_ADDED_SUCCESSFULLY) {
+      write_short_http_response(conn, 200, msg);
+    } else if (WRA_WAS_OUTDATED(r)) {
+      write_http_response_header_impl(conn, -1, NULL, NULL,
+                                      "X-Descriptor-Not-New: Yes\r\n", -1);
+    } else {
+      log_info(LD_DIRSERV,
+               "Rejected router descriptor or extra-info from %s "
+               "(\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, 400, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
+    const char *msg = "OK";
+    int status;
+    if (dirvote_add_vote(body, &msg, &status)) {
+      write_short_http_response(conn, status, "Vote stored");
+    } else {
+      tor_assert(msg);
+      log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
+               conn->base_.address, msg);
+      write_short_http_response(conn, status, msg);
+    }
+    goto done;
+  }
+
+  if (authdir_mode_v3(options) &&
+      !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
+    const char *msg = NULL;
+    if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
+      write_short_http_response(conn, 200, msg?msg:"Signatures stored");
+    } else {
+      log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
+               conn->base_.address, msg?msg:"???");
+      write_short_http_response(conn, 400,
+                                msg?msg:"Unable to store signatures");
+    }
+    goto done;
+  }
+
+  /* we didn't recognize the url */
+  write_short_http_response(conn, 404, "Not found");
+
+ done:
+  tor_free(url);
+  return 0;
+}
+
+/** If <b>headers</b> indicates that a proxy was involved, then rewrite
+ * <b>conn</b>-\>address to describe our best guess of the address that
+ * originated this HTTP request. */
+static void
+http_set_address_origin(const char *headers, connection_t *conn)
+{
+  char *fwd;
+
+  fwd = http_get_header(headers, "Forwarded-For: ");
+  if (!fwd)
+    fwd = http_get_header(headers, "X-Forwarded-For: ");
+  if (fwd) {
+    tor_addr_t toraddr;
+    if (tor_addr_parse(&toraddr,fwd) == -1 ||
+        tor_addr_is_internal(&toraddr,0)) {
+      log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
+      tor_free(fwd);
+      return;
+    }
+
+    tor_free(conn->address);
+    conn->address = tor_strdup(fwd);
+    tor_free(fwd);
+  }
+}
+
+/** Called when a dirserver receives data on a directory connection;
+ * looks for an HTTP request.  If the request is complete, remove it
+ * from the inbuf, try to process it; otherwise, leave it on the
+ * buffer.  Return a 0 on success, or -1 on error.
+ */
+int
+directory_handle_command(dir_connection_t *conn)
+{
+  char *headers=NULL, *body=NULL;
+  size_t body_len=0;
+  int r;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  switch (connection_fetch_from_buf_http(TO_CONN(conn),
+                              &headers, MAX_HEADERS_SIZE,
+                              &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
+    case -1: /* overflow */
+      log_warn(LD_DIRSERV,
+               "Request too large from address '%s' to DirPort. Closing.",
+               safe_str(conn->base_.address));
+      return -1;
+    case 0:
+      log_debug(LD_DIRSERV,"command not all here yet.");
+      return 0;
+    /* case 1, fall through */
+  }
+
+  http_set_address_origin(headers, TO_CONN(conn));
+  // we should escape headers here as well,
+  // but we can't call escaped() twice, as it uses the same buffer
+  //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
+
+  if (!strncasecmp(headers,"GET",3))
+    r = directory_handle_command_get(conn, headers, body, body_len);
+  else if (!strncasecmp(headers,"POST",4))
+    r = directory_handle_command_post(conn, headers, body, body_len);
+  else {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Got headers %s with unknown command. Closing.",
+           escaped(headers));
+    r = -1;
+  }
+
+  tor_free(headers); tor_free(body);
+  return r;
+}

+ 43 - 0
src/feature/dircache/dircache.h

@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dircache.h
+ * \brief Header file for dircache.c.
+ **/
+
+#ifndef TOR_DIRCACHE_H
+#define TOR_DIRCACHE_H
+
+int directory_handle_command(dir_connection_t *conn);
+
+#ifdef DIRCACHE_PRIVATE
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+                                                    const char *headers,
+                                                    const char *req_body,
+                                                    size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+                                                     const char *headers,
+                                                     const char *body,
+                                                     size_t body_len));
+
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+enum compression_level_t;
+STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                                       const struct get_handler_args_t *args);
+
+STATIC int parse_http_url(const char *headers, char **url);
+
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+                                      const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
+#endif
+
+#endif /* !defined(TOR_DIRCACHE_H) */

+ 0 - 347
src/feature/dircache/directory.h

@@ -1,347 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file directory.h
- * \brief Header file for directory.c.
- **/
-
-#ifndef TOR_DIRECTORY_H
-#define TOR_DIRECTORY_H
-
-#include "feature/hs/hs_ident.h"
-enum compress_method_t;
-
-dir_connection_t *TO_DIR_CONN(connection_t *c);
-
-#define DIR_CONN_STATE_MIN_ 1
-/** State for connection to directory server: waiting for connect(). */
-#define DIR_CONN_STATE_CONNECTING 1
-/** State for connection to directory server: sending HTTP request. */
-#define DIR_CONN_STATE_CLIENT_SENDING 2
-/** State for connection to directory server: reading HTTP response. */
-#define DIR_CONN_STATE_CLIENT_READING 3
-/** State for connection to directory server: happy and finished. */
-#define DIR_CONN_STATE_CLIENT_FINISHED 4
-/** State for connection at directory server: waiting for HTTP request. */
-#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
-/** State for connection at directory server: sending HTTP response. */
-#define DIR_CONN_STATE_SERVER_WRITING 6
-#define DIR_CONN_STATE_MAX_ 6
-
-#define DIR_PURPOSE_MIN_ 4
-/** A connection to a directory server: set after a v2 rendezvous
- * descriptor is downloaded. */
-#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
-/** A connection to a directory server: download one or more server
- * descriptors. */
-#define DIR_PURPOSE_FETCH_SERVERDESC 6
-/** A connection to a directory server: download one or more extra-info
- * documents. */
-#define DIR_PURPOSE_FETCH_EXTRAINFO 7
-/** A connection to a directory server: upload a server descriptor. */
-#define DIR_PURPOSE_UPLOAD_DIR 8
-/** A connection to a directory server: upload a v3 networkstatus vote. */
-#define DIR_PURPOSE_UPLOAD_VOTE 10
-/** A connection to a directory server: upload a v3 consensus signature */
-#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
-/** A connection to a directory server: download one or more v3 networkstatus
- * votes. */
-#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
-/** A connection to a directory server: download a v3 detached signatures
- * object for a consensus. */
-#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
-/** A connection to a directory server: download a v3 networkstatus
- * consensus. */
-#define DIR_PURPOSE_FETCH_CONSENSUS 14
-/** A connection to a directory server: download one or more directory
- * authority certificates. */
-#define DIR_PURPOSE_FETCH_CERTIFICATE 15
-
-/** Purpose for connection at a directory server. */
-#define DIR_PURPOSE_SERVER 16
-/** A connection to a hidden service directory server: upload a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
-/** A connection to a hidden service directory server: download a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
-/** A connection to a directory server: download a microdescriptor. */
-#define DIR_PURPOSE_FETCH_MICRODESC 19
-/** A connection to a hidden service directory: upload a v3 descriptor. */
-#define DIR_PURPOSE_UPLOAD_HSDESC 20
-/** A connection to a hidden service directory: fetch a v3 descriptor. */
-#define DIR_PURPOSE_FETCH_HSDESC 21
-/** A connection to a directory server: set after a hidden service descriptor
- * is downloaded. */
-#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
-#define DIR_PURPOSE_MAX_ 22
-
-/** True iff <b>p</b> is a purpose corresponding to uploading
- * data to a directory server. */
-#define DIR_PURPOSE_IS_UPLOAD(p)                \
-  ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
-   (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
-   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES ||        \
-   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||       \
-   (p)==DIR_PURPOSE_UPLOAD_HSDESC)
-
-int directories_have_accepted_server_descriptor(void);
-void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
-                                  dirinfo_type_t type, const char *payload,
-                                  size_t payload_len, size_t extrainfo_len);
-MOCK_DECL(void, directory_get_from_dirserver, (
-                          uint8_t dir_purpose,
-                          uint8_t router_purpose,
-                          const char *resource,
-                          int pds_flags,
-                          download_want_authority_t want_authority));
-void directory_get_from_all_authorities(uint8_t dir_purpose,
-                                        uint8_t router_purpose,
-                                        const char *resource);
-
-/** Enumeration of ways to connect to a directory server */
-typedef enum {
-  /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
-   * DirPort connections, clients, onion services, and bridges do not */
-  DIRIND_ONEHOP=0,
-  /** Connect over a multi-hop anonymizing Tor circuit */
-  DIRIND_ANONYMOUS=1,
-  /** Connect to the DirPort directly */
-  DIRIND_DIRECT_CONN,
-  /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
-  DIRIND_ANON_DIRPORT,
-} dir_indirection_t;
-
-int directory_must_use_begindir(const or_options_t *options);
-
-/**
- * A directory_request_t describes the information about a directory request
- * at the client side.  It describes what we're going to ask for, which
- * directory we're going to ask for it, how we're going to contact that
- * directory, and (in some cases) what to do with it when we're done.
- */
-typedef struct directory_request_t directory_request_t;
-directory_request_t *directory_request_new(uint8_t dir_purpose);
-void directory_request_free_(directory_request_t *req);
-#define directory_request_free(req) \
-  FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
-void directory_request_set_or_addr_port(directory_request_t *req,
-                                        const tor_addr_port_t *p);
-void directory_request_set_dir_addr_port(directory_request_t *req,
-                                         const tor_addr_port_t *p);
-void directory_request_set_directory_id_digest(directory_request_t *req,
-                                               const char *digest);
-struct circuit_guard_state_t;
-void directory_request_set_guard_state(directory_request_t *req,
-                                       struct circuit_guard_state_t *state);
-void directory_request_set_router_purpose(directory_request_t *req,
-                                          uint8_t router_purpose);
-void directory_request_set_indirection(directory_request_t *req,
-                                       dir_indirection_t indirection);
-void directory_request_set_resource(directory_request_t *req,
-                                    const char *resource);
-void directory_request_set_payload(directory_request_t *req,
-                                   const char *payload,
-                                   size_t payload_len);
-void directory_request_set_if_modified_since(directory_request_t *req,
-                                             time_t if_modified_since);
-void directory_request_set_rend_query(directory_request_t *req,
-                                      const rend_data_t *query);
-void directory_request_upload_set_hs_ident(directory_request_t *req,
-                                           const hs_ident_dir_conn_t *ident);
-void directory_request_fetch_set_hs_ident(directory_request_t *req,
-                                          const hs_ident_dir_conn_t *ident);
-
-void directory_request_set_routerstatus(directory_request_t *req,
-                                        const routerstatus_t *rs);
-void directory_request_add_header(directory_request_t *req,
-                                  const char *key,
-                                  const char *val);
-MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
-
-int parse_http_response(const char *headers, int *code, time_t *date,
-                        enum compress_method_t *compression, char **response);
-int parse_http_command(const char *headers,
-                       char **command_out, char **url_out);
-char *http_get_header(const char *headers, const char *which);
-
-int connection_dir_is_encrypted(const dir_connection_t *conn);
-int connection_dir_reached_eof(dir_connection_t *conn);
-int connection_dir_process_inbuf(dir_connection_t *conn);
-int connection_dir_finished_flushing(dir_connection_t *conn);
-int connection_dir_finished_connecting(dir_connection_t *conn);
-void connection_dir_about_to_close(dir_connection_t *dir_conn);
-
-#define DSR_HEX       (1<<0)
-#define DSR_BASE64    (1<<1)
-#define DSR_DIGEST256 (1<<2)
-#define DSR_SORT_UNIQ (1<<3)
-int dir_split_resource_into_fingerprints(const char *resource,
-                                     smartlist_t *fp_out, int *compressed_out,
-                                     int flags);
-enum dir_spool_source_t;
-int dir_split_resource_into_spoolable(const char *resource,
-                                      enum dir_spool_source_t source,
-                                      smartlist_t *spool_out,
-                                      int *compressed_out,
-                                      int flags);
-int dir_split_resource_into_fingerprint_pairs(const char *res,
-                                              smartlist_t *pairs_out);
-char *directory_dump_request_log(void);
-void note_request(const char *key, size_t bytes);
-int router_supports_extrainfo(const char *identity_digest, int is_authority);
-
-time_t download_status_increment_failure(download_status_t *dls,
-                                         int status_code, const char *item,
-                                         int server, time_t now);
-time_t download_status_increment_attempt(download_status_t *dls,
-                                         const char *item,  time_t now);
-/** Increment the failure count of the download_status_t <b>dls</b>, with
- * the optional status code <b>sc</b>. */
-#define download_status_failed(dls, sc)                                 \
-  download_status_increment_failure((dls), (sc), NULL,                  \
-                                    dir_server_mode(get_options()), \
-                                    time(NULL))
-
-void download_status_reset(download_status_t *dls);
-int download_status_is_ready(download_status_t *dls, time_t now);
-time_t download_status_get_next_attempt_at(const download_status_t *dls);
-void download_status_mark_impossible(download_status_t *dl);
-
-int download_status_get_n_failures(const download_status_t *dls);
-int download_status_get_n_attempts(const download_status_t *dls);
-
-int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
-                            const char *resource);
-
-#ifdef DIRECTORY_PRIVATE
-
-/** A structure to hold arguments passed into each directory response
- * handler */
-typedef struct response_handler_args_t {
-  int status_code;
-  const char *reason;
-  const char *body;
-  size_t body_len;
-  const char *headers;
-} response_handler_args_t;
-
-struct directory_request_t {
-  /**
-   * These fields specify which directory we're contacting.  Routerstatus,
-   * if present, overrides the other fields.
-   *
-   * @{ */
-  tor_addr_port_t or_addr_port;
-  tor_addr_port_t dir_addr_port;
-  char digest[DIGEST_LEN];
-
-  const routerstatus_t *routerstatus;
-  /** @} */
-  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
-   * kind of operation we'll be doing (upload/download), and of what kind
-   * of document. */
-  uint8_t dir_purpose;
-  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
-   * and extrainfo docs.  */
-  uint8_t router_purpose;
-  /** Enum: determines whether to anonymize, and whether to use dirport or
-   * orport. */
-  dir_indirection_t indirection;
-  /** Alias to the variable part of the URL for this request */
-  const char *resource;
-  /** Alias to the payload to upload (if any) */
-  const char *payload;
-  /** Number of bytes to upload from payload</b> */
-  size_t payload_len;
-  /** Value to send in an if-modified-since header, or 0 for none. */
-  time_t if_modified_since;
-  /** Hidden-service-specific information v2. */
-  const rend_data_t *rend_query;
-  /** Extra headers to append to the request */
-  struct config_line_t *additional_headers;
-  /** Hidden-service-specific information for v3+. */
-  const hs_ident_dir_conn_t *hs_ident;
-  /** Used internally to directory.c: gets informed when the attempt to
-   * connect to the directory succeeds or fails, if that attempt bears on the
-   * directory's usability as a directory guard. */
-  struct circuit_guard_state_t *guard_state;
-};
-
-struct get_handler_args_t;
-STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
-                                       const struct get_handler_args_t *args);
-STATIC int directory_handle_command(dir_connection_t *conn);
-STATIC char *accept_encoding_header(void);
-STATIC int allowed_anonymous_connection_compression_method(
-                                               enum compress_method_t);
-STATIC void warn_disallowed_anonymous_compression_method(
-                                               enum compress_method_t);
-
-STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
-                                          const response_handler_args_t *args);
-STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
-                                 const response_handler_args_t *args);
-
-STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
-                                         const response_handler_args_t *args);
-
-#endif /* defined(DIRECTORY_PRIVATE) */
-
-#ifdef TOR_UNIT_TESTS
-/* Used only by test_dir.c and test_hs_cache.c */
-
-STATIC int parse_http_url(const char *headers, char **url);
-STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
-                                     const char *resource);
-MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
-                                                    const char *headers,
-                                                    const char *req_body,
-                                                    size_t req_body_len));
-MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
-                                                     const char *headers,
-                                                     const char *body,
-                                                     size_t body_len));
-STATIC int download_status_schedule_get_delay(download_status_t *dls,
-                                              int min_delay,
-                                              time_t now);
-
-STATIC int handle_post_hs_descriptor(const char *url, const char *body);
-
-STATIC char* authdir_type_to_string(dirinfo_type_t auth);
-STATIC const char * dir_conn_purpose_to_string(int purpose);
-STATIC int should_use_directory_guards(const or_options_t *options);
-enum compression_level_t;
-STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
-STATIC int find_dl_min_delay(const download_status_t *dls,
-                             const or_options_t *options);
-
-STATIC int next_random_exponential_delay(int delay,
-                                         int base_delay);
-
-STATIC void next_random_exponential_delay_range(int *low_bound_out,
-                                                int *high_bound_out,
-                                                int delay,
-                                                int base_delay);
-
-STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
-                                      const char **end_pos);
-
-STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-#if defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE)
-/* Used only by directory.c and test_dir.c */
-
-/* no more than quadruple the previous delay (multiplier + 1) */
-#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
-/* no more than triple the previous delay */
-#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
-
-#endif /* defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) */
-
-#endif /* !defined(TOR_DIRECTORY_H) */

+ 1 - 1
src/feature/dircache/dirserv.c

@@ -10,7 +10,7 @@
 #include "core/mainloop/connection.h"
 #include "feature/dircache/conscache.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/routerlist.h"

File diff suppressed because it is too large
+ 96 - 2255
src/feature/dirclient/dirclient.c


+ 172 - 0
src/feature/dirclient/dirclient.h

@@ -0,0 +1,172 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirclient.h
+ * \brief Header file for dirclient.c.
+ **/
+
+#ifndef TOR_DIRCLIENT_H
+#define TOR_DIRCLIENT_H
+
+#include "feature/hs/hs_ident.h"
+
+int directories_have_accepted_server_descriptor(void);
+void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+                                  dirinfo_type_t type, const char *payload,
+                                  size_t payload_len, size_t extrainfo_len);
+MOCK_DECL(void, directory_get_from_dirserver, (
+                          uint8_t dir_purpose,
+                          uint8_t router_purpose,
+                          const char *resource,
+                          int pds_flags,
+                          download_want_authority_t want_authority));
+void directory_get_from_all_authorities(uint8_t dir_purpose,
+                                        uint8_t router_purpose,
+                                        const char *resource);
+
+/** Enumeration of ways to connect to a directory server */
+typedef enum {
+  /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
+   * DirPort connections, clients, onion services, and bridges do not */
+  DIRIND_ONEHOP=0,
+  /** Connect over a multi-hop anonymizing Tor circuit */
+  DIRIND_ANONYMOUS=1,
+  /** Connect to the DirPort directly */
+  DIRIND_DIRECT_CONN,
+  /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
+  DIRIND_ANON_DIRPORT,
+} dir_indirection_t;
+
+int directory_must_use_begindir(const or_options_t *options);
+
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side.  It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free_(directory_request_t *req);
+#define directory_request_free(req) \
+  FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
+void directory_request_set_or_addr_port(directory_request_t *req,
+                                        const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+                                         const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+                                               const char *digest);
+struct circuit_guard_state_t;
+void directory_request_set_guard_state(directory_request_t *req,
+                                       struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+                                          uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+                                       dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+                                    const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+                                   const char *payload,
+                                   size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+                                             time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+                                      const rend_data_t *query);
+void directory_request_upload_set_hs_ident(directory_request_t *req,
+                                           const hs_ident_dir_conn_t *ident);
+void directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                          const hs_ident_dir_conn_t *ident);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+                                        const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+                                  const char *key,
+                                  const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
+
+int router_supports_extrainfo(const char *identity_digest, int is_authority);
+
+void connection_dir_client_request_failed(dir_connection_t *conn);
+void connection_dir_client_refetch_hsdesc_if_needed(
+                                          dir_connection_t *dir_conn);
+
+#ifdef DIRCLIENT_PRIVATE
+struct directory_request_t {
+  /**
+   * These fields specify which directory we're contacting.  Routerstatus,
+   * if present, overrides the other fields.
+   *
+   * @{ */
+  tor_addr_port_t or_addr_port;
+  tor_addr_port_t dir_addr_port;
+  char digest[DIGEST_LEN];
+
+  const routerstatus_t *routerstatus;
+  /** @} */
+  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+   * kind of operation we'll be doing (upload/download), and of what kind
+   * of document. */
+  uint8_t dir_purpose;
+  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+   * and extrainfo docs.  */
+  uint8_t router_purpose;
+  /** Enum: determines whether to anonymize, and whether to use dirport or
+   * orport. */
+  dir_indirection_t indirection;
+  /** Alias to the variable part of the URL for this request */
+  const char *resource;
+  /** Alias to the payload to upload (if any) */
+  const char *payload;
+  /** Number of bytes to upload from payload</b> */
+  size_t payload_len;
+  /** Value to send in an if-modified-since header, or 0 for none. */
+  time_t if_modified_since;
+  /** Hidden-service-specific information v2. */
+  const rend_data_t *rend_query;
+  /** Extra headers to append to the request */
+  struct config_line_t *additional_headers;
+  /** Hidden-service-specific information for v3+. */
+  const hs_ident_dir_conn_t *hs_ident;
+  /** Used internally to directory.c: gets informed when the attempt to
+   * connect to the directory succeeds or fails, if that attempt bears on the
+   * directory's usability as a directory guard. */
+  struct circuit_guard_state_t *guard_state;
+};
+
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+  int status_code;
+  const char *reason;
+  const char *body;
+  size_t body_len;
+  const char *headers;
+} response_handler_args_t;
+
+enum compress_method_t;
+STATIC int allowed_anonymous_connection_compression_method(
+                                               enum compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(
+                                               enum compress_method_t);
+
+STATIC int should_use_directory_guards(const or_options_t *options);
+STATIC char *accept_encoding_header(void);
+STATIC const char *dir_conn_purpose_to_string(int purpose);
+
+STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                          const response_handler_args_t *args);
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+                                 const response_handler_args_t *args);
+
+STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
+                                         const response_handler_args_t *args);
+
+STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
+                                     const char *resource);
+#endif
+
+#endif /* !defined(TOR_DIRCLIENT_H) */

+ 422 - 0
src/feature/dirclient/dlstatus.c

@@ -0,0 +1,422 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DLSTATUS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/dirclient/download_status_st.h"
+
+/** Decide which download schedule we want to use based on descriptor type
+ * in <b>dls</b> and <b>options</b>.
+ *
+ * Then, return the initial delay for that download schedule, in seconds.
+ *
+ * Helper function for download_status_increment_failure(),
+ * download_status_reset(), and download_status_increment_attempt(). */
+STATIC int
+find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
+{
+  tor_assert(dls);
+  tor_assert(options);
+
+  switch (dls->schedule) {
+    case DL_SCHED_GENERIC:
+      /* Any other directory document */
+      if (dir_server_mode(options)) {
+        /* A directory authority or directory mirror */
+        return options->TestingServerDownloadInitialDelay;
+      } else {
+        return options->TestingClientDownloadInitialDelay;
+      }
+    case DL_SCHED_CONSENSUS:
+      if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+        /* A public relay */
+        return options->TestingServerConsensusDownloadInitialDelay;
+      } else {
+        /* A client or bridge */
+        if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
+          /* During bootstrapping */
+          if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
+            /* A bootstrapping client without extra fallback directories */
+            return options->
+              ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
+          } else if (dls->want_authority) {
+            /* A bootstrapping client with extra fallback directories, but
+             * connecting to an authority */
+            return
+             options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
+          } else {
+            /* A bootstrapping client connecting to extra fallback directories
+             */
+            return
+              options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
+          }
+        } else {
+          /* A client with a reasonably live consensus, with or without
+           * certificates */
+          return options->TestingClientConsensusDownloadInitialDelay;
+        }
+      }
+    case DL_SCHED_BRIDGE:
+      if (options->UseBridges && num_bridges_usable(0) > 0) {
+        /* A bridge client that is sure that one or more of its bridges are
+         * running can afford to wait longer to update bridge descriptors. */
+        return options->TestingBridgeDownloadInitialDelay;
+      } else {
+        /* A bridge client which might have no running bridges, must try to
+         * get bridge descriptors straight away. */
+        return options->TestingBridgeBootstrapDownloadInitialDelay;
+      }
+    default:
+      tor_assert(0);
+  }
+
+  /* Impossible, but gcc will fail with -Werror without a `return`. */
+  return 0;
+}
+
+/** As next_random_exponential_delay() below, but does not compute a random
+ * value. Instead, compute the range of values that
+ * next_random_exponential_delay() should use when computing its random value.
+ * Store the low bound into *<b>low_bound_out</b>, and the high bound into
+ * *<b>high_bound_out</b>.  Guarantees that the low bound is strictly less
+ * than the high bound. */
+STATIC void
+next_random_exponential_delay_range(int *low_bound_out,
+                                    int *high_bound_out,
+                                    int delay,
+                                    int base_delay)
+{
+  // This is the "decorrelated jitter" approach, from
+  //    https://www.awsarchitectureblog.com/2015/03/backoff.html
+  // The formula is
+  //    sleep = min(cap, random_between(base, sleep * 3))
+
+  const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
+  *low_bound_out = base_delay;
+  if (delay_times_3 > base_delay) {
+    *high_bound_out = delay_times_3;
+  } else {
+    *high_bound_out = base_delay+1;
+  }
+}
+
+/** Advance one delay step.  The algorithm will generate a random delay,
+ * such that each failure is possibly (random) longer than the ones before.
+ *
+ * We then clamp that value to be no larger than max_delay, and return it.
+ *
+ * The <b>base_delay</b> parameter is lowest possible delay time (can't be
+ * zero); the <b>backoff_position</b> parameter is the number of times we've
+ * generated a delay; and the <b>delay</b> argument is the most recently used
+ * delay.
+ */
+STATIC int
+next_random_exponential_delay(int delay,
+                              int base_delay)
+{
+  /* Check preconditions */
+  if (BUG(delay < 0))
+    delay = 0;
+
+  if (base_delay < 1)
+    base_delay = 1;
+
+  int low_bound=0, high_bound=INT_MAX;
+
+  next_random_exponential_delay_range(&low_bound, &high_bound,
+                                      delay, base_delay);
+
+  return crypto_rand_int_range(low_bound, high_bound);
+}
+
+/** Find the current delay for dls based on min_delay.
+ *
+ * This function sets dls->next_attempt_at based on now, and returns the delay.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+STATIC int
+download_status_schedule_get_delay(download_status_t *dls,
+                                   int min_delay,
+                                   time_t now)
+{
+  tor_assert(dls);
+  /* If we're using random exponential backoff, we do need min/max delay */
+  tor_assert(min_delay >= 0);
+
+  int delay = INT_MAX;
+  uint8_t dls_schedule_position = (dls->increment_on
+                                   == DL_SCHED_INCREMENT_ATTEMPT
+                                   ? dls->n_download_attempts
+                                   : dls->n_download_failures);
+
+  /* Check if we missed a reset somehow */
+  IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
+    dls->last_backoff_position = 0;
+    dls->last_delay_used = 0;
+  }
+
+  if (dls_schedule_position > 0) {
+    delay = dls->last_delay_used;
+
+    while (dls->last_backoff_position < dls_schedule_position) {
+      /* Do one increment step */
+      delay = next_random_exponential_delay(delay, min_delay);
+      /* Update our position */
+      ++(dls->last_backoff_position);
+    }
+  } else {
+    /* If we're just starting out, use the minimum delay */
+    delay = min_delay;
+  }
+
+  /* Clamp it within min/max if we have them */
+  if (min_delay >= 0 && delay < min_delay) delay = min_delay;
+
+  /* Store it for next time */
+  dls->last_backoff_position = dls_schedule_position;
+  dls->last_delay_used = delay;
+
+  /* A negative delay makes no sense. Knowing that delay is
+   * non-negative allows us to safely do the wrapping check below. */
+  tor_assert(delay >= 0);
+
+  /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
+   * that won't overflow (since delay is non-negative). */
+  if (delay < INT_MAX && now <= TIME_MAX - delay) {
+    dls->next_attempt_at = now+delay;
+  } else {
+    dls->next_attempt_at = TIME_MAX;
+  }
+
+  return delay;
+}
+
+/* Log a debug message about item, which increments on increment_action, has
+ * incremented dls_n_download_increments times. The message varies based on
+ * was_schedule_incremented (if not, not_incremented_response is logged), and
+ * the values of increment, dls_next_attempt_at, and now.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+static void
+download_status_log_helper(const char *item, int was_schedule_incremented,
+                           const char *increment_action,
+                           const char *not_incremented_response,
+                           uint8_t dls_n_download_increments, int increment,
+                           time_t dls_next_attempt_at, time_t now)
+{
+  if (item) {
+    if (!was_schedule_incremented)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
+                item, increment_action, (int)dls_n_download_increments,
+                not_incremented_response);
+    else if (increment == 0)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
+                item, increment_action, (int)dls_n_download_increments);
+    else if (dls_next_attempt_at < TIME_MAX)
+      log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
+                item, increment_action, (int)dls_n_download_increments,
+                (int)(dls_next_attempt_at-now));
+    else
+      log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
+                item, increment_action, (int)dls_n_download_increments);
+  }
+}
+
+/** Determine when a failed download attempt should be retried.
+ * Called when an attempt to download <b>dls</b> has failed with HTTP status
+ * <b>status_code</b>.  Increment the failure count (if the code indicates a
+ * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
+ * an appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
+ * failure count, and return a time in the far future for the next attempt (to
+ * avoid an immediate retry). */
+time_t
+download_status_increment_failure(download_status_t *dls, int status_code,
+                                  const char *item, int server, time_t now)
+{
+  (void) status_code; // XXXX no longer used.
+  (void) server; // XXXX no longer used.
+  int increment = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  /* count the failure */
+  if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
+    ++dls->n_download_failures;
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* We don't find out that a failure-based schedule has attempted a
+     * connection until that connection fails.
+     * We'll never find out about successful connections, but this doesn't
+     * matter, because schedules are reset after a successful download.
+     */
+    if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+      ++dls->n_download_attempts;
+
+    /* only return a failure retry time if this schedule increments on failures
+     */
+    min_delay = find_dl_min_delay(dls, get_options());
+    increment = download_status_schedule_get_delay(dls, min_delay, now);
+  }
+
+  download_status_log_helper(item, !dls->increment_on, "failed",
+                             "concurrently", dls->n_download_failures,
+                             increment,
+                             download_status_get_next_attempt_at(dls),
+                             now);
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
+    /* stop this schedule retrying on failure, it will launch concurrent
+     * connections instead */
+    return TIME_MAX;
+  } else {
+    return download_status_get_next_attempt_at(dls);
+  }
+}
+
+/** Determine when the next download attempt should be made when using an
+ * attempt-based (potentially concurrent) download schedule.
+ * Called when an attempt to download <b>dls</b> is being initiated.
+ * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
+ * appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
+ * the attempts, and return a time in the far future (to avoid launching a
+ * concurrent attempt). */
+time_t
+download_status_increment_attempt(download_status_t *dls, const char *item,
+                                  time_t now)
+{
+  int delay = -1;
+  int min_delay = 0;
+
+  tor_assert(dls);
+
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+    /* this schedule should retry on failure, and not launch any concurrent
+     attempts */
+    log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
+             "failure-based schedule.");
+    return TIME_MAX;
+  }
+
+  if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+    ++dls->n_download_attempts;
+
+  min_delay = find_dl_min_delay(dls, get_options());
+  delay = download_status_schedule_get_delay(dls, min_delay, now);
+
+  download_status_log_helper(item, dls->increment_on, "attempted",
+                             "on failure", dls->n_download_attempts,
+                             delay, download_status_get_next_attempt_at(dls),
+                             now);
+
+  return download_status_get_next_attempt_at(dls);
+}
+
+static time_t
+download_status_get_initial_delay_from_now(const download_status_t *dls)
+{
+  /* We use constant initial delays, even in exponential backoff
+   * schedules. */
+  return time(NULL) + find_dl_min_delay(dls, get_options());
+}
+
+/** Reset <b>dls</b> so that it will be considered downloadable
+ * immediately, and/or to show that we don't need it anymore.
+ *
+ * Must be called to initialise a download schedule, otherwise the zeroth item
+ * in the schedule will never be used.
+ *
+ * (We find the zeroth element of the download schedule, and set
+ * next_attempt_at to be the appropriate offset from 'now'. In most
+ * cases this means setting it to 'now', so the item will be immediately
+ * downloadable; when using authorities with fallbacks, there is a few seconds'
+ * delay.) */
+void
+download_status_reset(download_status_t *dls)
+{
+  if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
+      || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
+    return; /* Don't reset this. */
+
+  dls->n_download_failures = 0;
+  dls->n_download_attempts = 0;
+  dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
+  dls->last_backoff_position = 0;
+  dls->last_delay_used = 0;
+  /* Don't reset dls->want_authority or dls->increment_on */
+}
+
+/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
+ * ready to get its download reattempted. */
+int
+download_status_is_ready(download_status_t *dls, time_t now)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
+  return download_status_get_next_attempt_at(dls) <= now;
+}
+
+/** Mark <b>dl</b> as never downloadable. */
+void
+download_status_mark_impossible(download_status_t *dl)
+{
+  dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
+  dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
+}
+
+/** Return the number of failures on <b>dls</b> since the last success (if
+ * any). */
+int
+download_status_get_n_failures(const download_status_t *dls)
+{
+  return dls->n_download_failures;
+}
+
+/** Return the number of attempts to download <b>dls</b> since the last success
+ * (if any). This can differ from download_status_get_n_failures() due to
+ * outstanding concurrent attempts. */
+int
+download_status_get_n_attempts(const download_status_t *dls)
+{
+  return dls->n_download_attempts;
+}
+
+/** Return the next time to attempt to download <b>dls</b>. */
+time_t
+download_status_get_next_attempt_at(const download_status_t *dls)
+{
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    /* so give the answer we would have given if it had been */
+    return download_status_get_initial_delay_from_now(dls);
+  }
+
+  return dls->next_attempt_at;
+}

+ 58 - 0
src/feature/dirclient/dlstatus.h

@@ -0,0 +1,58 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dlstatus.h
+ * \brief Header file for dlstatus.c.
+ **/
+
+#ifndef TOR_DLSTATUS_H
+#define TOR_DLSTATUS_H
+
+time_t download_status_increment_failure(download_status_t *dls,
+                                         int status_code, const char *item,
+                                         int server, time_t now);
+time_t download_status_increment_attempt(download_status_t *dls,
+                                         const char *item,  time_t now);
+/** Increment the failure count of the download_status_t <b>dls</b>, with
+ * the optional status code <b>sc</b>. */
+#define download_status_failed(dls, sc)                                 \
+  download_status_increment_failure((dls), (sc), NULL,                  \
+                                    dir_server_mode(get_options()), \
+                                    time(NULL))
+
+void download_status_reset(download_status_t *dls);
+int download_status_is_ready(download_status_t *dls, time_t now);
+time_t download_status_get_next_attempt_at(const download_status_t *dls);
+void download_status_mark_impossible(download_status_t *dl);
+
+int download_status_get_n_failures(const download_status_t *dls);
+int download_status_get_n_attempts(const download_status_t *dls);
+
+#ifdef DLSTATUS_PRIVATE
+STATIC int download_status_schedule_get_delay(download_status_t *dls,
+                                              int min_delay,
+                                              time_t now);
+
+STATIC int find_dl_min_delay(const download_status_t *dls,
+                             const or_options_t *options);
+
+STATIC int next_random_exponential_delay(int delay,
+                                         int base_delay);
+
+STATIC void next_random_exponential_delay_range(int *low_bound_out,
+                                                int *high_bound_out,
+                                                int delay,
+                                                int base_delay);
+
+/* no more than quadruple the previous delay (multiplier + 1) */
+#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
+/* no more than triple the previous delay */
+#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
+
+#endif
+
+#endif /* !defined(TOR_DLSTATUS_H) */

+ 651 - 0
src/feature/dircommon/directory.c

@@ -0,0 +1,651 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/stats/geoip.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/**
+ * \file directory.c
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side.  In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network.  The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response.  The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[].  The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer.  Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c.  (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
+ **/
+
+/* In-points to directory.c:
+ *
+ * - directory_post_to_dirservers(), called from
+ *   router_upload_dir_desc_to_dirservers() in router.c
+ *   upload_service_descriptor() in rendservice.c
+ * - directory_get_from_dirserver(), called from
+ *   rend_client_refetch_renddesc() in rendclient.c
+ *   run_scheduled_events() in main.c
+ *   do_hup() in main.c
+ * - connection_dir_process_inbuf(), called from
+ *   connection_process_inbuf() in connection.c
+ * - connection_dir_finished_flushing(), called from
+ *   connection_finished_flushing() in connection.c
+ * - connection_dir_finished_connecting(), called from
+ *   connection_finished_connecting() in connection.c
+ */
+
+/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
+ * invalid. */
+dir_connection_t *
+TO_DIR_CONN(connection_t *c)
+{
+  tor_assert(c->magic == DIR_CONNECTION_MAGIC);
+  return DOWNCAST(dir_connection_t, c);
+}
+
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
+int
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                        const char *resource)
+{
+  if (get_options()->AllDirActionsPrivate)
+    return 1;
+
+  if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+    if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+        && resource && !strcmp(resource, "authority.z")) {
+      /* We are asking a bridge for its own descriptor. That doesn't need
+         anonymity. */
+      return 0;
+    }
+    /* Assume all other bridge stuff needs anonymity. */
+    return 1; /* if no circuits yet, this might break bootstrapping, but it's
+               * needed to be safe. */
+  }
+
+  switch (dir_purpose)
+  {
+    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_VOTE:
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return 0;
+    case DIR_PURPOSE_HAS_FETCHED_HSDESC:
+    case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_HSDESC:
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return 1;
+    case DIR_PURPOSE_SERVER:
+    default:
+      log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+               dir_purpose, router_purpose);
+      tor_assert_nonfatal_unreached();
+      return 1; /* Assume it needs anonymity; better safe than sorry. */
+  }
+}
+
+/** Return a newly allocated string describing <b>auth</b>. Only describes
+ * authority features. */
+char *
+authdir_type_to_string(dirinfo_type_t auth)
+{
+  char *result;
+  smartlist_t *lst = smartlist_new();
+  if (auth & V3_DIRINFO)
+    smartlist_add(lst, (void*)"V3");
+  if (auth & BRIDGE_DIRINFO)
+    smartlist_add(lst, (void*)"Bridge");
+  if (smartlist_len(lst)) {
+    result = smartlist_join_strings(lst, ", ", 0, NULL);
+  } else {
+    result = tor_strdup("[Not an authority]");
+  }
+  smartlist_free(lst);
+  return result;
+}
+
+/** Return true iff anything we say on <b>conn</b> is being encrypted before
+ * we send it to the client/server. */
+int
+connection_dir_is_encrypted(const dir_connection_t *conn)
+{
+  /* Right now it's sufficient to see if conn is or has been linked, since
+   * the only thing it could be linked to is an edge connection on a
+   * circuit, and the only way it could have been unlinked is at the edge
+   * connection getting closed.
+   */
+  return TO_CONN(conn)->linked;
+}
+
+/** Parse an HTTP request line at the start of a headers string.  On failure,
+ * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
+ * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
+ * return 0. */
+int
+parse_http_command(const char *headers, char **command_out, char **url_out)
+{
+  const char *command, *end_of_command;
+  char *s, *start, *tmp;
+
+  s = (char *)eat_whitespace_no_nl(headers);
+  if (!*s) return -1;
+  command = s;
+  s = (char *)find_whitespace(s); /* get past GET/POST */
+  if (!*s) return -1;
+  end_of_command = s;
+  s = (char *)eat_whitespace_no_nl(s);
+  if (!*s) return -1;
+  start = s; /* this is the URL, assuming it's valid */
+  s = (char *)find_whitespace(start);
+  if (!*s) return -1;
+
+  /* tolerate the http[s] proxy style of putting the hostname in the url */
+  if (s-start >= 4 && !strcmpstart(start,"http")) {
+    tmp = start + 4;
+    if (*tmp == 's')
+      tmp++;
+    if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
+      tmp = strchr(tmp+3, '/');
+      if (tmp && tmp < s) {
+        log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
+        start = tmp;
+      }
+    }
+  }
+
+  /* Check if the header is well formed (next sequence
+   * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
+  {
+    unsigned minor_ver;
+    char ch;
+    char *e = (char *)eat_whitespace_no_nl(s);
+    if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
+      return -1;
+    }
+    if (ch != '\r')
+      return -1;
+  }
+
+  *url_out = tor_memdup_nulterm(start, s-start);
+  *command_out = tor_memdup_nulterm(command, end_of_command - command);
+  return 0;
+}
+
+/** Return a copy of the first HTTP header in <b>headers</b> whose key is
+ * <b>which</b>.  The key should be given with a terminating colon and space;
+ * this function copies everything after, up to but not including the
+ * following \\r\\n. */
+char *
+http_get_header(const char *headers, const char *which)
+{
+  const char *cp = headers;
+  while (cp) {
+    if (!strcasecmpstart(cp, which)) {
+      char *eos;
+      cp += strlen(which);
+      if ((eos = strchr(cp,'\r')))
+        return tor_strndup(cp, eos-cp);
+      else
+        return tor_strdup(cp);
+    }
+    cp = strchr(cp, '\n');
+    if (cp)
+      ++cp;
+  }
+  return NULL;
+}
+/** Parse an HTTP response string <b>headers</b> of the form
+ * \verbatim
+ * "HTTP/1.\%d \%d\%s\r\n...".
+ * \endverbatim
+ *
+ * If it's well-formed, assign the status code to *<b>code</b> and
+ * return 0.  Otherwise, return -1.
+ *
+ * On success: If <b>date</b> is provided, set *date to the Date
+ * header in the http headers, or 0 if no such header is found.  If
+ * <b>compression</b> is provided, set *<b>compression</b> to the
+ * compression method given in the Content-Encoding header, or 0 if no
+ * such header is found, or -1 if the value of the header is not
+ * recognized.  If <b>reason</b> is provided, strdup the reason string
+ * into it.
+ */
+int
+parse_http_response(const char *headers, int *code, time_t *date,
+                    compress_method_t *compression, char **reason)
+{
+  unsigned n1, n2;
+  char datestr[RFC1123_TIME_LEN+1];
+  smartlist_t *parsed_headers;
+  tor_assert(headers);
+  tor_assert(code);
+
+  while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
+
+  if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
+      (n1 != 0 && n1 != 1) ||
+      (n2 < 100 || n2 >= 600)) {
+    log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
+    return -1;
+  }
+  *code = n2;
+
+  parsed_headers = smartlist_new();
+  smartlist_split_string(parsed_headers, headers, "\n",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (reason) {
+    smartlist_t *status_line_elements = smartlist_new();
+    tor_assert(smartlist_len(parsed_headers));
+    smartlist_split_string(status_line_elements,
+                           smartlist_get(parsed_headers, 0),
+                           " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+    tor_assert(smartlist_len(status_line_elements) <= 3);
+    if (smartlist_len(status_line_elements) == 3) {
+      *reason = smartlist_get(status_line_elements, 2);
+      smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
+    }
+    SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
+    smartlist_free(status_line_elements);
+  }
+  if (date) {
+    *date = 0;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Date: ")) {
+        strlcpy(datestr, s+6, sizeof(datestr));
+        /* This will do nothing on failure, so we don't need to check
+           the result.   We shouldn't warn, since there are many other valid
+           date formats besides the one we use. */
+        parse_rfc1123_time(datestr, date);
+        break;
+      });
+  }
+  if (compression) {
+    const char *enc = NULL;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Content-Encoding: ")) {
+        enc = s+18; break;
+      });
+
+    if (enc == NULL)
+      *compression = NO_METHOD;
+    else {
+      *compression = compression_method_get_by_name(enc);
+
+      if (*compression == UNKNOWN_METHOD)
+        log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+                 escaped(enc));
+    }
+  }
+  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+  smartlist_free(parsed_headers);
+
+  return 0;
+}
+
+/** If any directory object is arriving, and it's over 10MB large, we're
+ * getting DoS'd.  (As of 0.1.2.x, raw directories are about 1MB, and we never
+ * ask for more than 96 router descriptors at a time.)
+ */
+#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
+
+#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
+
+/** Read handler for directory connections.  (That's connections <em>to</em>
+ * directory servers and connections <em>at</em> directory servers.)
+ */
+int
+connection_dir_process_inbuf(dir_connection_t *conn)
+{
+  size_t max_size;
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  /* Directory clients write, then read data until they receive EOF;
+   * directory servers read data until they get an HTTP command, then
+   * write their response (when it's finished flushing, they mark for
+   * close).
+   */
+
+  /* If we're on the dirserver side, look for a command. */
+  if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
+    if (directory_handle_command(conn) < 0) {
+      connection_mark_for_close(TO_CONN(conn));
+      return -1;
+    }
+    return 0;
+  }
+
+  max_size =
+    (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
+    MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
+
+  if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
+    log_warn(LD_HTTP,
+             "Too much data received from directory connection (%s): "
+             "denial of service attempt, or you need to upgrade?",
+             conn->base_.address);
+    connection_mark_for_close(TO_CONN(conn));
+    return -1;
+  }
+
+  if (!conn->base_.inbuf_reached_eof)
+    log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
+  return 0;
+}
+
+/** Called when we're about to finally unlink and free a directory connection:
+ * perform necessary accounting and cleanup */
+void
+connection_dir_about_to_close(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
+    /* It's a directory connection and connecting or fetching
+     * failed: forget about this router, and maybe try again. */
+    connection_dir_client_request_failed(dir_conn);
+  }
+
+  connection_dir_client_refetch_hsdesc_if_needed(dir_conn);
+}
+
+/** Write handler for directory connections; called when all data has
+ * been flushed.  Close the connection or wait for a response as
+ * appropriate.
+ */
+int
+connection_dir_finished_flushing(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+  if (conn->base_.marked_for_close)
+    return 0;
+
+  /* Note that we have finished writing the directory response. For direct
+   * connections this means we're done; for tunneled connections it's only
+   * an intermediate step. */
+  if (conn->dirreq_id)
+    geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  else
+    geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
+                              DIRREQ_DIRECT,
+                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+  switch (conn->base_.state) {
+    case DIR_CONN_STATE_CONNECTING:
+    case DIR_CONN_STATE_CLIENT_SENDING:
+      log_debug(LD_DIR,"client finished sending command.");
+      conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
+      return 0;
+    case DIR_CONN_STATE_SERVER_WRITING:
+      if (conn->spool) {
+        log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
+        connection_mark_for_close(TO_CONN(conn));
+      } else {
+        log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
+        connection_mark_for_close(TO_CONN(conn));
+      }
+      return 0;
+    default:
+      log_warn(LD_BUG,"called in unexpected state %d.",
+               conn->base_.state);
+      tor_fragile_assert();
+      return -1;
+  }
+  return 0;
+}
+
+/** Connected handler for directory connections: begin sending data to the
+ * server, and return 0.
+ * Only used when connections don't immediately connect. */
+int
+connection_dir_finished_connecting(dir_connection_t *conn)
+{
+  tor_assert(conn);
+  tor_assert(conn->base_.type == CONN_TYPE_DIR);
+  tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
+
+  log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
+            conn->base_.address,conn->base_.port);
+
+  /* start flushing conn */
+  conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+  return 0;
+}
+
+/** Helper.  Compare two fp_pair_t objects, and return negative, 0, or
+ * positive as appropriate. */
+static int
+compare_pairs_(const void **a, const void **b)
+{
+  const fp_pair_t *fp1 = *a, *fp2 = *b;
+  int r;
+  if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
+    return r;
+  else
+    return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
+}
+
+/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
+ * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
+ * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
+ * fp_pair_t into <b>pairs_out</b>.  On failure, return -1. */
+int
+dir_split_resource_into_fingerprint_pairs(const char *res,
+                                          smartlist_t *pairs_out)
+{
+  smartlist_t *pairs_tmp = smartlist_new();
+  smartlist_t *pairs_result = smartlist_new();
+
+  smartlist_split_string(pairs_tmp, res, "+", 0, 0);
+  if (smartlist_len(pairs_tmp)) {
+    char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+    }
+  }
+  SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
+    if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with non-standard length.", escaped(cp));
+    } else if (cp[HEX_DIGEST_LEN] != '-') {
+      log_info(LD_DIR,
+             "Skipping digest pair %s with missing dash.", escaped(cp));
+    } else {
+      fp_pair_t pair;
+      if (base16_decode(pair.first, DIGEST_LEN,
+                        cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
+          base16_decode(pair.second,DIGEST_LEN,
+                        cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
+        log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
+      } else {
+        smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
+      }
+    }
+    tor_free(cp);
+  } SMARTLIST_FOREACH_END(cp);
+  smartlist_free(pairs_tmp);
+
+  /* Uniq-and-sort */
+  smartlist_sort(pairs_result, compare_pairs_);
+  smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
+
+  smartlist_add_all(pairs_out, pairs_result);
+  smartlist_free(pairs_result);
+  return 0;
+}
+
+/** Given a directory <b>resource</b> request, containing zero
+ * or more strings separated by plus signs, followed optionally by ".z", store
+ * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
+ * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
+ *
+ * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
+ * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
+ * a separator, delete all the elements that aren't base64-encoded digests,
+ * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
+ * 256 bits long; else they should be 160.
+ *
+ * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
+ */
+int
+dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags)
+{
+  const int decode_hex = flags & DSR_HEX;
+  const int decode_base64 = flags & DSR_BASE64;
+  const int digests_are_256 = flags & DSR_DIGEST256;
+  const int sort_uniq = flags & DSR_SORT_UNIQ;
+
+  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
+  const int hex_digest_len = digests_are_256 ?
+    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
+  const int base64_digest_len = digests_are_256 ?
+    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
+  smartlist_t *fp_tmp = smartlist_new();
+
+  tor_assert(!(decode_hex && decode_base64));
+  tor_assert(fp_out);
+
+  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
+  if (compressed_out)
+    *compressed_out = 0;
+  if (smartlist_len(fp_tmp)) {
+    char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
+    size_t last_len = strlen(last);
+    if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+      last[last_len-2] = '\0';
+      if (compressed_out)
+        *compressed_out = 1;
+    }
+  }
+  if (decode_hex || decode_base64) {
+    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
+    int i;
+    char *cp, *d = NULL;
+    for (i = 0; i < smartlist_len(fp_tmp); ++i) {
+      cp = smartlist_get(fp_tmp, i);
+      if (strlen(cp) != encoded_len) {
+        log_info(LD_DIR,
+                 "Skipping digest %s with non-standard length.", escaped(cp));
+        smartlist_del_keeporder(fp_tmp, i--);
+        goto again;
+      }
+      d = tor_malloc_zero(digest_len);
+      if (decode_hex ?
+          (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
+          (base64_decode(d, digest_len, cp, base64_digest_len)
+                         != digest_len)) {
+          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
+          smartlist_del_keeporder(fp_tmp, i--);
+          goto again;
+      }
+      smartlist_set(fp_tmp, i, d);
+      d = NULL;
+    again:
+      tor_free(cp);
+      tor_free(d);
+    }
+  }
+  if (sort_uniq) {
+    if (decode_hex || decode_base64) {
+      if (digests_are_256) {
+        smartlist_sort_digests256(fp_tmp);
+        smartlist_uniq_digests256(fp_tmp);
+      } else {
+        smartlist_sort_digests(fp_tmp);
+        smartlist_uniq_digests(fp_tmp);
+      }
+    } else {
+      smartlist_sort_strings(fp_tmp);
+      smartlist_uniq_strings(fp_tmp);
+    }
+  }
+  smartlist_add_all(fp_out, fp_tmp);
+  smartlist_free(fp_tmp);
+  return 0;
+}
+
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}

+ 129 - 0
src/feature/dircommon/directory.h

@@ -0,0 +1,129 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file directory.h
+ * \brief Header file for directory.c.
+ **/
+
+#ifndef TOR_DIRECTORY_H
+#define TOR_DIRECTORY_H
+
+dir_connection_t *TO_DIR_CONN(connection_t *c);
+
+#define DIR_CONN_STATE_MIN_ 1
+/** State for connection to directory server: waiting for connect(). */
+#define DIR_CONN_STATE_CONNECTING 1
+/** State for connection to directory server: sending HTTP request. */
+#define DIR_CONN_STATE_CLIENT_SENDING 2
+/** State for connection to directory server: reading HTTP response. */
+#define DIR_CONN_STATE_CLIENT_READING 3
+/** State for connection to directory server: happy and finished. */
+#define DIR_CONN_STATE_CLIENT_FINISHED 4
+/** State for connection at directory server: waiting for HTTP request. */
+#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
+/** State for connection at directory server: sending HTTP response. */
+#define DIR_CONN_STATE_SERVER_WRITING 6
+#define DIR_CONN_STATE_MAX_ 6
+
+#define DIR_PURPOSE_MIN_ 4
+/** A connection to a directory server: set after a v2 rendezvous
+ * descriptor is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
+/** A connection to a directory server: download one or more server
+ * descriptors. */
+#define DIR_PURPOSE_FETCH_SERVERDESC 6
+/** A connection to a directory server: download one or more extra-info
+ * documents. */
+#define DIR_PURPOSE_FETCH_EXTRAINFO 7
+/** A connection to a directory server: upload a server descriptor. */
+#define DIR_PURPOSE_UPLOAD_DIR 8
+/** A connection to a directory server: upload a v3 networkstatus vote. */
+#define DIR_PURPOSE_UPLOAD_VOTE 10
+/** A connection to a directory server: upload a v3 consensus signature */
+#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
+/** A connection to a directory server: download one or more v3 networkstatus
+ * votes. */
+#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
+/** A connection to a directory server: download a v3 detached signatures
+ * object for a consensus. */
+#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
+/** A connection to a directory server: download a v3 networkstatus
+ * consensus. */
+#define DIR_PURPOSE_FETCH_CONSENSUS 14
+/** A connection to a directory server: download one or more directory
+ * authority certificates. */
+#define DIR_PURPOSE_FETCH_CERTIFICATE 15
+
+/** Purpose for connection at a directory server. */
+#define DIR_PURPOSE_SERVER 16
+/** A connection to a hidden service directory server: upload a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
+/** A connection to a hidden service directory server: download a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+/** A connection to a hidden service directory: upload a v3 descriptor. */
+#define DIR_PURPOSE_UPLOAD_HSDESC 20
+/** A connection to a hidden service directory: fetch a v3 descriptor. */
+#define DIR_PURPOSE_FETCH_HSDESC 21
+/** A connection to a directory server: set after a hidden service descriptor
+ * is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
+#define DIR_PURPOSE_MAX_ 22
+
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
+#define DIR_PURPOSE_IS_UPLOAD(p)                \
+  ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
+   (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
+   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES ||        \
+   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||       \
+   (p)==DIR_PURPOSE_UPLOAD_HSDESC)
+
+enum compress_method_t;
+int parse_http_response(const char *headers, int *code, time_t *date,
+                        enum compress_method_t *compression, char **response);
+int parse_http_command(const char *headers,
+                       char **command_out, char **url_out);
+char *http_get_header(const char *headers, const char *which);
+
+int connection_dir_is_encrypted(const dir_connection_t *conn);
+int connection_dir_reached_eof(dir_connection_t *conn);
+int connection_dir_process_inbuf(dir_connection_t *conn);
+int connection_dir_finished_flushing(dir_connection_t *conn);
+int connection_dir_finished_connecting(dir_connection_t *conn);
+void connection_dir_about_to_close(dir_connection_t *dir_conn);
+
+#define DSR_HEX       (1<<0)
+#define DSR_BASE64    (1<<1)
+#define DSR_DIGEST256 (1<<2)
+#define DSR_SORT_UNIQ (1<<3)
+int dir_split_resource_into_fingerprints(const char *resource,
+                                     smartlist_t *fp_out, int *compressed_out,
+                                     int flags);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
+int dir_split_resource_into_fingerprint_pairs(const char *res,
+                                              smartlist_t *pairs_out);
+char *directory_dump_request_log(void);
+void note_request(const char *key, size_t bytes);
+
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                            const char *resource);
+
+char *authdir_type_to_string(dirinfo_type_t auth);
+
+#define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
+
+#endif /* !defined(TOR_DIRECTORY_H) */

+ 2 - 1
src/feature/hs/hs_client.c

@@ -19,7 +19,8 @@
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/hs/hs_cache.h"
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuit.h"

+ 2 - 1
src/feature/hs/hs_service.c

@@ -19,7 +19,8 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_ope.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"

+ 3 - 1
src/feature/nodelist/authcert.c

@@ -23,7 +23,9 @@
 #include "core/mainloop/mainloop.h"
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/fp_pair.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"

+ 1 - 1
src/feature/nodelist/dirlist.c

@@ -29,7 +29,7 @@
 #include "app/config/config.h"
 #include "core/or/policies.h"
 #include "feature/control/control.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"

+ 2 - 1
src/feature/nodelist/microdesc.c

@@ -14,8 +14,9 @@
 
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/client/entrynodes.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"

+ 3 - 1
src/feature/nodelist/networkstatus.c

@@ -51,7 +51,9 @@
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/reachability.h"
 #include "core/or/dos.h"

+ 2 - 1
src/feature/nodelist/node_select.c

@@ -18,7 +18,8 @@
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "feature/client/entrynodes.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"

+ 3 - 1
src/feature/nodelist/routerlist.c

@@ -69,8 +69,10 @@
 #include "feature/client/bridges.h"
 #include "feature/control/control.h"
 #include "feature/dirauth/mode.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/nodelist/authcert.h"

+ 2 - 1
src/feature/relay/router.c

@@ -16,7 +16,8 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/relay/dns.h"

+ 2 - 1
src/feature/rend/rendclient.c

@@ -19,7 +19,8 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"

+ 2 - 1
src/feature/rend/rendservice.c

@@ -19,7 +19,8 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_config.h"
 #include "core/mainloop/mainloop.h"

+ 2 - 3
src/test/fuzz/fuzz_http.c

@@ -4,14 +4,14 @@
 #include "orconfig.h"
 
 #define BUFFERS_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCACHE_PRIVATE
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
 #include "lib/container/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircache/dircache.h"
 #include "lib/log/log.h"
 
 #include "feature/dircommon/dir_connection_st.h"
@@ -132,4 +132,3 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
 
   return 0;
 }
-

+ 1 - 1
src/test/test_connection.c

@@ -18,7 +18,7 @@
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/rend/rendcache.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "core/or/connection_or.h"
 #include "lib/net/resolve.h"
 

+ 7 - 1
src/test/test_dir.c

@@ -9,8 +9,11 @@
 #define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
 #define CONTROL_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
 #define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
+#define DLSTATUS_PRIVATE
 #define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define NODE_SELECT_PRIVATE
@@ -30,8 +33,11 @@
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
 #include "feature/dirauth/bwauth.h"
+#include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/dirvote.h"

+ 3 - 1
src/test/test_dir_handle_get.c

@@ -8,12 +8,14 @@
 #define CONNECTION_PRIVATE
 #define CONFIG_PRIVATE
 #define RENDCACHE_PRIVATE
+#define DIRCACHE_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircache/dircache.h"
 #include "test/test.h"
 #include "lib/compress/compress.h"
 #include "feature/rend/rendcommon.h"

+ 3 - 2
src/test/test_entrynodes.c

@@ -8,7 +8,7 @@
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCLIENT_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
@@ -19,7 +19,8 @@
 #include "app/config/config.h"
 #include "app/config/confparse.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/client/entrynodes.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/networkstatus.h"

+ 4 - 2
src/test/test_hs_cache.c

@@ -7,13 +7,15 @@
  */
 
 #define CONNECTION_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
 #define HS_CACHE_PRIVATE
 
 #include "trunnel/ed25519_cert.h"
 #include "feature/hs/hs_cache.h"
 #include "feature/rend/rendcache.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/nodelist/networkstatus.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_http.h"

+ 1 - 1
src/test/test_hs_client.c

@@ -25,7 +25,7 @@
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "core/or/channeltls.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"

+ 1 - 1
src/test/test_hs_common.c

@@ -24,7 +24,7 @@
 #include "feature/hs/hs_service.h"
 #include "app/config/config.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/dircache/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"

+ 1 - 1
src/test/test_oos.c

@@ -9,7 +9,7 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
 #include "core/mainloop/mainloop.h"
 #include "test/test.h"
 

+ 3 - 2
src/test/test_routerlist.c

@@ -6,7 +6,7 @@
 #include <time.h>
 
 #define CONNECTION_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCLIENT_PRIVATE
 #define DIRVOTE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define HIBERNATE_PRIVATE
@@ -19,7 +19,8 @@
 #include "core/mainloop/connection.h"
 #include "feature/control/control.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"

Some files were not shown because too many files changed in this diff