Flush more changes from sandbox

 - make clients cache directories and use them to seed their router lists
   at startup.  This means clients have a datadir again.
 - Introduce a global_write_bucket.  We need to respond better to exhausting
   it.
 - Remove the last vestiges of LinkPadding and TrafficShaping.
 - Configuration infrastructure support for warning on obsolete options.
 - Refactor directory header parsing to use smartlist_split_string.
 - Respond to content-encoding headers by trying to uncompress as appropriate.
 - Reply with a deflated directory when a client asks for "dir.z".
   (We could use allow-encodings instead, but allow-encodings isn't
   specified in HTTP 1.0.)


svn:r2335

doc/TODO

@@ -12,32 +12,46 @@ ARMA    - arma claims
         X Abandoned
 
       0.0.9:
-        - the user interface interface
-        - let tor clients use http proxies for dir fetching
-        - let tor servers use http proxies for port 80 exits
-        - write instructions for port-forwarding directives or programs
+        o Fix OutboundBindAddress
+        o Config defaults should be consistant with config file and no
+          config file.
+        o write instructions for port-forwarding directives or programs
           to let people run on ports 80 and 443 without needing to bind
           tor to them.
-        - learn from ben about his openssl-reinitialization-trick to
-          rotate tls keys without making new connections.
-        - figure out how to handle rendezvousing with unverified nodes.
-        - clean up all the comma-separated stuff (eg exit policies) into
+        o clean up all the comma-separated stuff (eg exit policies) into
           smartlists.
-        - per-month byte allowances.
-        - node 'groups' that are known to be in the same zone of control.
-        - figure out enclaves, e.g. so we know what to recommend that people
+        o investigate sctp for alternate transport.
+ 	o Document all undocumented options, or mark them as undocumented
+	  in the source.
+        o bandwidth buckets for write as well as read.
+        . Cached-directory changes:
+            o make clients store the cached-directory to disk,
+            o and use it when they startup, so they don't need to bootstrap
+              from the authdirservers every time they start.
+            - also, once we've reduced authdirserver entries to config
+              lines, we can have lines that list cacheddirservers too.
+        . compress the directory.
+	    o Implement gzip/zlib wrappers
+	    o Compress directories as they're cached/generated
+	    o When requested, give a compressed directory.
+	    o Decompress incoming HTTP based on Content-Encoding
+	    - Once dirservers are running new code, make clients
+	      request compressed directories.  (Alternative: Switch
+	      to HTTP/1.1 and use Allowed-Encoding.  Is that really
+	      what we want?)
+N       - switch dirservers entries to config lines.
+N       - let tor clients use http proxies for dir fetching
+N       - per-month byte allowances.
+Nr      - figure out how to handle rendezvousing with unverified nodes.
+Nr      - figure out enclaves, e.g. so we know what to recommend that people
           do, and so running a tor server on your website is helpful.
-        - compress the directory.
-        - switch dirservers entries to config lines.
-        - investigate sctp for alternate transport.
-        - nt services on win32.
-        - bandwidth buckets for write as well as read.
-        - make clients store the cached-directory to disk, and use it
-          when they startup, so they don't need to bootstrap from the
-          authdirservers every time they start. also, once we've reduced
-          authdirserver entries to config lines, we can have lines that
-          list cacheddirservers too.
+        - node 'groups' that are known to be in the same zone of control.
+        - let tor servers use http proxies for port 80 exits
+        - the user interface interface
         - add ipv6 support.
+        - learn from ben about his openssl-reinitialization-trick to
+          rotate tls keys without making new connections.
+        D nt services on win32.
 
       0.0.8:
         - fix sprintf's to snprintf's?
@@ -94,6 +108,7 @@ NICK    . unify similar config entries that need to be split. put them
         - if destination IP is running a tor node, extend a circuit there
           before sending begin.
 * don't do this for now. figure out how enclaves work. but do enclaves soon.
+        - Track max ten-second b/w ever seen, to show operator
 
       more features, complex:
         - compress the directory. client sends http header

src/or/config.c

@@ -23,6 +23,7 @@ typedef enum config_type_t {
   CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and optional
                     * whitespace. */
   CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */
+  CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */
 } config_type_t;
 
 /** Largest allowed config line */
@@ -170,6 +171,9 @@ static int config_compare(struct config_line_t *c, const char *key, config_type_
       *(struct config_line_t**)arg =
         config_line_prepend(*(struct config_line_t**)arg, c->key, c->value);
       break;
+    case CONFIG_TYPE_OBSOLETE:
+      log_fn(LOG_WARN, "Skipping obsolete configuration option '%s'", c->key);
+      break;
   }
   return 1;
 }
@@ -225,7 +229,7 @@ static int config_assign(or_options_t *options, struct config_line_t *list) {
 
     config_compare(list, "LogLevel",       CONFIG_TYPE_LINELIST, &options->LogOptions) ||
     config_compare(list, "LogFile",        CONFIG_TYPE_LINELIST, &options->LogOptions) ||
-    config_compare(list, "LinkPadding",    CONFIG_TYPE_BOOL, &options->LinkPadding) ||
+    config_compare(list, "LinkPadding",    CONFIG_TYPE_OBSOLETE, NULL) ||
 
     config_compare(list, "MaxConn",        CONFIG_TYPE_INT, &options->MaxConn) ||
     config_compare(list, "MaxOnionsPending",CONFIG_TYPE_INT, &options->MaxOnionsPending) ||
@@ -252,7 +256,7 @@ static int config_assign(or_options_t *options, struct config_line_t *list) {
     config_compare(list, "SocksBindAddress",CONFIG_TYPE_LINELIST,&options->SocksBindAddress) ||
     config_compare(list, "SocksPolicy",     CONFIG_TYPE_LINELIST,&options->SocksPolicy) ||
 
-    config_compare(list, "TrafficShaping", CONFIG_TYPE_BOOL, &options->TrafficShaping) ||
+    config_compare(list, "TrafficShaping", CONFIG_TYPE_OBSOLETE, NULL) ||
 
     config_compare(list, "User",           CONFIG_TYPE_STRING, &options->User)
 
@@ -1024,7 +1028,7 @@ const char *get_data_directory(or_options_t *options) {
   const char *d;
   if (options->DataDirectory)
     d = options->DataDirectory;
-  else if (server_mode()) {
+  else {
 #ifdef MS_WINDOWS
     char *p;
     p = tor_malloc(MAX_PATH);
@@ -1037,10 +1041,7 @@ const char *get_data_directory(or_options_t *options) {
 #else
     d = "~/.tor";
 #endif
-  } else
-    d = NULL; /* XXX008 don't create datadir until we have something
-                 we'll be putting in it */
-
+  }
   if (d && strncmp(d,"~/",2)==0) {
     char *fn = expand_filename(d);
     tor_free(options->DataDirectory);

src/or/connection.c

@@ -592,26 +592,22 @@ int retry_all_listeners(void) {
   return 0;
 }
 
-extern int global_read_bucket;
+extern int global_read_bucket, global_write_bucket;
 
 /** How many bytes at most can we read onto this connection? */
 int connection_bucket_read_limit(connection_t *conn) {
   int at_most;
 
-  if(options.LinkPadding) {
-    at_most = global_read_bucket;
+  /* do a rudimentary round-robin so one circuit can't hog a connection */
+  if(connection_speaks_cells(conn)) {
+    at_most = 32*(CELL_NETWORK_SIZE);
   } else {
-    /* do a rudimentary round-robin so one circuit can't hog a connection */
-    if(connection_speaks_cells(conn)) {
-      at_most = 32*(CELL_NETWORK_SIZE);
-    } else {
-      at_most = 32*(RELAY_PAYLOAD_SIZE);
-    }
-
-    if(at_most > global_read_bucket)
-      at_most = global_read_bucket;
+    at_most = 32*(RELAY_PAYLOAD_SIZE);
   }
 
+  if(at_most > global_read_bucket)
+    at_most = global_read_bucket;
+
   if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN)
     if(at_most > conn->receiver_bucket)
       at_most = conn->receiver_bucket;
@@ -620,7 +616,7 @@ int connection_bucket_read_limit(connection_t *conn) {
 }
 
 /** We just read num_read onto conn. Decrement buckets appropriately. */
-void connection_bucket_decrement(connection_t *conn, int num_read) {
+static void connection_read_bucket_decrement(connection_t *conn, int num_read) {
   global_read_bucket -= num_read; tor_assert(global_read_bucket >= 0);
   if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
     conn->receiver_bucket -= num_read; tor_assert(conn->receiver_bucket >= 0);
@@ -648,6 +644,7 @@ static struct timeval current_time;
 void connection_bucket_init(void) {
   tor_gettimeofday(&current_time);
   global_read_bucket = options.BandwidthBurst; /* start it at max traffic */
+  global_write_bucket = options.BandwidthBurst; /* start it at max traffic */
 }
 
 /** Some time has passed; increment buckets appropriately. */
@@ -662,11 +659,15 @@ void connection_bucket_refill(struct timeval *now) {
   current_time.tv_sec = now->tv_sec; /* update current_time */
   /* (ignore usecs for now) */
 
-  /* refill the global bucket */
+  /* refill the global buckets */
   if(global_read_bucket < options.BandwidthBurst) {
     global_read_bucket += options.BandwidthRate;
     log_fn(LOG_DEBUG,"global_read_bucket now %d.", global_read_bucket);
   }
+  if(global_write_bucket < options.BandwidthBurst) {
+    global_write_bucket += options.BandwidthRate;
+    log_fn(LOG_DEBUG,"global_write_bucket now %d.", global_write_bucket);
+  }
 
   /* refill the per-connection buckets */
   get_connection_array(&carray,&n);
@@ -680,6 +681,8 @@ void connection_bucket_refill(struct timeval *now) {
 
     if(conn->wants_to_read == 1 /* it's marked to turn reading back on now */
        && global_read_bucket > 0 /* and we're allowed to read */
+       && global_write_bucket > 0 /* and we're allowed to write (XXXX,
+                                   * not the best place to check this.) */
        && (!connection_speaks_cells(conn) ||
            conn->state != OR_CONN_STATE_OPEN ||
            conn->receiver_bucket > 0)) {
@@ -815,7 +818,7 @@ static int connection_read_to_buf(connection_t *conn) {
     rep_hist_note_bytes_read(result, time(NULL));
   }
 
-  connection_bucket_decrement(conn, result);
+  connection_read_bucket_decrement(conn, result);
   return 0;
 }
 
@@ -947,6 +950,8 @@ int connection_handle_write(connection_t *conn) {
     rep_hist_note_bytes_written(result, now);
   }
 
+  global_write_bucket -= result;
+
   if(!connection_wants_to_flush(conn)) { /* it's done flushing */
     if(connection_finished_flushing(conn) < 0) {
       /* already marked */

src/or/directory.c

@@ -210,6 +210,7 @@ directory_initiate_command(routerinfo_t *router, uint8_t purpose,
 static void directory_send_command(connection_t *conn, int purpose,
                                    const char *payload, int payload_len) {
   char fetchwholedir[] = "GET / HTTP/1.0\r\n\r\n";
+  char fetchwholedir_z[] = "GET /dir.z HTTP/1.0\r\n\r\n";
   char fetchrunninglist[] = "GET /running-routers HTTP/1.0\r\n\r\n";
   char tmp[8192];
 
@@ -288,11 +289,12 @@ parse_http_url(char *headers, char **url)
  * Otherwise, return -1.
  */
 static int
-parse_http_response(char *headers, int *code, char **message, time_t *date)
+parse_http_response(char *headers, int *code, char **message, time_t *date,
+                    int *compression)
 {
   int n1, n2;
-  const char *cp;
   char datestr[RFC1123_TIME_LEN+1];
+  smartlist_t *parsed_headers;
   tor_assert(headers && code);
 
   while(isspace((int)*headers)) headers++; /* tolerate leading whitespace */
@@ -307,22 +309,40 @@ parse_http_response(char *headers, int *code, char **message, time_t *date)
   if(message) {
     /* XXX should set *message correctly */
   }
+  parsed_headers = smartlist_create();
+  smartlist_split_string(parsed_headers, headers, "\n",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
   if (date) {
-    cp = headers;
     *date = 0;
-    while (cp && (cp = strchr(cp, '\n'))) {
-      ++cp;
-      strlcpy(datestr, cp, 7);
-      if (strcmpstart(cp, "Date: ") == 0) {
-        strlcpy(datestr, cp+6, sizeof(datestr));
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Date: ")) {
+        strlcpy(datestr, s+6, sizeof(datestr));
         /* This will do nothing on failure, so we don't need to check
            the result.   We shouldn't warn, since there are many other valid
            date formats besides the one we use. */
         parse_rfc1123_time(datestr, date);
         break;
-      }
+      });
+  }
+  if (compression) {
+    const char *enc = NULL;
+    SMARTLIST_FOREACH(parsed_headers, const char *, s,
+      if (!strcmpstart(s, "Content-Encoding: ")) {
+        enc = s+16; break;
+      });
+    if (!enc || strcmp(enc, "identity")) {
+      *compression = 0;
+    } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
+      *compression = ZLIB_METHOD;
+    } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
+      *compression = GZIP_METHOD;
+    } else {
+      log_fn(LOG_WARN, "Unrecognized content encoding: '%s'", enc);
+      *compression = 0;
     }
   }
+  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+  smartlist_free(parsed_headers);
 
   return 0;
 }
@@ -342,6 +362,7 @@ connection_dir_client_reached_eof(connection_t *conn)
   int status_code;
   time_t now, date_header=0;
   int delta;
+  int compression;
 
   switch(fetch_from_buf_http(conn->inbuf,
                              &headers, MAX_HEADERS_SIZE,
@@ -355,7 +376,8 @@ connection_dir_client_reached_eof(connection_t *conn)
     /* case 1, fall through */
   }
 
-  if(parse_http_response(headers, &status_code, NULL, &date_header) < 0) {
+  if(parse_http_response(headers, &status_code, NULL, &date_header,
+                         &compression) < 0) {
     log_fn(LOG_WARN,"Unparseable headers. Closing.");
     free(body); free(headers);
     return -1;
@@ -372,6 +394,19 @@ connection_dir_client_reached_eof(connection_t *conn)
     }
   }
 
+  if (compression != 0) {
+    char *new_body;
+    size_t new_len;
+    if (tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression)) {
+      log_fn(LOG_WARN, "Unable to decompress HTTP body.");
+      tor_free(body); tor_free(headers);
+      return -1;
+    }
+    tor_free(body);
+    body = new_body;
+    body_len = (int)new_len;
+  }
+
   if(conn->purpose == DIR_PURPOSE_FETCH_DIR) {
     /* fetch/process the directory to learn about new routers. */
     log_fn(LOG_INFO,"Received directory (size %d):\n%s", body_len, body);
@@ -545,8 +580,8 @@ directory_handle_command_get(connection_t *conn, char *headers,
     return 0;
   }
 
-  if(!strcmp(url,"/")) { /* directory fetch */
-    dlen = dirserv_get_directory(&cp, 0);
+  if(!strcmp(url,"/") || !strcmp(url,"/dir.z")) { /* directory fetch */
+    dlen = dirserv_get_directory(&cp, !strcmp(url,"/dir.z"));
 
     if(dlen == 0) {
       log_fn(LOG_WARN,"My directory is empty. Closing.");
@@ -556,9 +591,10 @@ directory_handle_command_get(connection_t *conn, char *headers,
 
     log_fn(LOG_DEBUG,"Dumping directory to client.");
     format_rfc1123_time(date, time(NULL));
-    snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\n\r\n",
+    snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\nContent-Encoding: %s\r\n\r\n",
              date,
-             (int)dlen);
+             (int)dlen,
+             strcmp(url,"/dir.z")?"identity":"deflate");
     connection_write_to_buf(tmp, strlen(tmp), conn);
     connection_write_to_buf(cp, strlen(cp), conn);
     return 0;

src/or/dirserv.c

@@ -630,10 +630,16 @@ void dirserv_set_cached_directory(const char *directory, time_t when)
 {
   time_t now;
   size_t z_len;
+  char filename[512];
   tor_assert(!options.AuthoritativeDir);
   now = time(NULL);
-  if (when>cached_directory_published &&
-      when<now+ROUTER_ALLOW_SKEW) {
+  if (when<=cached_directory_published) {
+    log_fn(LOG_INFO, "Ignoring old directory; not caching.");
+  } else if (when>=now+ROUTER_ALLOW_SKEW) {
+    log_fn(LOG_INFO, "Ignoring future directory; not caching.");
+  } if (when>cached_directory_published &&
+        when<now+ROUTER_ALLOW_SKEW) {
+    log_fn(LOG_DEBUG, "Caching directory.");
     tor_free(cached_directory);
     cached_directory = tor_strdup(directory);
     cached_directory_len = strlen(cached_directory);
@@ -644,6 +650,12 @@ void dirserv_set_cached_directory(const char *directory, time_t when)
       log_fn(LOG_WARN,"Error compressing cached directory");
     }
     cached_directory_published = when;
+    if(get_data_directory(&options)) {
+      sprintf(filename,"%s/cached-directory", get_data_directory(&options));
+      if(write_str_to_file(filename,cached_directory) < 0) {
+        log_fn(LOG_WARN, "Couldn't write cached directory to disk. Ignoring.");
+      }
+    }
   }
 }
 

src/or/main.c

@@ -21,12 +21,17 @@ extern char *conn_state_to_string[][_CONN_TYPE_MAX+1];
 
 or_options_t options; /**< Command-line and config-file options. */
 int global_read_bucket; /**< Max number of bytes I can read this second. */
+int global_write_bucket; /**< Max number of bytes I can write this second. */
 
 /** What was the read bucket before the last call to prepare_for_pool?
  * (used to determine how many bytes we've read). */
 static int stats_prev_global_read_bucket;
-/** How many bytes have we read since we started the process? */
+/** What was the write bucket before the last call to prepare_for_pool?
+ * (used to determine how many bytes we've written). */
+static int stats_prev_global_write_bucket;
+/** How many bytes have we read/written since we started the process? */
 static uint64_t stats_n_bytes_read = 0;
+static uint64_t stats_n_bytes_written = 0;
 /** How many seconds have we been running? */
 long stats_n_seconds_uptime = 0;
 
@@ -632,8 +637,10 @@ static int prepare_for_poll(void) {
   /* Check how much bandwidth we've consumed, and increment the token
    * buckets. */
   stats_n_bytes_read += stats_prev_global_read_bucket - global_read_bucket;
+  stats_n_bytes_written += stats_prev_global_write_bucket - global_write_bucket;
   connection_bucket_refill(&now);
   stats_prev_global_read_bucket = global_read_bucket;
+  stats_prev_global_write_bucket = global_write_bucket;
 
   if(now.tv_sec > current_second) { /* the second has rolled over. check more stuff. */
 
@@ -698,6 +705,7 @@ static int init_from_config(int argc, char **argv) {
   /* Set up our buckets */
   connection_bucket_init();
   stats_prev_global_read_bucket = global_read_bucket;
+  stats_prev_global_write_bucket = global_write_bucket;
 
   /* Finish backgrounding the process */
   if(options.RunAsDaemon) {
@@ -782,15 +790,8 @@ static int do_main_loop(void) {
   }
 
   /* load the routers file, or assign the defaults. */
-  if(options.RouterFile) {
-    routerlist_clear_trusted_directories();
-    if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
-      log_fn(LOG_ERR,"Error loading router list.");
-      return -1;
-    }
-  } else {
-    if(config_assign_default_dirservers() < 0)
-      return -1;
+  if(router_reload_router_list()) {
+    return -1;
   }
 
   if(authdir_mode()) {

src/or/or.h

@@ -867,8 +867,6 @@ typedef struct {
   int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
   int ClientOnly; /**< Boolean: should we never evolve into a server role? */
   int MaxConn; /**< Maximum number of simultaneous connections. */
-  int TrafficShaping; /**< Unused. */
-  int LinkPadding; /**< Unused. */
   int IgnoreVersion; /**< If true, run no matter what versions of Tor the
                       * directory recommends. */
   int RunAsDaemon; /**< If true, run in the background. (Unix only) */
@@ -1392,6 +1390,7 @@ int is_legal_nickname_or_hexdigest(const char *s);
 
 /********************************* routerlist.c ***************************/
 
+int router_reload_router_list(void);
 routerinfo_t *router_pick_directory_server(int requireauth, int requireothers);
 int all_directory_servers_down(void);
 struct smartlist_t;

src/or/routerlist.c

@@ -40,6 +40,37 @@ static routerlist_t *routerlist = NULL;
 
 extern int has_fetched_directory; /**< from main.c */
 
+/**
+ * Reload the original list of trusted dirservers, and the most recent
+ * cached directory (if present).
+ */
+int router_reload_router_list(void)
+{
+  char filename[512];
+  routerlist_clear_trusted_directories();
+  if (options.RouterFile) {
+    if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
+      log_fn(LOG_ERR,"Error loading router list.");
+      return -1;
+    }
+  } else {
+    if (config_assign_default_dirservers() < 0)
+      return -1;
+  }
+  if (get_data_directory(&options)) {
+    char *s;
+    sprintf(filename,"%s/cached-directory", get_data_directory(&options));
+    s = read_file_to_str(filename);
+    if (s) {
+      log_fn(LOG_INFO, "Loading cached directory from %s", filename);
+      if (router_load_routerlist_from_string(s, 0) < 0) {
+        log_fn(LOG_WARN, "Cached directory was unparseable; ignoring.");
+      }
+    }
+  }
+  return 0;
+}
+
 /** Try to find a running dirserver. If there are no running dirservers
  * in our routerlist, set all the authoritative ones as running again,
  * and pick one. If there are no dirservers at all in our routerlist,
@@ -63,12 +94,8 @@ routerinfo_t *router_pick_directory_server(int requireauth, int requireothers) {
          options.FascistFirewall ? "reachable" : "known");
   has_fetched_directory=0; /* reset it */
   routerlist_clear_trusted_directories();
-  if(options.RouterFile) {
-    if(router_load_routerlist_from_file(options.RouterFile, 1) < 0)
-      return NULL;
-  } else {
-    if(config_assign_default_dirservers() < 0)
-      return NULL;
+  if(router_reload_router_list()) {
+    return NULL;
   }
   /* give it one last try */
   choice = router_pick_directory_server_impl(requireauth, requireothers, 0);