Merge commit 'tor-0.2.1.17-rc' into debian-merge

* commit 'tor-0.2.1.17-rc': (21 commits)
  Bump version to 0.2.1.17-rc
  Make "Invalid onion hostname" msg respect SafeLogging.
  updated rpm instructions for realtime libevent.
  Revise 0.2.1.17-rc changelog.
  Make an attempt to fix bug 1024.
  Update the year for the copyright statement in two more files
  another minor patch to add to 0.2.1.x
  and give the bug 969 fixes a changelog
  the third piece of bug 969 fixing
  the second piece of bug 969 fixing
  the first piece of bug 969 fixing
  Have eventdns set the "truncated" bit correctly.
  stop capping bandwidths we see in the consensus
  Added ChangeLog entry for control port fix
  Ignore control port commands after a QUIT
  Flush long replies over control port on QUIT
  add a changelog entry: clients use bw in consensus
  Clients now use bandwidth values in the consensus
  Serve DirPortFrontPage even if the write bucket	is low.
  Add warning that the results of --enable-geoip-stats are different from those in master.
  ...

ChangeLog

@@ -1,3 +1,50 @@
+Changes in version 0.2.1.17-rc - 2009-07-07
+  o Major features:
+    - Clients now use the bandwidth values in the consensus, rather than
+      the bandwidth values in each relay descriptor. This approach opens
+      the door to more accurate bandwidth estimates once the directory
+      authorities start doing active measurements. Implements more of
+      proposal 141.
+
+  o Major bugfixes:
+    - When Tor clients restart after 1-5 days, they discard all their
+      cached descriptors as too old, but they still use the cached
+      consensus document. This approach is good for robustness, but
+      bad for performance: since they don't know any bandwidths, they
+      end up choosing at random rather than weighting their choice by
+      speed. Fixed by the above feature of putting bandwidths in the
+      consensus. Bugfix on 0.2.0.x.
+    - Directory authorities were neglecting to mark relays down in their
+      internal histories if the relays fall off the routerlist without
+      ever being found unreachable. So there were relays in the histories
+      that haven't been seen for eight months, and are listed as being
+      up for eight months. This wreaked havoc on the "median wfu"
+      and "median mtbf" calculations, in turn making Guard and Stable
+      flags very wrong, hurting network performance. Fixes bugs 696 and
+      969. Bugfix on 0.2.0.6-alpha.
+
+  o Minor bugfixes:
+    - Serve the DirPortFrontPage page even when we have been approaching
+      our quotas recently. Fixes bug 1013; bugfix on 0.2.1.8-alpha.
+    - The control port would close the connection before flushing long
+      replies, such as the network consensus, if a QUIT command was issued
+      before the reply had completed. Now, the control port flushes all
+      pending replies before closing the connection. Also fixed a spurious
+      warning when a QUIT command is issued after a malformed or rejected
+      AUTHENTICATE command, but before the connection was closed. Patch
+      by Marcus Griep. Bugfix on 0.2.0.x; fixes bugs 1015 and 1016.
+    - When we can't find an intro key for a v2 hidden service descriptor,
+      fall back to the v0 hidden service descriptor and log a bug message.
+      Workaround for bug 1024.
+    - Fix a log message that did not respect the SafeLogging option.
+      Resolves bug 1027.
+
+  o Minor features:
+    - If we're a relay and we change our IP address, be more verbose
+      about the reason that made us change. Should help track down
+      further bugs for relays on dynamic IP addresses.
+
+
 Changes in version 0.2.1.16-rc - 2009-06-20
   o Security fixes:
     - Fix an edge case where a malicious exit relay could convince a

LICENSE

@@ -14,7 +14,7 @@ Tor is distributed under this license:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-Copyright (c) 2007-2008, The Tor Project, Inc.
+Copyright (c) 2007-2009, The Tor Project, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are

Makefile.am

@@ -1,7 +1,7 @@
 # $Id$
 # Copyright (c) 2001-2004, Roger Dingledine
 # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-# Copyright (c) 2007-2008, The Tor Project, Inc.
+# Copyright (c) 2007-2009, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 AUTOMAKE_OPTIONS = foreign

configure.in

@@ -5,7 +5,7 @@ dnl Copyright (c) 2007-2008, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_INIT
-AM_INIT_AUTOMAKE(tor, 0.2.1.16-rc)
+AM_INIT_AUTOMAKE(tor, 0.2.1.17-rc)
 AM_CONFIG_HEADER(orconfig.h)
 
 AC_CANONICAL_HOST

contrib/tor-mingw.nsi.in

@@ -9,7 +9,7 @@
 !include "FileFunc.nsh"
 !insertmacro GetParameters
   
-!define VERSION "0.2.1.16-rc"
+!define VERSION "0.2.1.17-rc"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"

doc/tor-rpm-creation.txt

@@ -2,60 +2,25 @@
 ##
 The process used to create the official rpms is as follows:
 
-Download and Extract the latest tor source code from https://www.torproject.org/.
-In the resulting directory:
-./configure
-make dist-rpm
-
-You should have at least two, maybe three, rpms.  There should be the binary
-i386.rpm, a src.rpm, and on redhat/centos machines, a debuginfo.rpm.
-
-## Optional customization
-##
-If you wish to further tune Tor binaries in rpm format beyond this list,
-see the GCC doc page for further options:
-http://gcc.gnu.org/onlinedocs/gcc-4.0.2/gcc/
-
-The tor.spec.in file contains the basic info needed to tune the binaries
-produced in rpm format. The key parameters to tune are located in the
-third section of the tor.spec.in file.  Locate the section similar to
-this:
+Download latest stable libevent from
+http://www.monkey.org/~provos/libevent/
 
-## Target a specific arch and OS
-#
-# default is i386 linux
-%define target gnu
-%define target_cpu i386
-%define target_os linux
+The first step of compiling libevent is to configure it as follows:
+./configure --enable-static --disable-shared 
 
-The three parameters: target, target_cpu, and target_os are used
-throughout the "make dist-rpm" process.  They control the parameters
-passed to "configure" and the final tuning of the binaries produced.
-The default settings, as shown above, create binaries for the widest
-range of Intel x86 or x86-compatible architectures.
+Complete the "make" and "make install".  You will need to be root,
+or sudo -s, to complete the "make install".
 
-The parameters can be set as follows:
-
-The "target" parameter:
-This should be "gnu", "redhat", or the short name of your linux distribution.
-Other possibilities are "mandrake" or "suse".  This is passed to
-"configure" through the --host, --build, and --target parameters.
-Therefore, this "target" parameter must be a valid OS for "configure" as
-well.
-
-The "target_cpu" parameter:
-This parameter controls the optimization and tuning of your binaries via
-gcc and "configure".  This parameter is passed to gcc via the -mtune= or
--mcpu= options.  The "configure" script will also receive this parameter
-through the --host, --build, and --target parameters.  Therefore, this
-"target_cpu" parameter must be valid for both gcc and "configure".  A
-few common options for this parameter may be "athlon64, i686, pentium4" or
-others.
-
-The "target_os" parameter:
-This parameter controls the target operating system.  Normally, this is
-only "linux".  If you wish to build rpms for a non-linux operating
-system, you can replace "linux" with your operating system.
+Check for a successful universal binary of libevent.a in, by default,
+/usr/local/lib by using the following command:
+	"file /usr/local/lib/libevent.a"
 
+Download and Extract the latest tor source code from
+https://www.torproject.org/download
 
+In the resulting directory:
+LIBS=-lrt ./configure
+make dist-rpm
 
+You should have at least two, maybe three, rpms.  There should be the binary
+i386.rpm, a src.rpm, and on redhat/centos machines, a debuginfo.rpm.

src/or/config.c

@@ -1381,6 +1381,13 @@ options_act(or_options_t *old_options)
     geoip_load_file(actual_fname, options);
     tor_free(actual_fname);
   }
+#ifdef ENABLE_GEOIP_STATS
+  log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but "
+           "the way these statistics are measured has changed "
+           "significantly in later versions of Tor. The results may not be "
+           "as expected if you are used to later versions.  Be sure you "
+           "know what you are doing.");
+#endif
   /* Check if we need to parse and add the EntryNodes config option. */
   if (options->EntryNodes &&
       (!old_options ||

src/or/connection_edge.c

@@ -1484,7 +1484,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn,
   addresstype = parse_extended_hostname(socks->address);
 
   if (addresstype == BAD_HOSTNAME) {
-    log_warn(LD_APP, "Invalid hostname %s; rejecting", socks->address);
+    log_warn(LD_APP, "Invalid onion hostname %s; rejecting",
+             safe_str(socks->address));
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
     connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);

src/or/control.c

@@ -2888,10 +2888,16 @@ connection_control_process_inbuf(control_connection_t *conn)
     --data_len;
   }
 
-  /* Quit is always valid. */
+  /* If the connection is already closing, ignore further commands */
+  if (TO_CONN(conn)->marked_for_close) {
+    return 0;
+  }
+
+  /* Otherwise, Quit is always valid. */
   if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
     connection_write_str_to_buf("250 closing connection\r\n", conn);
     connection_mark_for_close(TO_CONN(conn));
+    conn->_base.hold_open_until_flushed = 1;
     return 0;
   }
 

src/or/directory.c

@@ -2428,13 +2428,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
       dlen = strlen(frontpage);
       /* Let's return a disclaimer page (users shouldn't use V1 anymore,
          and caches don't fetch '/', so this is safe). */
-      if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) {
-        log_info(LD_DIRSERV,
-                 "Client asked for DirPortFrontPage content, but we've been "
-                 "writing too many bytes lately. Sending 503 Dir busy.");
-        write_http_status_line(conn, 503, "Directory busy, try again later");
-        goto done;
-      }
+
+      /* [We don't check for write_bucket_low here, since we want to serve
+       *  this page no matter what.] */
       note_request(url, dlen);
       write_http_response_header_impl(conn, dlen, "text/html", "identity",
                                       NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);

src/or/dirserv.c

@@ -797,7 +797,7 @@ directory_remove_invalid(void)
     if (r & FP_REJECT) {
       log_info(LD_DIRSERV, "Router '%s' is now rejected: %s",
                ent->nickname, msg?msg:"");
-      routerlist_remove(rl, ent, 0);
+      routerlist_remove(rl, ent, 0, time(NULL));
       i--;
       changed = 1;
       continue;
@@ -951,8 +951,8 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
     answer = get_options()->AssumeReachable ||
              now < router->last_reachable + REACHABLE_TIMEOUT;
 
-  if (router->is_running && !answer) {
-    /* it was running but now it's not. tell rephist. */
+  if (!answer) {
+    /* not considered reachable. tell rephist. */
     rep_hist_note_router_unreachable(router->cache_info.identity_digest, now);
   }
 

src/or/eventdns.c

@@ -1842,7 +1842,7 @@ evdns_server_request_format_response(struct server_request *req, int err)
 	if (j > 512) {
 overflow:
 		j = 512;
-		buf[3] |= 0x02; /* set the truncated bit. */
+		buf[2] |= 0x02; /* set the truncated bit. */
 	}
 
 	req->response_len = (size_t)j;

src/or/main.c

@@ -903,7 +903,7 @@ run_scheduled_events(time_t now)
     time_to_downrate_stability = rep_hist_downrate_old_runs(now);
   if (authdir_mode_tests_reachability(options)) {
     if (time_to_save_stability < now) {
-      if (time_to_save_stability && rep_hist_record_mtbf_data()<0) {
+      if (time_to_save_stability && rep_hist_record_mtbf_data(now, 1)<0) {
         log_warn(LD_GENERAL, "Couldn't store mtbf data.");
       }
 #define SAVE_STABILITY_INTERVAL (30*60)
@@ -1955,14 +1955,15 @@ tor_cleanup(void)
   /* Remove our pid file. We don't care if there was an error when we
    * unlink, nothing we could do about it anyways. */
   if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
     if (options->PidFile)
       unlink(options->PidFile);
     if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(time(NULL), get_or_state());
+      accounting_record_bandwidth_usage(now, get_or_state());
     or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(time(NULL));
+    or_state_save(now);
     if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data();
+      rep_hist_record_mtbf_data(now, 0);
   }
 #ifdef USE_DMALLOC
   dmalloc_log_stats();

src/or/or.h

@@ -3970,7 +3970,7 @@ void rep_history_clean(time_t before);
 
 void rep_hist_note_router_reachable(const char *id, time_t when);
 void rep_hist_note_router_unreachable(const char *id, time_t when);
-int rep_hist_record_mtbf_data(void);
+int rep_hist_record_mtbf_data(time_t now, int missing_means_down);
 int rep_hist_load_mtbf_data(time_t now);
 
 time_t rep_hist_downrate_old_runs(time_t now);
@@ -4411,7 +4411,8 @@ void routerinfo_free(routerinfo_t *router);
 void extrainfo_free(extrainfo_t *extrainfo);
 void routerlist_free(routerlist_t *rl);
 void dump_routerlist_mem_usage(int severity);
-void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old);
+void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
+                       time_t now);
 void routerlist_free_all(void);
 void routerlist_reset_warnings(void);
 void router_set_status(const char *digest, int up);

src/or/rendclient.c

@@ -94,8 +94,24 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
       }
     });
     if (!intro_key) {
-      log_warn(LD_BUG, "Internal error: could not find intro key.");
-      goto err;
+      if (rend_cache_lookup_entry(introcirc->rend_data->onion_address,
+          0, &entry) > 0) {
+        log_warn(LD_BUG, "We have both a v0 and a v2 rend desc for this "
+                 "service. The v2 desc doesn't contain the introduction "
+                 "point (and key) to send an INTRODUCE1/2 cell to this "
+                 "introduction point. Assuming the introduction point "
+                 "is for v0 rend clients and using the service key "
+                 "from the v0 desc instead. (This is probably a bug, "
+                 "because we shouldn't even have both a v0 and a v2 "
+                 "descriptor for the same service.)");
+        /* See flyspray task 1024. */
+        intro_key = entry->parsed->pk;
+      } else {
+        log_warn(LD_BUG, "Internal error: could not find intro key; we "
+                 "only have a v2 rend desc with %d intro points.",
+                 smartlist_len(entry->parsed->intro_nodes));
+        goto err;
+      }
     }
   }
   if (crypto_pk_get_digest(intro_key, payload)<0) {

src/or/rephist.c

@@ -683,9 +683,13 @@ rep_history_clean(time_t before)
   }
 }
 
-/** Write MTBF data to disk.  Returns 0 on success, negative on failure. */
+/** Write MTBF data to disk. Return 0 on success, negative on failure.
+ *
+ * If <b>missing_means_down</b>, then if we're about to write an entry
+ * that is still considered up but isn't in our routerlist, consider it
+ * to be down. */
 int
-rep_hist_record_mtbf_data(void)
+rep_hist_record_mtbf_data(time_t now, int missing_means_down)
 {
   char time_buf[ISO_TIME_LEN+1];
 
@@ -745,6 +749,18 @@ rep_hist_record_mtbf_data(void)
     hist = (or_history_t*) or_history_p;
 
     base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
+
+    if (missing_means_down && hist->start_of_run &&
+        !router_get_by_digest(digest)) {
+      /* We think this relay is running, but it's not listed in our
+       * routerlist. Somehow it fell out without telling us it went
+       * down. Complain and also correct it. */
+      log_info(LD_HIST,
+               "Relay '%s' is listed as up in rephist, but it's not in "
+               "our routerlist. Correcting.", dbuf);
+      rep_hist_note_router_unreachable(digest, now);
+    }
+
     PRINTF((f, "R %s\n", dbuf));
     if (hist->start_of_run > 0) {
       format_iso_time(time_buf, hist->start_of_run);

src/or/router.c

@@ -1509,8 +1509,8 @@ log_addr_has_changed(int severity, uint32_t prev, uint32_t cur,
            addrbuf_prev, addrbuf_cur, source);
   else
     log_notice(LD_GENERAL,
-             "Guessed our IP address as %s.",
-             addrbuf_cur);
+             "Guessed our IP address as %s (source: %s).",
+             addrbuf_cur, source);
 }
 
 /** Check whether our own address as defined by the Address configuration
@@ -1533,7 +1533,7 @@ check_descriptor_ipaddress_changed(time_t now)
   }
 
   if (prev != cur) {
-    log_addr_has_changed(LOG_INFO, prev, cur, "resolve");
+    log_addr_has_changed(LOG_NOTICE, prev, cur, "resolve");
     ip_address_changed(0);
   }
 }

src/or/routerlist.c

@@ -1523,15 +1523,12 @@ router_get_advertised_bandwidth_capped(routerinfo_t *router)
   return result;
 }
 
-/** Eventually, the number we return will come from the directory
- * consensus, so clients can dynamically update to better numbers.
- *
- * But for now, or in case there is no consensus available, just return
- * a sufficient default. */
-static uint32_t
-get_max_believable_bandwidth(void)
+/** Return bw*1000, unless bw*1000 would overflow, in which case return
+ * INT32_MAX. */
+static INLINE int32_t
+kb_to_bytes(uint32_t bw)
 {
-  return DEFAULT_MAX_BELIEVABLE_BANDWIDTH;
+  return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000;
 }
 
 /** Helper function:
@@ -1568,7 +1565,6 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule,
   int n_unknown = 0;
   bitarray_t *exit_bits;
   bitarray_t *guard_bits;
-  uint32_t max_believable_bw = get_max_believable_bandwidth();
   int me_idx = -1;
 
   /* Can't choose exit and guard at same time */
@@ -1591,48 +1587,50 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule,
     int32_t flags = 0;
     uint32_t this_bw = 0;
     if (statuses) {
-      /* need to extract router info */
       status = smartlist_get(sl, i);
       if (router_digest_is_me(status->identity_digest))
         me_idx = i;
       router = router_get_by_digest(status->identity_digest);
       is_exit = status->is_exit;
       is_guard = status->is_possible_guard;
-      if (router) {
-        this_bw = router_get_advertised_bandwidth(router);
+      if (status->has_bandwidth) {
+        this_bw = kb_to_bytes(status->bandwidth);
       } else { /* guess */
+        /* XXX022 once consensuses always list bandwidths, we can take
+         * this guessing business out. -RD */
         is_known = 0;
         flags = status->is_fast ? 1 : 0;
         flags |= is_exit ? 2 : 0;
         flags |= is_guard ? 4 : 0;
       }
     } else {
+      routerstatus_t *rs;
       router = smartlist_get(sl, i);
+      rs = router_get_consensus_status_by_id(
+             router->cache_info.identity_digest);
       if (router_digest_is_me(router->cache_info.identity_digest))
         me_idx = i;
       is_exit = router->is_exit;
       is_guard = router->is_possible_guard;
-      this_bw = router_get_advertised_bandwidth(router);
+      if (rs && rs->has_bandwidth) {
+        this_bw = kb_to_bytes(rs->bandwidth);
+      } else if (rs) { /* guess; don't trust the descriptor */
+        /* XXX022 once consensuses always list bandwidths, we can take
+         * this guessing business out. -RD */
+        is_known = 0;
+        flags = router->is_fast ? 1 : 0;
+        flags |= is_exit ? 2 : 0;
+        flags |= is_guard ? 4 : 0;
+      } else /* bridge or other descriptor not in our consensus */
+        this_bw = router_get_advertised_bandwidth_capped(router);
     }
     if (is_exit)
       bitarray_set(exit_bits, i);
     if (is_guard)
       bitarray_set(guard_bits, i);
-    /* if they claim something huge, don't believe it */
-    if (this_bw > max_believable_bw) {
-      char fp[HEX_DIGEST_LEN+1];
-      base16_encode(fp, sizeof(fp), statuses ?
-                      status->identity_digest :
-                      router->cache_info.identity_digest,
-                    DIGEST_LEN);
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-             "Bandwidth %d for router %s (%s) exceeds allowed max %d, capping",
-             this_bw, router ? router->nickname : "(null)",
-             fp, max_believable_bw);
-      this_bw = max_believable_bw;
-    }
     if (is_known) {
       bandwidths[i] = (int32_t) this_bw; // safe since MAX_BELIEVABLE<INT32_MAX
+      tor_assert(bandwidths[i] >= 0);
       if (is_guard)
         total_guard_bw += this_bw;
       else
@@ -2631,7 +2629,7 @@ routerlist_insert_old(routerlist_t *rl, routerinfo_t *ri)
  * If <b>make_old</b> is true, instead of deleting the router, we try adding
  * it to rl-&gt;old_routers. */
 void
-routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old)
+routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, time_t now)
 {
   routerinfo_t *ri_tmp;
   extrainfo_t *ei_tmp;
@@ -2639,6 +2637,9 @@ routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old)
   tor_assert(0 <= idx && idx < smartlist_len(rl->routers));
   tor_assert(smartlist_get(rl->routers, idx) == ri);
 
+  /* make sure the rephist module knows that it's not running */
+  rep_hist_note_router_unreachable(ri->cache_info.identity_digest, now);
+
   ri->cache_info.routerlist_index = -1;
   smartlist_del(rl->routers, idx);
   if (idx < smartlist_len(rl->routers)) {
@@ -3330,7 +3331,7 @@ routerlist_remove_old_routers(void)
         log_info(LD_DIR,
                  "Forgetting obsolete (too old) routerinfo for router '%s'",
                  router->nickname);
-        routerlist_remove(routerlist, router, 1);
+        routerlist_remove(routerlist, router, 1, now);
         i--;
       }
     }

src/win32/orconfig.h

@@ -226,6 +226,6 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.2.1.16-rc"
+#define VERSION "0.2.1.17-rc"