Merge remote-tracking branch 'donncha/feature14846_4'

changes/feature14846

@@ -0,0 +1,4 @@
+  o Major features (controller):
+    - New "GETINFO hs/service/desc/id/" command to retrieve a hidden service
+      descriptor from a service's local hidden service descriptor cache.
+      Closes ticket 14846.

src/or/control.c

@@ -1927,6 +1927,22 @@ getinfo_helper_dir(control_connection_t *control_conn,
       *errmsg = "Not found in cache";
       return -1;
     }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    rend_cache_entry_t *e = NULL;
+
+    question += strlen("hs/service/desc/id/");
+    if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+      /* Descriptor found in cache */
+      *answer = tor_strdup(e->desc);
+    } else {
+      *errmsg = "Not found in cache";
+      return -1;
+    }
   } else if (!strcmpstart(question, "md/id/")) {
     const node_t *node = node_get_by_hex_id(question+strlen("md/id/"));
     const microdesc_t *md = NULL;
@@ -2481,6 +2497,8 @@ static const getinfo_item_t getinfo_items[] = {
   PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
   PREFIX("hs/client/desc/id", dir,
          "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
   PREFIX("net/listeners/", listeners, "Bound addresses by type"),
   ITEM("ns/all", networkstatus,
        "Brief summary of router status (v2 directory format)"),
@@ -6233,6 +6251,31 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
   return desc_id;
 }
 
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>service_id</b> is the descriptor onion address.
+ * <b>desc_id_base32</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number.
+ */
+void
+control_event_hs_descriptor_created(const char *service_id,
+                                    const char *desc_id_base32,
+                                    int replica)
+{
+  if (!service_id || !desc_id_base32) {
+    log_warn(LD_BUG, "Called with service_digest==%p, "
+             "desc_id_base32==%p", service_id, desc_id_base32);
+    return;
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s "
+                     "REPLICA=%d\r\n",
+                     service_id,
+                     desc_id_base32,
+                     replica);
+}
+
 /** send HS_DESC upload event.
  *
  * <b>service_id</b> is the descriptor onion address.

src/or/control.h

@@ -117,6 +117,9 @@ MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
 void control_event_hs_descriptor_requested(const rend_data_t *rend_query,
                                            const char *desc_id_base32,
                                            const char *hs_dir);
+void control_event_hs_descriptor_created(const char *service_id,
+                                         const char *desc_id_base32,
+                                         int replica);
 void control_event_hs_descriptor_upload(const char *service_id,
                                         const char *desc_id_base32,
                                         const char *hs_dir);

src/or/main.c

@@ -1518,7 +1518,8 @@ run_scheduled_events(time_t now)
   /* Remove old information from rephist and the rend cache. */
   if (time_to.clean_caches < now) {
     rep_history_clean(now - options->RephistTrackTime);
-    rend_cache_clean(now);
+    rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
+    rend_cache_clean(now, REND_CACHE_TYPE_SERVICE);
     rend_cache_clean_v2_descs_as_dir(now, 0);
     microdesc_cache_rebuild(NULL, 0);
 #define CLEAN_CACHES_INTERVAL (30*60)

src/or/rendcache.c

@@ -3,7 +3,7 @@
 
 /**
  * \file rendcache.c
- * \brief Hidden service desriptor cache.
+ * \brief Hidden service descriptor cache.
  **/
 
 #include "rendcache.h"
@@ -17,6 +17,9 @@
  * rend_cache_entry_t. */
 static strmap_t *rend_cache = NULL;
 
+/** Map from service id to rend_cache_entry_t; only for hidden services. */
+static strmap_t *rend_cache_local_service = NULL;
+
 /** Map from descriptor id to rend_cache_entry_t; only for hidden service
  * directories. */
 static digestmap_t *rend_cache_v2_dir = NULL;
@@ -58,6 +61,7 @@ rend_cache_init(void)
 {
   rend_cache = strmap_new();
   rend_cache_v2_dir = digestmap_new();
+  rend_cache_local_service = strmap_new();
   rend_cache_failure = strmap_new();
 }
 
@@ -218,9 +222,11 @@ rend_cache_free_all(void)
 {
   strmap_free(rend_cache, rend_cache_entry_free_);
   digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_);
+  strmap_free(rend_cache_local_service, rend_cache_entry_free_);
   strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
   rend_cache = NULL;
   rend_cache_v2_dir = NULL;
+  rend_cache_local_service = NULL;
   rend_cache_failure = NULL;
   rend_cache_total_allocation = 0;
 }
@@ -254,24 +260,33 @@ rend_cache_failure_clean(time_t now)
   } STRMAP_FOREACH_END;
 }
 
-/** Removes all old entries from the service descriptor cache.
+/** Removes all old entries from the client or service descriptor cache.
 */
 void
-rend_cache_clean(time_t now)
+rend_cache_clean(time_t now, rend_cache_type_t cache_type)
 {
   strmap_iter_t *iter;
   const char *key;
   void *val;
   rend_cache_entry_t *ent;
   time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW;
-  for (iter = strmap_iter_init(rend_cache); !strmap_iter_done(iter); ) {
+  strmap_t *cache = NULL;
+
+  if (cache_type == REND_CACHE_TYPE_CLIENT) {
+    cache = rend_cache;
+  } else if (cache_type == REND_CACHE_TYPE_SERVICE)  {
+    cache = rend_cache_local_service;
+  }
+  tor_assert(cache);
+
+  for (iter = strmap_iter_init(cache); !strmap_iter_done(iter); ) {
     strmap_iter_get(iter, &key, &val);
     ent = (rend_cache_entry_t*)val;
     if (ent->parsed->timestamp < cutoff) {
-      iter = strmap_iter_next_rmv(rend_cache, iter);
+      iter = strmap_iter_next_rmv(cache, iter);
       rend_cache_entry_free(ent);
     } else {
-      iter = strmap_iter_next(rend_cache, iter);
+      iter = strmap_iter_next(cache, iter);
     }
   }
 }
@@ -527,6 +542,42 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
   return ret;
 }
 
+/*
+ * Lookup the v2 service descriptor with the service ID <b>query</b> in the
+ * local service descriptor cache. Return 0 if found and if <b>e</b> is
+ * non NULL, set it with the entry found. Else, a negative value is returned
+ * and <b>e</b> is untouched.
+ * -EINVAL means that <b>query</b> is not a valid service id.
+ * -ENOENT means that no entry in the cache was found. */
+int
+rend_cache_lookup_v2_desc_as_service(const char *query, rend_cache_entry_t **e)
+{
+  int ret = 0;
+  rend_cache_entry_t *entry = NULL;
+
+  tor_assert(rend_cache_local_service);
+  tor_assert(query);
+
+  if (!rend_valid_service_id(query)) {
+    ret = -EINVAL;
+    goto end;
+  }
+
+  /* Lookup descriptor and return. */
+  entry = strmap_get_lc(rend_cache_local_service, query);
+  if (!entry) {
+    ret = -ENOENT;
+    goto end;
+  }
+
+  if (e) {
+    *e = entry;
+  }
+
+ end:
+  return ret;
+}
+
 /** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and
  * copy the pointer to it to *<b>desc</b>.  Return 1 on success, 0 on
  * well-formed-but-not-found, and -1 on failure.
@@ -679,6 +730,80 @@ rend_cache_store_v2_desc_as_dir(const char *desc)
   return RCS_OKAY;
 }
 
+/** Parse the v2 service descriptor in <b>desc</b> and store it to the
+* local service rend cache. Don't attempt to decrypt the included list of
+* introduction points.
+*
+* If we have a newer descriptor with the same ID, ignore this one.
+* If we have an older descriptor with the same ID, replace it.
+*
+* Return an appropriate rend_cache_store_status_t.
+*/
+rend_cache_store_status_t
+rend_cache_store_v2_desc_as_service(const char *desc)
+{
+  rend_service_descriptor_t *parsed = NULL;
+  char desc_id[DIGEST_LEN];
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  char service_id[REND_SERVICE_ID_LEN_BASE32+1];
+  rend_cache_entry_t *e;
+  rend_cache_store_status_t retval = RCS_BADDESC;
+  tor_assert(rend_cache_local_service);
+  tor_assert(desc);
+
+  /* Parse the descriptor. */
+  if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content,
+                                       &intro_size, &encoded_size,
+                                       &next_desc, desc, 0) < 0) {
+    log_warn(LD_REND, "Could not parse descriptor.");
+    goto err;
+  }
+  /* Compute service ID from public key. */
+  if (rend_get_service_id(parsed->pk, service_id)<0) {
+    log_warn(LD_REND, "Couldn't compute service ID.");
+    goto err;
+  }
+
+  /* Do we already have a newer descriptor? Allow new descriptors with a
+     rounded timestamp equal to or newer than the current descriptor */
+  e = (rend_cache_entry_t*) strmap_get_lc(rend_cache_local_service,
+                                          service_id);
+  if (e && e->parsed->timestamp > parsed->timestamp) {
+    log_info(LD_REND, "We already have a newer service descriptor for "
+             "service ID %s.", safe_str_client(service_id));
+    goto okay;
+  }
+  /* We don't care about the introduction points. */
+  tor_free(intro_content);
+  if (!e) {
+    e = tor_malloc_zero(sizeof(rend_cache_entry_t));
+    strmap_set_lc(rend_cache_local_service, service_id, e);
+  } else {
+    rend_cache_decrement_allocation(rend_cache_entry_allocation(e));
+    rend_service_descriptor_free(e->parsed);
+    tor_free(e->desc);
+  }
+  e->parsed = parsed;
+  e->desc = tor_malloc_zero(encoded_size + 1);
+  strlcpy(e->desc, desc, encoded_size + 1);
+  e->len = encoded_size;
+  rend_cache_increment_allocation(rend_cache_entry_allocation(e));
+  log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.",
+            safe_str_client(service_id), (int)encoded_size);
+  return RCS_OKAY;
+
+ okay:
+  retval = RCS_OKAY;
+
+ err:
+  rend_service_descriptor_free(parsed);
+  tor_free(intro_content);
+  return retval;
+}
+
 /** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list
  * of introduction points with <b>descriptor_cookie</b> (which may also be
  * <b>NULL</b> if decryption is not necessary), and store the descriptor to

src/or/rendcache.h

@@ -48,14 +48,21 @@ typedef struct rend_cache_failure_t {
   digestmap_t *intro_failures;
 } rend_cache_failure_t;
 
+typedef enum {
+  REND_CACHE_TYPE_CLIENT  = 1,
+  REND_CACHE_TYPE_SERVICE = 2,
+} rend_cache_type_t;
+
 void rend_cache_init(void);
-void rend_cache_clean(time_t now);
+void rend_cache_clean(time_t now, rend_cache_type_t cache_type);
 void rend_cache_failure_clean(time_t now);
 void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove);
 void rend_cache_purge(void);
 void rend_cache_free_all(void);
 int rend_cache_lookup_entry(const char *query, int version,
                             rend_cache_entry_t **entry_out);
+int rend_cache_lookup_v2_desc_as_service(const char *query,
+                                         rend_cache_entry_t **entry_out);
 int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc);
 /** Return value from rend_cache_store_v2_desc_as_{dir,client}. */
 typedef enum {
@@ -65,6 +72,8 @@ typedef enum {
 } rend_cache_store_status_t;
 
 rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc);
+rend_cache_store_status_t rend_cache_store_v2_desc_as_service(
+                                                const char *desc);
 rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc,
                                                 const char *desc_id_base32,
                                                 const rend_data_t *rend_query,

src/or/rendcommon.c

@@ -11,6 +11,7 @@
 #include "or.h"
 #include "circuitbuild.h"
 #include "config.h"
+#include "control.h"
 #include "rendclient.h"
 #include "rendcommon.h"
 #include "rendmid.h"
@@ -461,6 +462,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
                            smartlist_t *client_cookies)
 {
   char service_id[DIGEST_LEN];
+  char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1];
   uint32_t time_period;
   char *ipos_base64 = NULL, *ipos = NULL, *ipos_encrypted = NULL,
        *descriptor_cookie = NULL;
@@ -655,6 +657,11 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
       goto err;
     }
     smartlist_add(descs_out, enc);
+    /* Add the uploaded descriptor to the local service's descriptor cache */
+    rend_cache_store_v2_desc_as_service(enc->desc_str);
+    base32_encode(service_id_base32, sizeof(service_id_base32),
+          service_id, REND_SERVICE_ID_LEN);
+    control_event_hs_descriptor_created(service_id_base32, desc_id_base32, k);
   }
 
   log_info(LD_REND, "Successfully encoded a v2 descriptor and "

src/or/rendservice.c

@@ -3202,39 +3202,72 @@ upload_service_descriptor(rend_service_t *service)
 
   rendpostperiod = get_options()->RendPostPeriod;
 
-  /* Upload descriptor? */
-  if (get_options()->PublishHidServDescriptors) {
-    networkstatus_t *c = networkstatus_get_latest_consensus();
-    if (c && smartlist_len(c->routerstatus_list) > 0) {
-      int seconds_valid, i, j, num_descs;
-      smartlist_t *descs = smartlist_new();
-      smartlist_t *client_cookies = smartlist_new();
-      /* Either upload a single descriptor (including replicas) or one
-       * descriptor for each authorized client in case of authorization
-       * type 'stealth'. */
-      num_descs = service->auth_type == REND_STEALTH_AUTH ?
-                      smartlist_len(service->clients) : 1;
-      for (j = 0; j < num_descs; j++) {
-        crypto_pk_t *client_key = NULL;
-        rend_authorized_client_t *client = NULL;
-        smartlist_clear(client_cookies);
-        switch (service->auth_type) {
-          case REND_NO_AUTH:
-            /* Do nothing here. */
-            break;
-          case REND_BASIC_AUTH:
-            SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *,
-                cl, smartlist_add(client_cookies, cl->descriptor_cookie));
-            break;
-          case REND_STEALTH_AUTH:
-            client = smartlist_get(service->clients, j);
-            client_key = client->client_key;
-            smartlist_add(client_cookies, client->descriptor_cookie);
-            break;
-        }
-        /* Encode the current descriptor. */
+  networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && smartlist_len(c->routerstatus_list) > 0) {
+    int seconds_valid, i, j, num_descs;
+    smartlist_t *descs = smartlist_new();
+    smartlist_t *client_cookies = smartlist_new();
+    /* Either upload a single descriptor (including replicas) or one
+     * descriptor for each authorized client in case of authorization
+     * type 'stealth'. */
+    num_descs = service->auth_type == REND_STEALTH_AUTH ?
+                    smartlist_len(service->clients) : 1;
+    for (j = 0; j < num_descs; j++) {
+      crypto_pk_t *client_key = NULL;
+      rend_authorized_client_t *client = NULL;
+      smartlist_clear(client_cookies);
+      switch (service->auth_type) {
+        case REND_NO_AUTH:
+          /* Do nothing here. */
+          break;
+        case REND_BASIC_AUTH:
+          SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *,
+              cl, smartlist_add(client_cookies, cl->descriptor_cookie));
+          break;
+        case REND_STEALTH_AUTH:
+          client = smartlist_get(service->clients, j);
+          client_key = client->client_key;
+          smartlist_add(client_cookies, client->descriptor_cookie);
+          break;
+      }
+      /* Encode the current descriptor. */
+      seconds_valid = rend_encode_v2_descriptors(descs, service->desc,
+                                                 now, 0,
+                                                 service->auth_type,
+                                                 client_key,
+                                                 client_cookies);
+      if (seconds_valid < 0) {
+        log_warn(LD_BUG, "Internal error: couldn't encode service "
+                 "descriptor; not uploading.");
+        smartlist_free(descs);
+        smartlist_free(client_cookies);
+        return;
+      }
+      rend_get_service_id(service->desc->pk, serviceid);
+      if (get_options()->PublishHidServDescriptors) {
+        /* Post the current descriptors to the hidden service directories. */
+        log_info(LD_REND, "Launching upload for hidden service %s",
+                     serviceid);
+        directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
+                                 seconds_valid);
+      }
+      /* Free memory for descriptors. */
+      for (i = 0; i < smartlist_len(descs); i++)
+        rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+      smartlist_clear(descs);
+      /* Update next upload time. */
+      if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS
+          > rendpostperiod)
+        service->next_upload_time = now + rendpostperiod;
+      else if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS)
+        service->next_upload_time = now + seconds_valid + 1;
+      else
+        service->next_upload_time = now + seconds_valid -
+            REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + 1;
+      /* Post also the next descriptors, if necessary. */
+      if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) {
         seconds_valid = rend_encode_v2_descriptors(descs, service->desc,
-                                                   now, 0,
+                                                   now, 1,
                                                    service->auth_type,
                                                    client_key,
                                                    client_cookies);
@@ -3245,51 +3278,23 @@ upload_service_descriptor(rend_service_t *service)
           smartlist_free(client_cookies);
           return;
         }
-        /* Post the current descriptors to the hidden service directories. */
-        rend_get_service_id(service->desc->pk, serviceid);
-        log_info(LD_REND, "Launching upload for hidden service %s",
-                     serviceid);
-        directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
-                                 seconds_valid);
+        if (get_options()->PublishHidServDescriptors) {
+          directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
+                                   seconds_valid);
+        }
         /* Free memory for descriptors. */
         for (i = 0; i < smartlist_len(descs); i++)
           rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
         smartlist_clear(descs);
-        /* Update next upload time. */
-        if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS
-            > rendpostperiod)
-          service->next_upload_time = now + rendpostperiod;
-        else if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS)
-          service->next_upload_time = now + seconds_valid + 1;
-        else
-          service->next_upload_time = now + seconds_valid -
-              REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + 1;
-        /* Post also the next descriptors, if necessary. */
-        if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) {
-          seconds_valid = rend_encode_v2_descriptors(descs, service->desc,
-                                                     now, 1,
-                                                     service->auth_type,
-                                                     client_key,
-                                                     client_cookies);
-          if (seconds_valid < 0) {
-            log_warn(LD_BUG, "Internal error: couldn't encode service "
-                     "descriptor; not uploading.");
-            smartlist_free(descs);
-            smartlist_free(client_cookies);
-            return;
-          }
-          directory_post_to_hs_dir(service->desc, descs, NULL, serviceid,
-                                   seconds_valid);
-          /* Free memory for descriptors. */
-          for (i = 0; i < smartlist_len(descs); i++)
-            rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
-          smartlist_clear(descs);
-        }
       }
-      smartlist_free(descs);
-      smartlist_free(client_cookies);
-      uploaded = 1;
+    }
+    smartlist_free(descs);
+    smartlist_free(client_cookies);
+    uploaded = 1;
+    if (get_options()->PublishHidServDescriptors) {
       log_info(LD_REND, "Successfully uploaded v2 rend descriptors!");
+    } else {
+      log_info(LD_REND, "Successfully stored created v2 rend descriptors!");
     }
   }
 
@@ -3634,9 +3639,6 @@ rend_consider_services_upload(time_t now)
                               MIN_REND_INITIAL_POST_DELAY_TESTING :
                               MIN_REND_INITIAL_POST_DELAY);
 
-  if (!get_options()->PublishHidServDescriptors)
-    return;
-
   for (i=0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
     if (!service->next_upload_time) { /* never been uploaded yet */

src/test/test.c

@@ -47,6 +47,7 @@ double fabs(double x);
 #include "connection_edge.h"
 #include "geoip.h"
 #include "rendcommon.h"
+#include "rendcache.h"
 #include "test.h"
 #include "torgzip.h"
 #include "memarea.h"
@@ -494,6 +495,9 @@ test_rend_fns(void *arg)
   tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
   tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
 
+  /* Initialize the service cache. */
+  rend_cache_init();
+
   pk1 = pk_generate(0);
   pk2 = pk_generate(1);
   generated = tor_malloc_zero(sizeof(rend_service_descriptor_t));