tor/src/feature/hs/hs_cache.c

/* Copyright (c) 2016-2019, The Tor Project, Inc. */
/* See LICENSE for licensing information */

/**
 * \file hs_cache.c
 * \brief Handle hidden service descriptor caches.
 **/

/* For unit tests.*/
#define HS_CACHE_PRIVATE

#include "core/or/or.h"
#include "app/config/config.h"
#include "lib/crypt_ops/crypto_format.h"
#include "lib/crypt_ops/crypto_util.h"
#include "feature/hs/hs_ident.h"
#include "feature/hs/hs_common.h"
#include "feature/hs/hs_client.h"
#include "feature/hs/hs_descriptor.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/rend/rendcache.h"

#include "feature/hs/hs_cache.h"

#include "feature/dircache/dircache.h"

#include "feature/nodelist/networkstatus_st.h"

#define SUBPROCESS_PRIVATE
#include "lib/process/env.h"
#include "lib/process/subprocess.h"
#include "lib/evloop/compat_libevent.h"

#include <event2/event.h>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

static int cached_client_descriptor_has_expired(time_t now,
           const hs_cache_client_descriptor_t *cached_desc);

/********************** Directory HS cache ******************/

/* Directory descriptor cache. Map indexed by blinded key. */
static digest256map_t *hs_cache_v3_dir;

/* Remove a given descriptor from our cache. */
static void
remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
{
  tor_assert(desc);
  digest256map_remove(hs_cache_v3_dir, desc->key);
}

/* Store a given descriptor in our cache. */
static void
store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
{
  tor_assert(desc);
  digest256map_set(hs_cache_v3_dir, desc->key, desc);
}

/* Query our cache and return the entry or NULL if not found. */
static hs_cache_dir_descriptor_t *
lookup_v3_desc_as_dir(const uint8_t *key)
{
  tor_assert(key);
  return digest256map_get(hs_cache_v3_dir, key);
}

#define cache_dir_desc_free(val) \
  FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))

/* Free a directory descriptor object. */
static void
cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
{
  if (desc == NULL) {
    return;
  }
  hs_desc_plaintext_data_free(desc->plaintext_data);
  tor_free(desc->encoded_desc);
  tor_free(desc);
}

/* Helper function: Use by the free all function using the digest256map
 * interface to cache entries. */
static void
cache_dir_desc_free_void(void *ptr)
{
  cache_dir_desc_free_(ptr);
}

/* Create a new directory cache descriptor object from a encoded descriptor.
 * On success, return the heap-allocated cache object, otherwise return NULL if
 * we can't decode the descriptor. */
static hs_cache_dir_descriptor_t *
cache_dir_desc_new(const char *desc)
{
  hs_cache_dir_descriptor_t *dir_desc;

  tor_assert(desc);

  dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t));
  dir_desc->plaintext_data =
    tor_malloc_zero(sizeof(hs_desc_plaintext_data_t));
  dir_desc->encoded_desc = tor_strdup(desc);

  if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) {
    log_debug(LD_DIR, "Unable to decode descriptor. Rejecting.");
    goto err;
  }

  /* The blinded pubkey is the indexed key. */
  dir_desc->key = dir_desc->plaintext_data->blinded_pubkey.pubkey;
  dir_desc->created_ts = time(NULL);
  return dir_desc;

 err:
  cache_dir_desc_free(dir_desc);
  return NULL;
}

/* Return the size of a cache entry in bytes. */
static size_t
cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
{
  return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data)
          + strlen(entry->encoded_desc));
}

// PIRONION: v3 store

static int
hs_cache_pirserver_insert_desc(hs_cache_dir_descriptor_t *desc);

/* Try to store a valid version 3 descriptor in the directory cache. Return 0
 * on success else a negative value is returned indicating that we have a
 * newer version in our cache. On error, caller is responsible to free the
 * given descriptor desc. */
static int
cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
{
  hs_cache_dir_descriptor_t *cache_entry;

  tor_assert(desc);

  /* Verify if we have an entry in the cache for that key and if yes, check
   * if we should replace it? */
  cache_entry = lookup_v3_desc_as_dir(desc->key);
  if (cache_entry != NULL) {
    /* Only replace descriptor if revision-counter is greater than the one
     * in our cache */
    if (cache_entry->plaintext_data->revision_counter >=
        desc->plaintext_data->revision_counter) {
      log_info(LD_REND, "Descriptor revision counter in our cache is "
               "greater or equal than the one we received (%d/%d). "
               "Rejecting!",
               (int)cache_entry->plaintext_data->revision_counter,
               (int)desc->plaintext_data->revision_counter);
      goto err;
    }
    /* We now know that the descriptor we just received is a new one so
     * remove the entry we currently have from our cache so we can then
     * store the new one. */
    remove_v3_desc_as_dir(cache_entry);
    rend_cache_decrement_allocation(cache_get_dir_entry_size(cache_entry));
    cache_dir_desc_free(cache_entry);
  }
  /* Store the descriptor we just got. We are sure here that either we
   * don't have the entry or we have a newer descriptor and the old one
   * has been removed from the cache. */
  store_v3_desc_as_dir(desc);

  /* Also send it to the PIR server if we have one. */
  hs_cache_pirserver_insert_desc(desc);

  /* Update our total cache size with this entry for the OOM. This uses the
   * old HS protocol cache subsystem for which we are tied with. */
  rend_cache_increment_allocation(cache_get_dir_entry_size(desc));

  /* XXX: Update HS statistics. We should have specific stats for v3. */

  return 0;

 err:
  return -1;
}

// PIRONION: v3 lookup

/* Using the query which is the base64 encoded blinded key of a version 3
 * descriptor, lookup in our directory cache the entry. If found, 1 is
 * returned and desc_out is populated with a newly allocated string being the
 * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
 * On error, a negative value is returned and desc_out is untouched. */
static int
cache_lookup_v3_as_dir(const char *query, const char **desc_out)
{
  int found = 0;
  ed25519_public_key_t blinded_key;
  const hs_cache_dir_descriptor_t *entry;

  tor_assert(query);

  /* Decode blinded key using the given query value. */
  if (ed25519_public_from_base64(&blinded_key, query) < 0) {
    log_info(LD_REND, "Unable to decode the v3 HSDir query %s.",
             safe_str_client(query));
    goto err;
  }

  entry = lookup_v3_desc_as_dir(blinded_key.pubkey);
  if (entry != NULL) {
    found = 1;
    if (desc_out) {
      *desc_out = entry->encoded_desc;
    }
  }

  return found;

 err:
  return -1;
}

/* Clean the v3 cache by removing any entry that has expired using the
 * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
 * process will use the lifetime found in the plaintext data section. Return
 * the number of bytes cleaned. */
STATIC size_t
cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
{
  size_t bytes_removed = 0;

  /* Code flow error if this ever happens. */
  tor_assert(global_cutoff >= 0);

  if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */
    return 0;
  }

  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key,
                              hs_cache_dir_descriptor_t *, entry) {
    size_t entry_size;
    time_t cutoff = global_cutoff;
    if (!cutoff) {
      /* Cutoff is the lifetime of the entry found in the descriptor. */
      cutoff = now - entry->plaintext_data->lifetime_sec;
    }

    /* If the entry has been created _after_ the cutoff, not expired so
     * continue to the next entry in our v3 cache. */
    if (entry->created_ts > cutoff) {
      continue;
    }
    /* Here, our entry has expired, remove and free. */
    MAP_DEL_CURRENT(key);
    entry_size = cache_get_dir_entry_size(entry);
    bytes_removed += entry_size;
    /* Entry is not in the cache anymore, destroy it. */
    cache_dir_desc_free(entry);
    /* Update our cache entry allocation size for the OOM. */
    rend_cache_decrement_allocation(entry_size);
    /* Logging. */
    {
      char key_b64[BASE64_DIGEST256_LEN + 1];
      digest256_to_base64(key_b64, (const char *) key);
      log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache",
               safe_str_client(key_b64));
    }
  } DIGEST256MAP_FOREACH_END;

  return bytes_removed;
}

/* Given an encoded descriptor, store it in the directory cache depending on
 * which version it is. Return a negative value on error. On success, 0 is
 * returned. */
int
hs_cache_store_as_dir(const char *desc)
{
  hs_cache_dir_descriptor_t *dir_desc = NULL;

  tor_assert(desc);

  /* Create a new cache object. This can fail if the descriptor plaintext data
   * is unparseable which in this case a log message will be triggered. */
  dir_desc = cache_dir_desc_new(desc);
  if (dir_desc == NULL) {
    goto err;
  }

  /* Call the right function against the descriptor version. At this point,
   * we are sure that the descriptor's version is supported else the
   * decoding would have failed. */
  switch (dir_desc->plaintext_data->version) {
  case HS_VERSION_THREE:
  default:
    if (cache_store_v3_as_dir(dir_desc) < 0) {
      goto err;
    }
    break;
  }
  return 0;

 err:
  cache_dir_desc_free(dir_desc);
  return -1;
}

/* Using the query, lookup in our directory cache the entry. If found, 1 is
 * returned and desc_out is populated with a newly allocated string being
 * the encoded descriptor. If not found, 0 is returned and desc_out is
 * untouched. On error, a negative value is returned and desc_out is
 * untouched. */
int
hs_cache_lookup_as_dir(uint32_t version, const char *query,
                       const char **desc_out)
{
  int found;

  tor_assert(query);
  /* This should never be called with an unsupported version. */
  tor_assert(hs_desc_is_supported_version(version));

  switch (version) {
  case HS_VERSION_THREE:
  default:
    found = cache_lookup_v3_as_dir(query, desc_out);
    break;
  }

  return found;
}

/* Clean all directory caches using the current time now. */
void
hs_cache_clean_as_dir(time_t now)
{
  time_t cutoff;

  /* Start with v2 cache cleaning. */
  cutoff = now - rend_cache_max_entry_lifetime();
  rend_cache_clean_v2_descs_as_dir(cutoff);

  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
   * to compute the cutoff by itself using the lifetime value. */
  cache_clean_v3_as_dir(now, 0);
}

/********************** Client-side HS cache ******************/

/* Client-side HS descriptor cache. Map indexed by service identity key. */
static digest256map_t *hs_cache_v3_client;

/* Client-side introduction point state cache. Map indexed by service public
 * identity key (onion address). It contains hs_cache_client_intro_state_t
 * objects all related to a specific service. */
static digest256map_t *hs_cache_client_intro_state;

/* Return the size of a client cache entry in bytes. */
static size_t
cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
{
  return sizeof(*entry) +
         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
}

/* Remove a given descriptor from our cache. */
static void
remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
{
  tor_assert(desc);
  digest256map_remove(hs_cache_v3_client, desc->key.pubkey);
  /* Update cache size with this entry for the OOM handler. */
  rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
}

/* Store a given descriptor in our cache. */
static void
store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
{
  tor_assert(desc);
  digest256map_set(hs_cache_v3_client, desc->key.pubkey, desc);
  /* Update cache size with this entry for the OOM handler. */
  rend_cache_increment_allocation(cache_get_client_entry_size(desc));
}

/* Query our cache and return the entry or NULL if not found or if expired. */
STATIC hs_cache_client_descriptor_t *
lookup_v3_desc_as_client(const uint8_t *key)
{
  time_t now = approx_time();
  hs_cache_client_descriptor_t *cached_desc;

  tor_assert(key);

  /* Do the lookup */
  cached_desc = digest256map_get(hs_cache_v3_client, key);
  if (!cached_desc) {
    return NULL;
  }

  /* Don't return expired entries */
  if (cached_client_descriptor_has_expired(now, cached_desc)) {
    return NULL;
  }

  return cached_desc;
}

/* Parse the encoded descriptor in <b>desc_str</b> using
 * <b>service_identity_pk<b> to decrypt it first.
 *
 * If everything goes well, allocate and return a new
 * hs_cache_client_descriptor_t object. In case of error, return NULL. */
static hs_cache_client_descriptor_t *
cache_client_desc_new(const char *desc_str,
                      const ed25519_public_key_t *service_identity_pk)
{
  hs_descriptor_t *desc = NULL;
  hs_cache_client_descriptor_t *client_desc = NULL;

  tor_assert(desc_str);
  tor_assert(service_identity_pk);

  /* Decode the descriptor we just fetched. */
  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
    goto end;
  }
  tor_assert(desc);

  /* All is good: make a cache object for this descriptor */
  client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
  ed25519_pubkey_copy(&client_desc->key, service_identity_pk);
  /* Set expiration time for this cached descriptor to be the start of the next
   * time period since that's when clients need to start using the next blinded
   * pk of the service (and hence will need its next descriptor). */
  client_desc->expiration_ts = hs_get_start_time_of_next_time_period(0);
  client_desc->desc = desc;
  client_desc->encoded_desc = tor_strdup(desc_str);

 end:
  return client_desc;
}

#define cache_client_desc_free(val) \
  FREE_AND_NULL(hs_cache_client_descriptor_t, cache_client_desc_free_, (val))

/** Free memory allocated by <b>desc</b>. */
static void
cache_client_desc_free_(hs_cache_client_descriptor_t *desc)
{
  if (desc == NULL) {
    return;
  }
  hs_descriptor_free(desc->desc);
  memwipe(&desc->key, 0, sizeof(desc->key));
  memwipe(desc->encoded_desc, 0, strlen(desc->encoded_desc));
  tor_free(desc->encoded_desc);
  tor_free(desc);
}

/** Helper function: Use by the free all function to clear the client cache */
static void
cache_client_desc_free_void(void *ptr)
{
  hs_cache_client_descriptor_t *desc = ptr;
  cache_client_desc_free(desc);
}

/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
static hs_cache_intro_state_t *
cache_intro_state_new(void)
{
  hs_cache_intro_state_t *state = tor_malloc_zero(sizeof(*state));
  state->created_ts = approx_time();
  return state;
}

#define cache_intro_state_free(val) \
  FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))

/* Free an hs_cache_intro_state_t object. */
static void
cache_intro_state_free_(hs_cache_intro_state_t *state)
{
  tor_free(state);
}

/* Helper function: used by the free all function. */
static void
cache_intro_state_free_void(void *state)
{
  cache_intro_state_free_(state);
}

/* Return a newly allocated and initialized hs_cache_client_intro_state_t
 * object. */
static hs_cache_client_intro_state_t *
cache_client_intro_state_new(void)
{
  hs_cache_client_intro_state_t *cache = tor_malloc_zero(sizeof(*cache));
  cache->intro_points = digest256map_new();
  return cache;
}

#define cache_client_intro_state_free(val)              \
  FREE_AND_NULL(hs_cache_client_intro_state_t,          \
                cache_client_intro_state_free_, (val))

/* Free a cache_client_intro_state object. */
static void
cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
{
  if (cache == NULL) {
    return;
  }
  digest256map_free(cache->intro_points, cache_intro_state_free_void);
  tor_free(cache);
}

/* Helper function: used by the free all function. */
static void
cache_client_intro_state_free_void(void *entry)
{
  cache_client_intro_state_free_(entry);
}

/* For the given service identity key service_pk and an introduction
 * authentication key auth_key, lookup the intro state object. Return 1 if
 * found and put it in entry if not NULL. Return 0 if not found and entry is
 * untouched. */
static int
cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
                                const ed25519_public_key_t *auth_key,
                                hs_cache_intro_state_t **entry)
{
  hs_cache_intro_state_t *state;
  hs_cache_client_intro_state_t *cache;

  tor_assert(service_pk);
  tor_assert(auth_key);

  /* Lookup the intro state cache for this service key. */
  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
  if (cache == NULL) {
    goto not_found;
  }

  /* From the cache we just found for the service, lookup in the introduction
   * points map for the given authentication key. */
  state = digest256map_get(cache->intro_points, auth_key->pubkey);
  if (state == NULL) {
    goto not_found;
  }
  if (entry) {
    *entry = state;
  }
  return 1;
 not_found:
  return 0;
}

/* Note the given failure in state. */
static void
cache_client_intro_state_note(hs_cache_intro_state_t *state,
                              rend_intro_point_failure_t failure)
{
  tor_assert(state);
  switch (failure) {
  case INTRO_POINT_FAILURE_GENERIC:
    state->error = 1;
    break;
  case INTRO_POINT_FAILURE_TIMEOUT:
    state->timed_out = 1;
    break;
  case INTRO_POINT_FAILURE_UNREACHABLE:
    state->unreachable_count++;
    break;
  default:
    tor_assert_nonfatal_unreached();
    return;
  }
}

/* For the given service identity key service_pk and an introduction
 * authentication key auth_key, add an entry in the client intro state cache
 * If no entry exists for the service, it will create one. If state is non
 * NULL, it will point to the new intro state entry. */
static void
cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
                             const ed25519_public_key_t *auth_key,
                             hs_cache_intro_state_t **state)
{
  hs_cache_intro_state_t *entry, *old_entry;
  hs_cache_client_intro_state_t *cache;

  tor_assert(service_pk);
  tor_assert(auth_key);

  /* Lookup the state cache for this service key. */
  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
  if (cache == NULL) {
    cache = cache_client_intro_state_new();
    digest256map_set(hs_cache_client_intro_state, service_pk->pubkey, cache);
  }

  entry = cache_intro_state_new();
  old_entry = digest256map_set(cache->intro_points, auth_key->pubkey, entry);
  /* This should never happened because the code flow is to lookup the entry
   * before adding it. But, just in case, non fatal assert and free it. */
  tor_assert_nonfatal(old_entry == NULL);
  tor_free(old_entry);

  if (state) {
    *state = entry;
  }
}

/* Remove every intro point state entry from cache that has been created
 * before or at the cutoff. */
static void
cache_client_intro_state_clean(time_t cutoff,
                               hs_cache_client_intro_state_t *cache)
{
  tor_assert(cache);

  DIGEST256MAP_FOREACH_MODIFY(cache->intro_points, key,
                              hs_cache_intro_state_t *, entry) {
    if (entry->created_ts <= cutoff) {
      cache_intro_state_free(entry);
      MAP_DEL_CURRENT(key);
    }
  } DIGEST256MAP_FOREACH_END;
}

/* Return true iff no intro points are in this cache. */
static int
cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
{
  return digest256map_isempty(cache->intro_points);
}

/** Check whether <b>client_desc</b> is useful for us, and store it in the
 *  client-side HS cache if so. The client_desc is freed if we already have a
 *  fresher (higher revision counter count) in the cache. */
static int
cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
{
  hs_cache_client_descriptor_t *cache_entry;

  /* TODO: Heavy code duplication with cache_store_as_dir(). Consider
   * refactoring and uniting! */

  tor_assert(client_desc);

  /* Check if we already have a descriptor from this HS in cache. If we do,
   * check if this descriptor is newer than the cached one */
  cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
  if (cache_entry != NULL) {
    /* If we have an entry in our cache that has a revision counter greater
     * than the one we just fetched, discard the one we fetched. */
    if (cache_entry->desc->plaintext_data.revision_counter >
        client_desc->desc->plaintext_data.revision_counter) {
      cache_client_desc_free(client_desc);
      goto done;
    }
    /* Remove old entry. Make space for the new one! */
    remove_v3_desc_as_client(cache_entry);

    /* We just removed an old descriptor and will replace it. We'll close all
     * intro circuits related to this old one so we don't have leftovers. We
     * leave the rendezvous circuits opened because they could be in use. */
    hs_client_close_intro_circuits_from_desc(cache_entry->desc);

    /* Free it. */
    cache_client_desc_free(cache_entry);
  }

  /* Store descriptor in cache */
  store_v3_desc_as_client(client_desc);

 done:
  return 0;
}

/* Return true iff the cached client descriptor at <b>cached_desc</b has
 * expired. */
static int
cached_client_descriptor_has_expired(time_t now,
                               const hs_cache_client_descriptor_t *cached_desc)
{
  /* We use the current consensus time to see if we should expire this
   * descriptor since we use consensus time for all other parts of the protocol
   * as well (e.g. to build the blinded key and compute time periods). */
  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
  /* If we don't have a recent consensus, consider this entry expired since we
   * will want to fetch a new HS desc when we get a live consensus. */
  if (!ns) {
    return 1;
  }

  if (cached_desc->expiration_ts <= ns->valid_after) {
    return 1;
  }

  return 0;
}

/* clean the client cache using now as the current time. Return the total size
 * of removed bytes from the cache. */
static size_t
cache_clean_v3_as_client(time_t now)
{
  size_t bytes_removed = 0;

  if (!hs_cache_v3_client) { /* No cache to clean. Just return. */
    return 0;
  }

  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
                              hs_cache_client_descriptor_t *, entry) {
    size_t entry_size;

    /* If the entry has not expired, continue to the next cached entry */
    if (!cached_client_descriptor_has_expired(now, entry)) {
      continue;
    }
    /* Here, our entry has expired, remove and free. */
    MAP_DEL_CURRENT(key);
    entry_size = cache_get_client_entry_size(entry);
    bytes_removed += entry_size;
    /* Entry is not in the cache anymore, destroy it. */
    cache_client_desc_free(entry);
    /* Update our OOM. We didn't use the remove() function because we are in
     * a loop so we have to explicitly decrement. */
    rend_cache_decrement_allocation(entry_size);
    /* Logging. */
    {
      char key_b64[BASE64_DIGEST256_LEN + 1];
      digest256_to_base64(key_b64, (const char *) key);
      log_info(LD_REND, "Removing hidden service v3 descriptor '%s' "
                        "from client cache",
               safe_str_client(key_b64));
    }
  } DIGEST256MAP_FOREACH_END;

  return bytes_removed;
}

/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
 *  its HS encoded descriptor if it's stored in our cache, or NULL if not. */
const char *
hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
{
  hs_cache_client_descriptor_t *cached_desc = NULL;

  tor_assert(key);

  cached_desc = lookup_v3_desc_as_client(key->pubkey);
  if (cached_desc) {
    tor_assert(cached_desc->encoded_desc);
    return cached_desc->encoded_desc;
  }

  return NULL;
}

/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
 *  its HS descriptor if it's stored in our cache, or NULL if not. */
const hs_descriptor_t *
hs_cache_lookup_as_client(const ed25519_public_key_t *key)
{
  hs_cache_client_descriptor_t *cached_desc = NULL;

  tor_assert(key);

  cached_desc = lookup_v3_desc_as_client(key->pubkey);
  if (cached_desc) {
    tor_assert(cached_desc->desc);
    return cached_desc->desc;
  }

  return NULL;
}

/** Public API: Given an encoded descriptor, store it in the client HS
 *  cache. Return -1 on error, 0 on success .*/
int
hs_cache_store_as_client(const char *desc_str,
                         const ed25519_public_key_t *identity_pk)
{
  hs_cache_client_descriptor_t *client_desc = NULL;

  tor_assert(desc_str);
  tor_assert(identity_pk);

  /* Create client cache descriptor object */
  client_desc = cache_client_desc_new(desc_str, identity_pk);
  if (!client_desc) {
    log_warn(LD_GENERAL, "HSDesc parsing failed!");
    log_debug(LD_GENERAL, "Failed to parse HSDesc: %s.", escaped(desc_str));
    goto err;
  }

  /* Push it to the cache */
  if (cache_store_as_client(client_desc) < 0) {
    goto err;
  }

  return 0;

 err:
  cache_client_desc_free(client_desc);
  return -1;
}

/* Clean all client caches using the current time now. */
void
hs_cache_clean_as_client(time_t now)
{
  /* Start with v2 cache cleaning. */
  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
   * to compute the cutoff by itself using the lifetime value. */
  cache_clean_v3_as_client(now);
}

/* Purge the client descriptor cache. */
void
hs_cache_purge_as_client(void)
{
  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
                              hs_cache_client_descriptor_t *, entry) {
    size_t entry_size = cache_get_client_entry_size(entry);
    MAP_DEL_CURRENT(key);
    cache_client_desc_free(entry);
    /* Update our OOM. We didn't use the remove() function because we are in
     * a loop so we have to explicitly decrement. */
    rend_cache_decrement_allocation(entry_size);
  } DIGEST256MAP_FOREACH_END;

  log_info(LD_REND, "Hidden service client descriptor cache purged.");
}

/* For a given service identity public key and an introduction authentication
 * key, note the given failure in the client intro state cache. */
void
hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
                                 const ed25519_public_key_t *auth_key,
                                 rend_intro_point_failure_t failure)
{
  int found;
  hs_cache_intro_state_t *entry;

  tor_assert(service_pk);
  tor_assert(auth_key);

  found = cache_client_intro_state_lookup(service_pk, auth_key, &entry);
  if (!found) {
    /* Create a new entry and add it to the cache. */
    cache_client_intro_state_add(service_pk, auth_key, &entry);
  }
  /* Note down the entry. */
  cache_client_intro_state_note(entry, failure);
}

/* For a given service identity public key and an introduction authentication
 * key, return true iff it is present in the failure cache. */
const hs_cache_intro_state_t *
hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
                                 const ed25519_public_key_t *auth_key)
{
  hs_cache_intro_state_t *state = NULL;
  cache_client_intro_state_lookup(service_pk, auth_key, &state);
  return state;
}

/* Cleanup the client introduction state cache. */
void
hs_cache_client_intro_state_clean(time_t now)
{
  time_t cutoff = now - HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE;

  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
                              hs_cache_client_intro_state_t *, cache) {
    /* Cleanup intro points failure. */
    cache_client_intro_state_clean(cutoff, cache);

    /* Is this cache empty for this service key? If yes, remove it from the
     * cache. Else keep it. */
    if (cache_client_intro_state_is_empty(cache)) {
      cache_client_intro_state_free(cache);
      MAP_DEL_CURRENT(key);
    }
  } DIGEST256MAP_FOREACH_END;
}

/* Purge the client introduction state cache. */
void
hs_cache_client_intro_state_purge(void)
{
  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
                              hs_cache_client_intro_state_t *, cache) {
    MAP_DEL_CURRENT(key);
    cache_client_intro_state_free(cache);
  } DIGEST256MAP_FOREACH_END;

  log_info(LD_REND, "Hidden service client introduction point state "
                    "cache purged.");
}

/**************** Generics *********************************/

/* Do a round of OOM cleanup on all directory caches. Return the amount of
 * removed bytes. It is possible that the returned value is lower than
 * min_remove_bytes if the caches get emptied out so the caller should be
 * aware of this. */
size_t
hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
{
  time_t k;
  size_t bytes_removed = 0;

  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
  tor_assert(min_remove_bytes != 0);

  /* The algorithm is as follow. K is the oldest expected descriptor age.
   *
   *   1) Deallocate all entries from v2 cache that are older than K hours.
   *      1.1) If the amount of remove bytes has been reached, stop.
   *   2) Deallocate all entries from v3 cache that are older than K hours
   *      2.1) If the amount of remove bytes has been reached, stop.
   *   3) Set K = K - RendPostPeriod and repeat process until K is < 0.
   *
   * This ends up being O(Kn).
   */

  /* Set K to the oldest expected age in seconds which is the maximum
   * lifetime of a cache entry. We'll use the v2 lifetime because it's much
   * bigger than the v3 thus leading to cleaning older descriptors. */
  k = rend_cache_max_entry_lifetime();

  do {
    time_t cutoff;

    /* If K becomes negative, it means we've empty the caches so stop and
     * return what we were able to cleanup. */
    if (k < 0) {
      break;
    }
    /* Compute a cutoff value with K and the current time. */
    cutoff = now - k;

    /* Start by cleaning the v2 cache with that cutoff. */
    bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff);

    if (bytes_removed < min_remove_bytes) {
      /* We haven't remove enough bytes so clean v3 cache. */
      bytes_removed += cache_clean_v3_as_dir(now, cutoff);
      /* Decrement K by a post period to shorten the cutoff. */
      k -= get_options()->RendPostPeriod;
    }
  } while (bytes_removed < min_remove_bytes);

  return bytes_removed;
}

/* Return the maximum size of a v3 HS descriptor. */
unsigned int
hs_cache_get_max_descriptor_size(void)
{
  return (unsigned) networkstatus_get_param(NULL,
                                            "HSV3MaxDescriptorSize",
                                            HS_DESC_MAX_LEN, 1, INT32_MAX);
}

static process_handle_t *pirserver;
static buf_t *pirserver_stdin_buf;
static struct event *pirserver_stdin_ev;
static struct event *pirserver_stdout_ev;
static struct event *pirserver_stderr_ev;

typedef enum {
    PIRSERVER_READSTATE_HEADER,
    PIRSERVER_READSTATE_BODY
} PIRServerReadState;

#define PIRSERVER_HDR_SIZE 13

#define PIRSERVER_REQUEST_PARAMS 0x01
#define PIRSERVER_REQUEST_STORE 0x02
#define PIRSERVER_REQUEST_LOOKUP 0x03

#define PIRSERVER_RESPONSE_PARAMS 0xFF
#define PIRSERVER_RESPONSE_LOOKUP_SUCCESS 0xFE
#define PIRSERVER_RESPONSE_LOOKUP_FAILURE 0xFD

static void
hs_cache_pirserver_received(const unsigned char *hdrbuf,
    const char *bodybuf, size_t bodylen)
{
    /* PIRONION TODO: actually deliver the received message */
    log_info(LD_DIRSERV,"PIRSERVER response header %p type %02x body len %ld body %s", *(void**)hdrbuf, hdrbuf[8], bodylen, escaped(bodybuf));
    if (hdrbuf[8] == PIRSERVER_RESPONSE_PARAMS) {
        /* Send the params back to the client */
        dir_connection_t *conn;
        memmove((char *)(&conn), hdrbuf, sizeof(conn));
        dircache_pirserver_reply_params(conn, bodybuf, bodylen);
    }
}

/* This is called when the pirserver has output for us. */
static void
hs_cache_pirserver_stdoutcb(evutil_socket_t fd, short what,
        ATTR_UNUSED void *arg) {
    static PIRServerReadState readstate = PIRSERVER_READSTATE_HEADER;
    static size_t readoff = 0;
    static size_t readleft = PIRSERVER_HDR_SIZE;
    static unsigned char hdrbuf[PIRSERVER_HDR_SIZE];
    static char *bodybuf = NULL;

    if (!(what & EV_READ)) {
        /* Not sure why we're here */
        return;
    }

    if (readstate == PIRSERVER_READSTATE_HEADER) {
        int res = read(fd, hdrbuf + readoff, readleft);
        if (res <= 0) return;
        readoff += res;
        readleft -= res;
        if (readleft == 0) {
            readleft = ntohl(*(uint32_t*)(hdrbuf+PIRSERVER_HDR_SIZE-4));
            tor_free(bodybuf);
            if (readleft > 0) {
                bodybuf = tor_malloc(readleft);
                readoff = 0;
                readstate = PIRSERVER_READSTATE_BODY;
            } else {
                hs_cache_pirserver_received(hdrbuf, NULL, 0);
                readoff = 0;
                readleft = PIRSERVER_HDR_SIZE;
                readstate = PIRSERVER_READSTATE_HEADER;
            }
        }
    } else if (readstate == PIRSERVER_READSTATE_BODY) {
        int res = read(fd, bodybuf + readoff, readleft);
        if (res <= 0) return;
        readoff += res;
        readleft -= res;
        if (readleft == 0) {
            /* Reading is complete */
            hs_cache_pirserver_received(hdrbuf, bodybuf, readoff);

            /* Prepare for the next output from the PIR server */
            tor_free(bodybuf);
            readoff = 0;
            readleft = PIRSERVER_HDR_SIZE;
            readstate = PIRSERVER_READSTATE_HEADER;
        }
    }
}

/* This is called when the pirserver is ready to read from its stdin. */
static void
hs_cache_pirserver_stdincb(evutil_socket_t fd, short what,
        ATTR_UNUSED void *arg) {
    int res;
    size_t bufsize = buf_datalen(pirserver_stdin_buf);
    char *netbuf = NULL;

    if (!(what & EV_WRITE)) {
        /* Not sure why we're here */
        log_info(LD_DIRSERV,"PIRSERVER bailing");
        return;
    }

    if (bufsize == 0) {
        log_err(LD_DIRSERV,"PIRSERVER trying to write 0-length buffer");
        return;
    }

    netbuf = tor_malloc(bufsize);
    if (netbuf == NULL) {
        log_err(LD_DIRSERV,"PIRSERVER failed to allocate buffer");
        return;
    }

    /* One might think that just calling buf_flush_to_socket would be
     * the thing to do, but that function ends up calling sendto()
     * instead of write(), which doesn't work on pipes.  So we do it
     * more manually.  Using a bufferevent may be another option. */
    buf_peek(pirserver_stdin_buf, netbuf, bufsize);
    res = write(fd, netbuf, bufsize);
    tor_free(netbuf);
    if (res < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
        /* Try writing again later */
        event_add(pirserver_stdin_ev, NULL);
        return;
    }
    if (res <= 0) {
        /* Stop trying to write. */
        return;
    }
    buf_drain(pirserver_stdin_buf, res);
    bufsize -= res;

    if (bufsize > 0) {
        /* There's more to write */
        event_add(pirserver_stdin_ev, NULL);
    }
}

/* This is called when the pirserver writes something to its stderr. */
static void
hs_cache_pirserver_stderrcb(evutil_socket_t fd, short what,
        ATTR_UNUSED void *arg) {
    if (!(what & EV_READ)) {
        /* Not sure why we're here */
        return;
    }

    char buf[1000];
    int res = read(fd, buf, sizeof(buf)-1);
    if (res <= 0) return;
    buf[res] = '\0';
    log_info(LD_DIRSERV,"PIRSERVER %s", escaped(buf));
}

/* Poke the hidden service cache PIR subsystem to launch the PIR
   server if needed. */
static void
hs_cache_pir_poke(void)
{
    int res;
    const char *pirserverpath = getenv("PIR_SERVER_PATH");
    smartlist_t *env_vars = get_current_process_environment_variables();
    const char *argv[2];
    process_environment_t *env;

    if (pirserver && pirserver->status == PROCESS_STATUS_RUNNING) {
        /* The PIR server appears to be open */
        return;
    }

    if (pirserver) {
        if (pirserver_stdin_buf) {
            buf_free(pirserver_stdin_buf);
            pirserver_stdin_buf = NULL;
        }
        if (pirserver_stdin_ev) {
            event_free(pirserver_stdin_ev);
            pirserver_stdin_ev = NULL;
        }
        if (pirserver_stdout_ev) {
            event_free(pirserver_stdout_ev);
            pirserver_stdout_ev = NULL;
        }
        if (pirserver_stderr_ev) {
            event_free(pirserver_stderr_ev);
            pirserver_stderr_ev = NULL;
        }
        tor_process_handle_destroy(pirserver, 1);
        pirserver = NULL;
    }

    if (!pirserverpath) {
        /* We don't have a configured PIR server */
        return;
    }

    argv[0] = pirserverpath;
    argv[1] = NULL;

    env = process_environment_make(env_vars);

    res = tor_spawn_background(pirserverpath, argv, env, &pirserver);

    SMARTLIST_FOREACH(env_vars, void *, x, tor_free(x));
    smartlist_free(env_vars);

    if (res != PROCESS_STATUS_RUNNING) {
        /* Launch failure */
        return;
    }

    /* Create a libevent event to listen to the PIR server's responses. */
    pirserver_stdout_ev = event_new(tor_libevent_get_base(),
        pirserver->stdout_pipe, EV_READ|EV_PERSIST,
        hs_cache_pirserver_stdoutcb, NULL);
    event_add(pirserver_stdout_ev, NULL);

    /* And one to listen to the PIR server's stderr. */
    pirserver_stderr_ev = event_new(tor_libevent_get_base(),
        pirserver->stderr_pipe, EV_READ|EV_PERSIST,
        hs_cache_pirserver_stderrcb, NULL);
    event_add(pirserver_stderr_ev, NULL);

    /* And one for writability to the pirserver's stdin, but don't add
     * it just yet.  Also create the buffer it will use. */
    pirserver_stdin_buf = buf_new();
    pirserver_stdin_ev = event_new(tor_libevent_get_base(),
        pirserver->stdin_pipe, EV_WRITE, hs_cache_pirserver_stdincb,
        NULL);
}

/* Initialize the hidden service cache PIR subsystem. */
static void
hs_cache_pir_init(void)
{
    pirserver = NULL;
    pirserver_stdin_buf = NULL;
    pirserver_stdin_ev = NULL;
    pirserver_stdout_ev = NULL;
    pirserver_stderr_ev = NULL;
}

static int
hs_cache_pirserver_send(const unsigned char *buf, size_t len)
{
    hs_cache_pir_poke();
    if (pirserver == NULL || pirserver->status != PROCESS_STATUS_RUNNING) {
        /* Launch failed */
        return -1;
    }

    /* Write the data to the stdin buffer */
    if (len > 0) {
        buf_add(pirserver_stdin_buf, (const char *)buf, len);
        event_add(pirserver_stdin_ev, NULL);
    }
    return len;
}

static int
hs_cache_pirserver_insert_desc(hs_cache_dir_descriptor_t *desc)
{
    unsigned char hdr[13];
    size_t encoded_desc_len;
    size_t len;
    int res;
    int written = 0;
    static uint32_t inscounter = 0;

    memmove(hdr, "\0\0\0\0", 4);
    ++inscounter;
    *(uint32_t*)(hdr+4) = htonl(inscounter);
    hdr[8] = PIRSERVER_REQUEST_STORE;

    encoded_desc_len = strlen(desc->encoded_desc);
    len = DIGEST256_LEN + encoded_desc_len;
    *(uint32_t*)(hdr+9) = htonl(len);

    res = hs_cache_pirserver_send(hdr, PIRSERVER_HDR_SIZE);
    if (res <= 0) return -1;
    written += res;

    res = hs_cache_pirserver_send(desc->key, DIGEST256_LEN);
    if (res <= 0) return -1;
    written += res;

    res = hs_cache_pirserver_send(
            (const unsigned char *)desc->encoded_desc,
            encoded_desc_len);
    if (res <= 0) return -1;
    written += res;

    return written;
}

/** Helper function for GET /tor/pironion/params.
 */
int
hs_cache_pirserver_get_params(dir_connection_t *conn)
{
    /* Ask the pirserver for the params */

    int res;
    unsigned char hdr[PIRSERVER_HDR_SIZE];

    /* PIRONION TODO: For now, the request id is just literally the
     * dir_connection_t pointer itself, so that when we get the
     * response, we can write to it.  But of course it's possible that
     * object will have disappeared between now and then.  Fix this
     * later by hooking the callback that closes a dir_connection_t. */
    memmove(hdr, (const char *)(&conn), sizeof(conn));
    hdr[8] = PIRSERVER_REQUEST_PARAMS;
    memmove(hdr+9, "\0\0\0\0", 4);
    res = hs_cache_pirserver_send(hdr, PIRSERVER_HDR_SIZE);
    if (res < PIRSERVER_HDR_SIZE) {
        return -1;
    }
    return 0;
}

/* Initialize the hidden service cache subsystem. */
void
hs_cache_init(void)
{
  /* Calling this twice is very wrong code flow. */
  tor_assert(!hs_cache_v3_dir);
  hs_cache_v3_dir = digest256map_new();

  tor_assert(!hs_cache_v3_client);
  hs_cache_v3_client = digest256map_new();

  tor_assert(!hs_cache_client_intro_state);
  hs_cache_client_intro_state = digest256map_new();

  hs_cache_pir_init();
}

/* Cleanup the hidden service cache subsystem. */
void
hs_cache_free_all(void)
{
  digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_void);
  hs_cache_v3_dir = NULL;

  digest256map_free(hs_cache_v3_client, cache_client_desc_free_void);
  hs_cache_v3_client = NULL;

  digest256map_free(hs_cache_client_intro_state,
                    cache_client_intro_state_free_void);
  hs_cache_client_intro_state = NULL;
}