123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059 |
- /* Copyright (c) 2009-2018, The Tor Project, Inc. */
- /* See LICENSE for licensing information */
- /**
- * \file microdesc.c
- *
- * \brief Implements microdescriptors -- an abbreviated description of
- * less-frequently-changing router information.
- */
- #include "or/or.h"
- #include "lib/fdio/fdio.h"
- #include "or/circuitbuild.h"
- #include "or/config.h"
- #include "or/directory.h"
- #include "or/dirserv.h"
- #include "or/entrynodes.h"
- #include "or/microdesc.h"
- #include "or/networkstatus.h"
- #include "or/nodelist.h"
- #include "or/policies.h"
- #include "or/router.h"
- #include "or/routerlist.h"
- #include "or/routerparse.h"
- #include "or/microdesc_st.h"
- #include "or/networkstatus_st.h"
- #include "or/node_st.h"
- #include "or/routerstatus_st.h"
- #ifdef HAVE_FCNTL_H
- #include <fcntl.h>
- #endif
- #ifdef HAVE_SYS_STAT_H
- #include <sys/stat.h>
- #endif
- /** A data structure to hold a bunch of cached microdescriptors. There are
- * two active files in the cache: a "cache file" that we mmap, and a "journal
- * file" that we append to. Periodically, we rebuild the cache file to hold
- * only the microdescriptors that we want to keep */
- struct microdesc_cache_t {
- /** Map from sha256-digest to microdesc_t for every microdesc_t in the
- * cache. */
- HT_HEAD(microdesc_map, microdesc_t) map;
- /** Name of the cache file. */
- char *cache_fname;
- /** Name of the journal file. */
- char *journal_fname;
- /** Mmap'd contents of the cache file, or NULL if there is none. */
- tor_mmap_t *cache_content;
- /** Number of bytes used in the journal file. */
- size_t journal_len;
- /** Number of bytes in descriptors removed as too old. */
- size_t bytes_dropped;
- /** Total bytes of microdescriptor bodies we have added to this cache */
- uint64_t total_len_seen;
- /** Total number of microdescriptors we have added to this cache */
- unsigned n_seen;
- /** True iff we have loaded this cache from disk ever. */
- int is_loaded;
- };
- static microdesc_cache_t *get_microdesc_cache_noload(void);
- /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
- static inline unsigned int
- microdesc_hash_(microdesc_t *md)
- {
- return (unsigned) siphash24g(md->digest, sizeof(md->digest));
- }
- /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
- * purposes. */
- static inline int
- microdesc_eq_(microdesc_t *a, microdesc_t *b)
- {
- return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
- }
- HT_PROTOTYPE(microdesc_map, microdesc_t, node,
- microdesc_hash_, microdesc_eq_)
- HT_GENERATE2(microdesc_map, microdesc_t, node,
- microdesc_hash_, microdesc_eq_, 0.6,
- tor_reallocarray_, tor_free_)
- /************************* md fetch fail cache *****************************/
- /* If we end up with too many outdated dirservers, something probably went
- * wrong so clean up the list. */
- #define TOO_MANY_OUTDATED_DIRSERVERS 30
- /** List of dirservers with outdated microdesc information. The smartlist is
- * filled with the hex digests of outdated dirservers. */
- static smartlist_t *outdated_dirserver_list = NULL;
- /** Note that we failed to fetch a microdescriptor from the relay with
- * <b>relay_digest</b> (of size DIGEST_LEN). */
- void
- microdesc_note_outdated_dirserver(const char *relay_digest)
- {
- char relay_hexdigest[HEX_DIGEST_LEN+1];
- /* Don't register outdated dirservers if we don't have a live consensus,
- * since we might be trying to fetch microdescriptors that are not even
- * currently active. */
- if (!networkstatus_get_live_consensus(approx_time())) {
- return;
- }
- if (!outdated_dirserver_list) {
- outdated_dirserver_list = smartlist_new();
- }
- tor_assert(outdated_dirserver_list);
- /* If the list grows too big, clean it up */
- if (BUG(smartlist_len(outdated_dirserver_list) >
- TOO_MANY_OUTDATED_DIRSERVERS)) {
- microdesc_reset_outdated_dirservers_list();
- }
- /* Turn the binary relay digest to a hex since smartlists have better support
- * for strings than digests. */
- base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
- relay_digest, DIGEST_LEN);
- /* Make sure we don't add a dirauth as an outdated dirserver */
- if (router_get_trusteddirserver_by_digest(relay_digest)) {
- log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
- return;
- }
- /* Don't double-add outdated dirservers */
- if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
- return;
- }
- /* Add it to the list of outdated dirservers */
- smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
- log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
- }
- /** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
- * outdated dirserver */
- int
- microdesc_relay_is_outdated_dirserver(const char *relay_digest)
- {
- char relay_hexdigest[HEX_DIGEST_LEN+1];
- if (!outdated_dirserver_list) {
- return 0;
- }
- /* Convert identity digest to hex digest */
- base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
- relay_digest, DIGEST_LEN);
- /* Last time we tried to fetch microdescs, was this directory mirror missing
- * any mds we asked for? */
- if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
- return 1;
- }
- return 0;
- }
- /** Reset the list of outdated dirservers. */
- void
- microdesc_reset_outdated_dirservers_list(void)
- {
- if (!outdated_dirserver_list) {
- return;
- }
- SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
- smartlist_clear(outdated_dirserver_list);
- }
- /****************************************************************************/
- /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
- * On success, return the total number of bytes written, and set
- * *<b>annotation_len_out</b> to the number of bytes written as
- * annotations. */
- static ssize_t
- dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
- {
- ssize_t r = 0;
- ssize_t written;
- if (md->body == NULL) {
- *annotation_len_out = 0;
- return 0;
- }
- /* XXXX drops unknown annotations. */
- if (md->last_listed) {
- char buf[ISO_TIME_LEN+1];
- char annotation[ISO_TIME_LEN+32];
- format_iso_time(buf, md->last_listed);
- tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
- if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
- log_warn(LD_DIR,
- "Couldn't write microdescriptor annotation: %s",
- strerror(errno));
- return -1;
- }
- r += strlen(annotation);
- *annotation_len_out = r;
- } else {
- *annotation_len_out = 0;
- }
- md->off = tor_fd_getpos(fd);
- written = write_all_to_fd(fd, md->body, md->bodylen);
- if (written != (ssize_t)md->bodylen) {
- written = written < 0 ? 0 : written;
- log_warn(LD_DIR,
- "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
- (long)written, (unsigned long)md->bodylen,
- strerror(errno));
- return -1;
- }
- r += md->bodylen;
- return r;
- }
- /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
- * such object has been allocated. */
- static microdesc_cache_t *the_microdesc_cache = NULL;
- /** Return a pointer to the microdescriptor cache, loading it if necessary. */
- microdesc_cache_t *
- get_microdesc_cache(void)
- {
- microdesc_cache_t *cache = get_microdesc_cache_noload();
- if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
- microdesc_cache_reload(cache);
- }
- return cache;
- }
- /** Return a pointer to the microdescriptor cache, creating (but not loading)
- * it if necessary. */
- static microdesc_cache_t *
- get_microdesc_cache_noload(void)
- {
- if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
- microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
- HT_INIT(microdesc_map, &cache->map);
- cache->cache_fname = get_cachedir_fname("cached-microdescs");
- cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
- the_microdesc_cache = cache;
- }
- return the_microdesc_cache;
- }
- /* There are three sources of microdescriptors:
- 1) Generated by us while acting as a directory authority.
- 2) Loaded from the cache on disk.
- 3) Downloaded.
- */
- /** Decode the microdescriptors from the string starting at <b>s</b> and
- * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
- * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
- * leave their bodies as pointers to the mmap'd cache. If where is
- * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
- * set the last_listed field of every microdesc to listed_at. If
- * requested_digests is non-null, then it contains a list of digests we mean
- * to allow, so we should reject any non-requested microdesc with a different
- * digest, and alter the list to contain only the digests of those microdescs
- * we didn't find.
- * Return a newly allocated list of the added microdescriptors, or NULL */
- smartlist_t *
- microdescs_add_to_cache(microdesc_cache_t *cache,
- const char *s, const char *eos, saved_location_t where,
- int no_save, time_t listed_at,
- smartlist_t *requested_digests256)
- {
- void * const DIGEST_REQUESTED = (void*)1;
- void * const DIGEST_RECEIVED = (void*)2;
- void * const DIGEST_INVALID = (void*)3;
- smartlist_t *descriptors, *added;
- const int allow_annotations = (where != SAVED_NOWHERE);
- smartlist_t *invalid_digests = smartlist_new();
- descriptors = microdescs_parse_from_string(s, eos,
- allow_annotations,
- where, invalid_digests);
- if (listed_at != (time_t)-1) {
- SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
- md->last_listed = listed_at);
- }
- if (requested_digests256) {
- digest256map_t *requested;
- requested = digest256map_new();
- /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
- SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
- digest256map_set(requested, cp, DIGEST_REQUESTED));
- /* Set requested[d] to DIGEST_INVALID for every md we requested which we
- * will never be able to parse. Remove the ones we didn't request from
- * invalid_digests.
- */
- SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
- if (digest256map_get(requested, cp)) {
- digest256map_set(requested, cp, DIGEST_INVALID);
- } else {
- tor_free(cp);
- SMARTLIST_DEL_CURRENT(invalid_digests, cp);
- }
- } SMARTLIST_FOREACH_END(cp);
- /* Update requested[d] to 2 for the mds we asked for and got. Delete the
- * ones we never requested from the 'descriptors' smartlist.
- */
- SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
- if (digest256map_get(requested, (const uint8_t*)md->digest)) {
- digest256map_set(requested, (const uint8_t*)md->digest,
- DIGEST_RECEIVED);
- } else {
- log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
- microdesc_free(md);
- SMARTLIST_DEL_CURRENT(descriptors, md);
- }
- } SMARTLIST_FOREACH_END(md);
- /* Remove the ones we got or the invalid ones from requested_digests256.
- */
- SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
- void *status = digest256map_get(requested, cp);
- if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
- tor_free(cp);
- SMARTLIST_DEL_CURRENT(requested_digests256, cp);
- }
- } SMARTLIST_FOREACH_END(cp);
- digest256map_free(requested, NULL);
- }
- /* For every requested microdescriptor that was unparseable, mark it
- * as not to be retried. */
- if (smartlist_len(invalid_digests)) {
- networkstatus_t *ns =
- networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
- if (ns) {
- SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
- routerstatus_t *rs =
- router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
- if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
- download_status_mark_impossible(&rs->dl_status);
- }
- } SMARTLIST_FOREACH_END(d);
- }
- }
- SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
- smartlist_free(invalid_digests);
- added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
- smartlist_free(descriptors);
- return added;
- }
- /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
- * a string to decode. Frees any members of <b>descriptors</b> that it does
- * not add. */
- smartlist_t *
- microdescs_add_list_to_cache(microdesc_cache_t *cache,
- smartlist_t *descriptors, saved_location_t where,
- int no_save)
- {
- smartlist_t *added;
- open_file_t *open_file = NULL;
- int fd = -1;
- // int n_added = 0;
- ssize_t size = 0;
- if (where == SAVED_NOWHERE && !no_save) {
- fd = start_writing_to_file(cache->journal_fname,
- OPEN_FLAGS_APPEND|O_BINARY,
- 0600, &open_file);
- if (fd < 0) {
- log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
- cache->journal_fname, strerror(errno));
- }
- }
- added = smartlist_new();
- SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
- microdesc_t *md2;
- md2 = HT_FIND(microdesc_map, &cache->map, md);
- if (md2) {
- /* We already had this one. */
- if (md2->last_listed < md->last_listed)
- md2->last_listed = md->last_listed;
- microdesc_free(md);
- if (where != SAVED_NOWHERE)
- cache->bytes_dropped += size;
- continue;
- }
- /* Okay, it's a new one. */
- if (fd >= 0) {
- size_t annotation_len;
- size = dump_microdescriptor(fd, md, &annotation_len);
- if (size < 0) {
- /* we already warned in dump_microdescriptor */
- abort_writing_to_file(open_file);
- fd = -1;
- } else {
- md->saved_location = SAVED_IN_JOURNAL;
- cache->journal_len += size;
- }
- } else {
- md->saved_location = where;
- }
- md->no_save = no_save;
- HT_INSERT(microdesc_map, &cache->map, md);
- md->held_in_map = 1;
- smartlist_add(added, md);
- ++cache->n_seen;
- cache->total_len_seen += md->bodylen;
- } SMARTLIST_FOREACH_END(md);
- if (fd >= 0) {
- if (finish_writing_to_file(open_file) < 0) {
- log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
- strerror(errno));
- smartlist_clear(added);
- return added;
- }
- }
- {
- networkstatus_t *ns = networkstatus_get_latest_consensus();
- if (ns && ns->flavor == FLAV_MICRODESC)
- SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
- }
- if (smartlist_len(added))
- router_dir_info_changed();
- return added;
- }
- /** Remove every microdescriptor in <b>cache</b>. */
- void
- microdesc_cache_clear(microdesc_cache_t *cache)
- {
- microdesc_t **entry, **next;
- for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
- microdesc_t *md = *entry;
- next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
- md->held_in_map = 0;
- microdesc_free(md);
- }
- HT_CLEAR(microdesc_map, &cache->map);
- if (cache->cache_content) {
- int res = tor_munmap_file(cache->cache_content);
- if (res != 0) {
- log_warn(LD_FS,
- "tor_munmap_file() failed clearing microdesc cache; "
- "we are probably about to leak memory.");
- /* TODO something smarter? */
- }
- cache->cache_content = NULL;
- }
- cache->total_len_seen = 0;
- cache->n_seen = 0;
- cache->bytes_dropped = 0;
- }
- /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
- * for the first time. Return 0 on success, -1 on failure. */
- int
- microdesc_cache_reload(microdesc_cache_t *cache)
- {
- struct stat st;
- char *journal_content;
- smartlist_t *added;
- tor_mmap_t *mm;
- int total = 0;
- microdesc_cache_clear(cache);
- cache->is_loaded = 1;
- mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
- if (mm) {
- added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
- SAVED_IN_CACHE, 0, -1, NULL);
- if (added) {
- total += smartlist_len(added);
- smartlist_free(added);
- }
- }
- journal_content = read_file_to_str(cache->journal_fname,
- RFTS_IGNORE_MISSING, &st);
- if (journal_content) {
- cache->journal_len = (size_t) st.st_size;
- added = microdescs_add_to_cache(cache, journal_content,
- journal_content+st.st_size,
- SAVED_IN_JOURNAL, 0, -1, NULL);
- if (added) {
- total += smartlist_len(added);
- smartlist_free(added);
- }
- tor_free(journal_content);
- }
- log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
- total);
- microdesc_cache_rebuild(cache, 0 /* don't force */);
- return 0;
- }
- /** By default, we remove any microdescriptors that have gone at least this
- * long without appearing in a current consensus. */
- #define TOLERATE_MICRODESC_AGE (7*24*60*60)
- /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
- * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
- * positive, specifically remove microdescriptors that have been unlisted
- * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
- * if we have no current live microdescriptor consensus.
- */
- void
- microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
- {
- microdesc_t **mdp, *victim;
- int dropped=0, kept=0;
- size_t bytes_dropped = 0;
- time_t now = time(NULL);
- /* If we don't know a live consensus, don't believe last_listed values: we
- * might be starting up after being down for a while. */
- if (! force &&
- ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
- return;
- if (cutoff <= 0)
- cutoff = now - TOLERATE_MICRODESC_AGE;
- for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
- const int is_old = (*mdp)->last_listed < cutoff;
- const unsigned held_by_nodes = (*mdp)->held_by_nodes;
- if (is_old && !held_by_nodes) {
- ++dropped;
- victim = *mdp;
- mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
- victim->held_in_map = 0;
- bytes_dropped += victim->bodylen;
- microdesc_free(victim);
- } else {
- if (is_old) {
- /* It's old, but it has held_by_nodes set. That's not okay. */
- /* Let's try to diagnose and fix #7164 . */
- smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
- const networkstatus_t *ns = networkstatus_get_latest_consensus();
- long networkstatus_age = -1;
- const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
- if (ns) {
- networkstatus_age = now - ns->valid_after;
- }
- log_warn(LD_BUG, "Microdescriptor seemed very old "
- "(last listed %d hours ago vs %d hour cutoff), but is still "
- "marked as being held by %d node(s). I found %d node(s) "
- "holding it. Current networkstatus is %ld hours old. "
- "Hashtable badness is %d.",
- (int)((now - (*mdp)->last_listed) / 3600),
- (int)((now - cutoff) / 3600),
- held_by_nodes,
- smartlist_len(nodes),
- networkstatus_age / 3600,
- ht_badness);
- SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
- const char *rs_match = "No RS";
- const char *rs_present = "";
- if (node->rs) {
- if (tor_memeq(node->rs->descriptor_digest,
- (*mdp)->digest, DIGEST256_LEN)) {
- rs_match = "Microdesc digest in RS matches";
- } else {
- rs_match = "Microdesc digest in RS does match";
- }
- if (ns) {
- /* This should be impossible, but let's see! */
- rs_present = " RS not present in networkstatus.";
- SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
- if (rs == node->rs) {
- rs_present = " RS okay in networkstatus.";
- }
- });
- }
- }
- log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
- node_sl_idx,
- hex_str(node->identity, DIGEST_LEN),
- node->md, node->rs, node->ri, rs_match, rs_present);
- } SMARTLIST_FOREACH_END(node);
- smartlist_free(nodes);
- (*mdp)->last_listed = now;
- }
- ++kept;
- mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
- }
- }
- if (dropped) {
- log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
- dropped,dropped+kept);
- cache->bytes_dropped += bytes_dropped;
- }
- }
- static int
- should_rebuild_md_cache(microdesc_cache_t *cache)
- {
- const size_t old_len =
- cache->cache_content ? cache->cache_content->size : 0;
- const size_t journal_len = cache->journal_len;
- const size_t dropped = cache->bytes_dropped;
- if (journal_len < 16384)
- return 0; /* Don't bother, not enough has happened yet. */
- if (dropped > (journal_len + old_len) / 3)
- return 1; /* We could save 1/3 or more of the currently used space. */
- if (journal_len > old_len / 2)
- return 1; /* We should append to the regular file */
- return 0;
- }
- /**
- * Mark <b>md</b> as having no body, and release any storage previously held
- * by its body.
- */
- static void
- microdesc_wipe_body(microdesc_t *md)
- {
- if (!md)
- return;
- if (md->saved_location != SAVED_IN_CACHE)
- tor_free(md->body);
- md->off = 0;
- md->saved_location = SAVED_NOWHERE;
- md->body = NULL;
- md->bodylen = 0;
- md->no_save = 1;
- }
- /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
- * and update every microdesc_t in the cache with pointers to its new
- * location. If <b>force</b> is true, do this unconditionally. If
- * <b>force</b> is false, do it only if we expect to save space on disk. */
- int
- microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
- {
- open_file_t *open_file;
- int fd = -1, res;
- microdesc_t **mdp;
- smartlist_t *wrote;
- ssize_t size;
- off_t off = 0, off_real;
- int orig_size, new_size;
- if (cache == NULL) {
- cache = the_microdesc_cache;
- if (cache == NULL)
- return 0;
- }
- /* Remove dead descriptors */
- microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
- if (!force && !should_rebuild_md_cache(cache))
- return 0;
- log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
- orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
- orig_size += (int)cache->journal_len;
- fd = start_writing_to_file(cache->cache_fname,
- OPEN_FLAGS_REPLACE|O_BINARY,
- 0600, &open_file);
- if (fd < 0)
- return -1;
- wrote = smartlist_new();
- HT_FOREACH(mdp, microdesc_map, &cache->map) {
- microdesc_t *md = *mdp;
- size_t annotation_len;
- if (md->no_save || !md->body)
- continue;
- size = dump_microdescriptor(fd, md, &annotation_len);
- if (size < 0) {
- microdesc_wipe_body(md);
- /* rewind, in case it was a partial write. */
- tor_fd_setpos(fd, off);
- continue;
- }
- tor_assert(((size_t)size) == annotation_len + md->bodylen);
- md->off = off + annotation_len;
- off += size;
- off_real = tor_fd_getpos(fd);
- if (off_real != off) {
- log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
- "By my count, I'm at %"PRId64
- ", but I should be at %"PRId64,
- (off), (off_real));
- if (off_real >= 0)
- off = off_real;
- }
- if (md->saved_location != SAVED_IN_CACHE) {
- tor_free(md->body);
- md->saved_location = SAVED_IN_CACHE;
- }
- smartlist_add(wrote, md);
- }
- /* We must do this unmap _before_ we call finish_writing_to_file(), or
- * windows will not actually replace the file. */
- if (cache->cache_content) {
- res = tor_munmap_file(cache->cache_content);
- if (res != 0) {
- log_warn(LD_FS,
- "Failed to unmap old microdescriptor cache while rebuilding");
- }
- cache->cache_content = NULL;
- }
- if (finish_writing_to_file(open_file) < 0) {
- log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
- strerror(errno));
- /* Okay. Let's prevent from making things worse elsewhere. */
- cache->cache_content = NULL;
- HT_FOREACH(mdp, microdesc_map, &cache->map) {
- microdesc_t *md = *mdp;
- if (md->saved_location == SAVED_IN_CACHE) {
- microdesc_wipe_body(md);
- }
- }
- smartlist_free(wrote);
- return -1;
- }
- cache->cache_content = tor_mmap_file(cache->cache_fname);
- if (!cache->cache_content && smartlist_len(wrote)) {
- log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
- cache->cache_fname);
- smartlist_free(wrote);
- return -1;
- }
- SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
- tor_assert(md->saved_location == SAVED_IN_CACHE);
- md->body = (char*)cache->cache_content->data + md->off;
- if (PREDICT_UNLIKELY(
- md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
- /* XXXX once bug 2022 is solved, we can kill this block and turn it
- * into just the tor_assert(fast_memeq) */
- off_t avail = cache->cache_content->size - md->off;
- char *bad_str;
- tor_assert(avail >= 0);
- bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
- log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
- " At offset %d, I expected to find a microdescriptor starting "
- " with \"onion-key\". Instead I got %s.",
- (int)md->off, escaped(bad_str));
- tor_free(bad_str);
- tor_assert(fast_memeq(md->body, "onion-key", 9));
- }
- } SMARTLIST_FOREACH_END(md);
- smartlist_free(wrote);
- write_str_to_file(cache->journal_fname, "", 1);
- cache->journal_len = 0;
- cache->bytes_dropped = 0;
- new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
- log_info(LD_DIR, "Done rebuilding microdesc cache. "
- "Saved %d bytes; %d still used.",
- orig_size-new_size, new_size);
- return 0;
- }
- /** Make sure that the reference count of every microdescriptor in cache is
- * accurate. */
- void
- microdesc_check_counts(void)
- {
- microdesc_t **mdp;
- if (!the_microdesc_cache)
- return;
- HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
- microdesc_t *md = *mdp;
- unsigned int found=0;
- const smartlist_t *nodes = nodelist_get_list();
- SMARTLIST_FOREACH(nodes, node_t *, node, {
- if (node->md == md) {
- ++found;
- }
- });
- tor_assert(found == md->held_by_nodes);
- }
- }
- /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
- * previously been removed from the cache if it had ever been inserted. */
- void
- microdesc_free_(microdesc_t *md, const char *fname, int lineno)
- {
- if (!md)
- return;
- /* Make sure that the microdesc was really removed from the appropriate data
- structures. */
- if (md->held_in_map) {
- microdesc_cache_t *cache = get_microdesc_cache_noload();
- microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
- if (md2 == md) {
- log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
- "in microdesc_map", fname, lineno);
- HT_REMOVE(microdesc_map, &cache->map, md);
- } else {
- log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
- "set, but microdesc was not in the map.", fname, lineno);
- }
- tor_fragile_assert();
- }
- if (md->held_by_nodes) {
- microdesc_cache_t *cache = get_microdesc_cache_noload();
- int found=0;
- const smartlist_t *nodes = nodelist_get_list();
- const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
- SMARTLIST_FOREACH(nodes, node_t *, node, {
- if (node->md == md) {
- ++found;
- node->md = NULL;
- }
- });
- if (found) {
- log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
- "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
- fname, lineno, found, md->held_by_nodes, ht_badness);
- } else {
- log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
- "set to %u, but md was not referenced by any nodes. "
- "ht_badness == %d",
- fname, lineno, md->held_by_nodes, ht_badness);
- }
- tor_fragile_assert();
- }
- //tor_assert(md->held_in_map == 0);
- //tor_assert(md->held_by_nodes == 0);
- if (md->onion_pkey)
- crypto_pk_free(md->onion_pkey);
- tor_free(md->onion_curve25519_pkey);
- tor_free(md->ed25519_identity_pkey);
- if (md->body && md->saved_location != SAVED_IN_CACHE)
- tor_free(md->body);
- if (md->family) {
- SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
- smartlist_free(md->family);
- }
- short_policy_free(md->exit_policy);
- short_policy_free(md->ipv6_exit_policy);
- tor_free(md);
- }
- /** Free all storage held in the microdesc.c module. */
- void
- microdesc_free_all(void)
- {
- if (the_microdesc_cache) {
- microdesc_cache_clear(the_microdesc_cache);
- tor_free(the_microdesc_cache->cache_fname);
- tor_free(the_microdesc_cache->journal_fname);
- tor_free(the_microdesc_cache);
- }
- if (outdated_dirserver_list) {
- SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
- smartlist_free(outdated_dirserver_list);
- }
- }
- /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
- * <b>d</b>, return it. Otherwise return NULL. */
- microdesc_t *
- microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
- {
- microdesc_t *md, search;
- if (!cache)
- cache = get_microdesc_cache();
- memcpy(search.digest, d, DIGEST256_LEN);
- md = HT_FIND(microdesc_map, &cache->map, &search);
- return md;
- }
- /** Return a smartlist of all the sha256 digest of the microdescriptors that
- * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
- * to internals of <b>ns</b>; you should not free the members of the resulting
- * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
- smartlist_t *
- microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
- int downloadable_only, digest256map_t *skip)
- {
- smartlist_t *result = smartlist_new();
- time_t now = time(NULL);
- tor_assert(ns->flavor == FLAV_MICRODESC);
- SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
- if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
- continue;
- if (downloadable_only &&
- !download_status_is_ready(&rs->dl_status, now))
- continue;
- if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
- continue;
- if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
- continue;
- /* XXXX Also skip if we're a noncache and wouldn't use this router.
- * XXXX NM Microdesc
- */
- smartlist_add(result, rs->descriptor_digest);
- } SMARTLIST_FOREACH_END(rs);
- return result;
- }
- /** Launch download requests for microdescriptors as appropriate.
- *
- * Specifically, we should launch download requests if we are configured to
- * download mirodescriptors, and there are some microdescriptors listed in the
- * current microdesc consensus that we don't have, and either we never asked
- * for them, or we failed to download them but we're willing to retry.
- */
- void
- update_microdesc_downloads(time_t now)
- {
- const or_options_t *options = get_options();
- networkstatus_t *consensus;
- smartlist_t *missing;
- digest256map_t *pending;
- if (should_delay_dir_fetches(options, NULL))
- return;
- if (directory_too_idle_to_fetch_descriptors(options, now))
- return;
- consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
- if (!consensus)
- return;
- if (!we_fetch_microdescriptors(options))
- return;
- pending = digest256map_new();
- list_pending_microdesc_downloads(pending);
- missing = microdesc_list_missing_digest256(consensus,
- get_microdesc_cache(),
- 1,
- pending);
- digest256map_free(pending, NULL);
- launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
- missing, NULL, now);
- smartlist_free(missing);
- }
- /** For every microdescriptor listed in the current microdescriptor consensus,
- * update its last_listed field to be at least as recent as the publication
- * time of the current microdescriptor consensus.
- */
- void
- update_microdescs_from_networkstatus(time_t now)
- {
- microdesc_cache_t *cache = get_microdesc_cache();
- microdesc_t *md;
- networkstatus_t *ns =
- networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
- if (! ns)
- return;
- tor_assert(ns->flavor == FLAV_MICRODESC);
- SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
- md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
- if (md && ns->valid_after > md->last_listed)
- md->last_listed = ns->valid_after;
- } SMARTLIST_FOREACH_END(rs);
- }
- /** Return true iff we should prefer to use microdescriptors rather than
- * routerdescs for building circuits. */
- int
- we_use_microdescriptors_for_circuits(const or_options_t *options)
- {
- if (options->UseMicrodescriptors == 0)
- return 0; /* the user explicitly picked no */
- return 1; /* yes and auto both mean yes */
- }
- /** Return true iff we should try to download microdescriptors at all. */
- int
- we_fetch_microdescriptors(const or_options_t *options)
- {
- if (directory_caches_dir_info(options))
- return 1;
- if (options->FetchUselessDescriptors)
- return 1;
- return we_use_microdescriptors_for_circuits(options);
- }
- /** Return true iff we should try to download router descriptors at all. */
- int
- we_fetch_router_descriptors(const or_options_t *options)
- {
- if (directory_caches_dir_info(options))
- return 1;
- if (options->FetchUselessDescriptors)
- return 1;
- return ! we_use_microdescriptors_for_circuits(options);
- }
- /** Return the consensus flavor we actually want to use to build circuits. */
- MOCK_IMPL(int,
- usable_consensus_flavor,(void))
- {
- if (we_use_microdescriptors_for_circuits(get_options())) {
- return FLAV_MICRODESC;
- } else {
- return FLAV_NS;
- }
- }
|