microdesc.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967
  1. /* Copyright (c) 2009-2016, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file microdesc.c
  5. *
  6. * \brief Implements microdescriptors -- an abbreviated description of
  7. * less-frequently-changing router information.
  8. */
  9. #include "or.h"
  10. #include "circuitbuild.h"
  11. #include "config.h"
  12. #include "directory.h"
  13. #include "dirserv.h"
  14. #include "entrynodes.h"
  15. #include "microdesc.h"
  16. #include "networkstatus.h"
  17. #include "nodelist.h"
  18. #include "policies.h"
  19. #include "router.h"
  20. #include "routerlist.h"
  21. #include "routerparse.h"
  22. /** A data structure to hold a bunch of cached microdescriptors. There are
  23. * two active files in the cache: a "cache file" that we mmap, and a "journal
  24. * file" that we append to. Periodically, we rebuild the cache file to hold
  25. * only the microdescriptors that we want to keep */
  26. struct microdesc_cache_t {
  27. /** Map from sha256-digest to microdesc_t for every microdesc_t in the
  28. * cache. */
  29. HT_HEAD(microdesc_map, microdesc_t) map;
  30. /** Name of the cache file. */
  31. char *cache_fname;
  32. /** Name of the journal file. */
  33. char *journal_fname;
  34. /** Mmap'd contents of the cache file, or NULL if there is none. */
  35. tor_mmap_t *cache_content;
  36. /** Number of bytes used in the journal file. */
  37. size_t journal_len;
  38. /** Number of bytes in descriptors removed as too old. */
  39. size_t bytes_dropped;
  40. /** Total bytes of microdescriptor bodies we have added to this cache */
  41. uint64_t total_len_seen;
  42. /** Total number of microdescriptors we have added to this cache */
  43. unsigned n_seen;
  44. /** True iff we have loaded this cache from disk ever. */
  45. int is_loaded;
  46. };
  47. static microdesc_cache_t *get_microdesc_cache_noload(void);
  48. /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
  49. static inline unsigned int
  50. microdesc_hash_(microdesc_t *md)
  51. {
  52. return (unsigned) siphash24g(md->digest, sizeof(md->digest));
  53. }
  54. /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
  55. * purposes. */
  56. static inline int
  57. microdesc_eq_(microdesc_t *a, microdesc_t *b)
  58. {
  59. return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
  60. }
  61. HT_PROTOTYPE(microdesc_map, microdesc_t, node,
  62. microdesc_hash_, microdesc_eq_);
  63. HT_GENERATE2(microdesc_map, microdesc_t, node,
  64. microdesc_hash_, microdesc_eq_, 0.6,
  65. tor_reallocarray_, tor_free_)
  66. /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  67. * On success, return the total number of bytes written, and set
  68. * *<b>annotation_len_out</b> to the number of bytes written as
  69. * annotations. */
  70. static ssize_t
  71. dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
  72. {
  73. ssize_t r = 0;
  74. ssize_t written;
  75. if (md->body == NULL) {
  76. *annotation_len_out = 0;
  77. return 0;
  78. }
  79. /* XXXX drops unknown annotations. */
  80. if (md->last_listed) {
  81. char buf[ISO_TIME_LEN+1];
  82. char annotation[ISO_TIME_LEN+32];
  83. format_iso_time(buf, md->last_listed);
  84. tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
  85. if (write_all(fd, annotation, strlen(annotation), 0) < 0) {
  86. log_warn(LD_DIR,
  87. "Couldn't write microdescriptor annotation: %s",
  88. strerror(errno));
  89. return -1;
  90. }
  91. r += strlen(annotation);
  92. *annotation_len_out = r;
  93. } else {
  94. *annotation_len_out = 0;
  95. }
  96. md->off = tor_fd_getpos(fd);
  97. written = write_all(fd, md->body, md->bodylen, 0);
  98. if (written != (ssize_t)md->bodylen) {
  99. log_warn(LD_DIR,
  100. "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
  101. (long)written, (unsigned long)md->bodylen,
  102. strerror(errno));
  103. return -1;
  104. }
  105. r += md->bodylen;
  106. return r;
  107. }
  108. /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
  109. * such object has been allocated. */
  110. static microdesc_cache_t *the_microdesc_cache = NULL;
  111. /** Return a pointer to the microdescriptor cache, loading it if necessary. */
  112. microdesc_cache_t *
  113. get_microdesc_cache(void)
  114. {
  115. microdesc_cache_t *cache = get_microdesc_cache_noload();
  116. if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
  117. microdesc_cache_reload(cache);
  118. }
  119. return cache;
  120. }
  121. /** Return a pointer to the microdescriptor cache, creating (but not loading)
  122. * it if necessary. */
  123. static microdesc_cache_t *
  124. get_microdesc_cache_noload(void)
  125. {
  126. if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
  127. microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
  128. HT_INIT(microdesc_map, &cache->map);
  129. cache->cache_fname = get_datadir_fname("cached-microdescs");
  130. cache->journal_fname = get_datadir_fname("cached-microdescs.new");
  131. the_microdesc_cache = cache;
  132. }
  133. return the_microdesc_cache;
  134. }
  135. /* There are three sources of microdescriptors:
  136. 1) Generated by us while acting as a directory authority.
  137. 2) Loaded from the cache on disk.
  138. 3) Downloaded.
  139. */
  140. /** Decode the microdescriptors from the string starting at <b>s</b> and
  141. * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
  142. * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
  143. * leave their bodies as pointers to the mmap'd cache. If where is
  144. * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
  145. * set the last_listed field of every microdesc to listed_at. If
  146. * requested_digests is non-null, then it contains a list of digests we mean
  147. * to allow, so we should reject any non-requested microdesc with a different
  148. * digest, and alter the list to contain only the digests of those microdescs
  149. * we didn't find.
  150. * Return a newly allocated list of the added microdescriptors, or NULL */
  151. smartlist_t *
  152. microdescs_add_to_cache(microdesc_cache_t *cache,
  153. const char *s, const char *eos, saved_location_t where,
  154. int no_save, time_t listed_at,
  155. smartlist_t *requested_digests256)
  156. {
  157. void * const DIGEST_REQUESTED = (void*)1;
  158. void * const DIGEST_RECEIVED = (void*)2;
  159. void * const DIGEST_INVALID = (void*)3;
  160. smartlist_t *descriptors, *added;
  161. const int allow_annotations = (where != SAVED_NOWHERE);
  162. smartlist_t *invalid_digests = smartlist_new();
  163. descriptors = microdescs_parse_from_string(s, eos,
  164. allow_annotations,
  165. where, invalid_digests);
  166. if (listed_at != (time_t)-1) {
  167. SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
  168. md->last_listed = listed_at);
  169. }
  170. if (requested_digests256) {
  171. digest256map_t *requested;
  172. requested = digest256map_new();
  173. /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
  174. SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
  175. digest256map_set(requested, cp, DIGEST_REQUESTED));
  176. /* Set requested[d] to DIGEST_INVALID for every md we requested which we
  177. * will never be able to parse. Remove the ones we didn't request from
  178. * invalid_digests.
  179. */
  180. SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
  181. if (digest256map_get(requested, cp)) {
  182. digest256map_set(requested, cp, DIGEST_INVALID);
  183. } else {
  184. tor_free(cp);
  185. SMARTLIST_DEL_CURRENT(invalid_digests, cp);
  186. }
  187. } SMARTLIST_FOREACH_END(cp);
  188. /* Update requested[d] to 2 for the mds we asked for and got. Delete the
  189. * ones we never requested from the 'descriptors' smartlist.
  190. */
  191. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  192. if (digest256map_get(requested, (const uint8_t*)md->digest)) {
  193. digest256map_set(requested, (const uint8_t*)md->digest,
  194. DIGEST_RECEIVED);
  195. } else {
  196. log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
  197. microdesc_free(md);
  198. SMARTLIST_DEL_CURRENT(descriptors, md);
  199. }
  200. } SMARTLIST_FOREACH_END(md);
  201. /* Remove the ones we got or the invalid ones from requested_digests256.
  202. */
  203. SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
  204. void *status = digest256map_get(requested, cp);
  205. if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
  206. tor_free(cp);
  207. SMARTLIST_DEL_CURRENT(requested_digests256, cp);
  208. }
  209. } SMARTLIST_FOREACH_END(cp);
  210. digest256map_free(requested, NULL);
  211. }
  212. /* For every requested microdescriptor that was unparseable, mark it
  213. * as not to be retried. */
  214. if (smartlist_len(invalid_digests)) {
  215. networkstatus_t *ns =
  216. networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
  217. if (ns) {
  218. SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
  219. routerstatus_t *rs =
  220. router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
  221. if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
  222. download_status_mark_impossible(&rs->dl_status);
  223. }
  224. } SMARTLIST_FOREACH_END(d);
  225. }
  226. }
  227. SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
  228. smartlist_free(invalid_digests);
  229. added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
  230. smartlist_free(descriptors);
  231. return added;
  232. }
  233. /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
  234. * a string to decode. Frees any members of <b>descriptors</b> that it does
  235. * not add. */
  236. smartlist_t *
  237. microdescs_add_list_to_cache(microdesc_cache_t *cache,
  238. smartlist_t *descriptors, saved_location_t where,
  239. int no_save)
  240. {
  241. smartlist_t *added;
  242. open_file_t *open_file = NULL;
  243. int fd = -1;
  244. // int n_added = 0;
  245. ssize_t size = 0;
  246. if (where == SAVED_NOWHERE && !no_save) {
  247. fd = start_writing_to_file(cache->journal_fname,
  248. OPEN_FLAGS_APPEND|O_BINARY,
  249. 0600, &open_file);
  250. if (fd < 0) {
  251. log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
  252. cache->journal_fname, strerror(errno));
  253. }
  254. }
  255. added = smartlist_new();
  256. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  257. microdesc_t *md2;
  258. md2 = HT_FIND(microdesc_map, &cache->map, md);
  259. if (md2) {
  260. /* We already had this one. */
  261. if (md2->last_listed < md->last_listed)
  262. md2->last_listed = md->last_listed;
  263. microdesc_free(md);
  264. if (where != SAVED_NOWHERE)
  265. cache->bytes_dropped += size;
  266. continue;
  267. }
  268. /* Okay, it's a new one. */
  269. if (fd >= 0) {
  270. size_t annotation_len;
  271. size = dump_microdescriptor(fd, md, &annotation_len);
  272. if (size < 0) {
  273. /* we already warned in dump_microdescriptor */
  274. abort_writing_to_file(open_file);
  275. fd = -1;
  276. } else {
  277. md->saved_location = SAVED_IN_JOURNAL;
  278. cache->journal_len += size;
  279. }
  280. } else {
  281. md->saved_location = where;
  282. }
  283. md->no_save = no_save;
  284. HT_INSERT(microdesc_map, &cache->map, md);
  285. md->held_in_map = 1;
  286. smartlist_add(added, md);
  287. ++cache->n_seen;
  288. cache->total_len_seen += md->bodylen;
  289. } SMARTLIST_FOREACH_END(md);
  290. if (fd >= 0) {
  291. if (finish_writing_to_file(open_file) < 0) {
  292. log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
  293. strerror(errno));
  294. smartlist_clear(added);
  295. return added;
  296. }
  297. }
  298. {
  299. networkstatus_t *ns = networkstatus_get_latest_consensus();
  300. if (ns && ns->flavor == FLAV_MICRODESC)
  301. SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
  302. }
  303. if (smartlist_len(added))
  304. router_dir_info_changed();
  305. return added;
  306. }
  307. /** Remove every microdescriptor in <b>cache</b>. */
  308. void
  309. microdesc_cache_clear(microdesc_cache_t *cache)
  310. {
  311. microdesc_t **entry, **next;
  312. for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
  313. microdesc_t *md = *entry;
  314. next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
  315. md->held_in_map = 0;
  316. microdesc_free(md);
  317. }
  318. HT_CLEAR(microdesc_map, &cache->map);
  319. if (cache->cache_content) {
  320. int res = tor_munmap_file(cache->cache_content);
  321. if (res != 0) {
  322. log_warn(LD_FS,
  323. "tor_munmap_file() failed clearing microdesc cache; "
  324. "we are probably about to leak memory.");
  325. /* TODO something smarter? */
  326. }
  327. cache->cache_content = NULL;
  328. }
  329. cache->total_len_seen = 0;
  330. cache->n_seen = 0;
  331. cache->bytes_dropped = 0;
  332. }
  333. /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
  334. * for the first time. Return 0 on success, -1 on failure. */
  335. int
  336. microdesc_cache_reload(microdesc_cache_t *cache)
  337. {
  338. struct stat st;
  339. char *journal_content;
  340. smartlist_t *added;
  341. tor_mmap_t *mm;
  342. int total = 0;
  343. microdesc_cache_clear(cache);
  344. cache->is_loaded = 1;
  345. mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
  346. if (mm) {
  347. added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
  348. SAVED_IN_CACHE, 0, -1, NULL);
  349. if (added) {
  350. total += smartlist_len(added);
  351. smartlist_free(added);
  352. }
  353. }
  354. journal_content = read_file_to_str(cache->journal_fname,
  355. RFTS_IGNORE_MISSING, &st);
  356. if (journal_content) {
  357. cache->journal_len = (size_t) st.st_size;
  358. added = microdescs_add_to_cache(cache, journal_content,
  359. journal_content+st.st_size,
  360. SAVED_IN_JOURNAL, 0, -1, NULL);
  361. if (added) {
  362. total += smartlist_len(added);
  363. smartlist_free(added);
  364. }
  365. tor_free(journal_content);
  366. }
  367. log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
  368. total);
  369. microdesc_cache_rebuild(cache, 0 /* don't force */);
  370. return 0;
  371. }
  372. /** By default, we remove any microdescriptors that have gone at least this
  373. * long without appearing in a current consensus. */
  374. #define TOLERATE_MICRODESC_AGE (7*24*60*60)
  375. /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
  376. * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
  377. * positive, specifically remove microdescriptors that have been unlisted
  378. * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
  379. * if we have no current live microdescriptor consensus.
  380. */
  381. void
  382. microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
  383. {
  384. microdesc_t **mdp, *victim;
  385. int dropped=0, kept=0;
  386. size_t bytes_dropped = 0;
  387. time_t now = time(NULL);
  388. /* If we don't know a live consensus, don't believe last_listed values: we
  389. * might be starting up after being down for a while. */
  390. if (! force &&
  391. ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
  392. return;
  393. if (cutoff <= 0)
  394. cutoff = now - TOLERATE_MICRODESC_AGE;
  395. for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
  396. const int is_old = (*mdp)->last_listed < cutoff;
  397. const unsigned held_by_nodes = (*mdp)->held_by_nodes;
  398. if (is_old && !held_by_nodes) {
  399. ++dropped;
  400. victim = *mdp;
  401. mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
  402. victim->held_in_map = 0;
  403. bytes_dropped += victim->bodylen;
  404. microdesc_free(victim);
  405. } else {
  406. if (is_old) {
  407. /* It's old, but it has held_by_nodes set. That's not okay. */
  408. /* Let's try to diagnose and fix #7164 . */
  409. smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
  410. const networkstatus_t *ns = networkstatus_get_latest_consensus();
  411. long networkstatus_age = -1;
  412. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  413. if (ns) {
  414. networkstatus_age = now - ns->valid_after;
  415. }
  416. log_warn(LD_BUG, "Microdescriptor seemed very old "
  417. "(last listed %d hours ago vs %d hour cutoff), but is still "
  418. "marked as being held by %d node(s). I found %d node(s) "
  419. "holding it. Current networkstatus is %ld hours old. "
  420. "Hashtable badness is %d.",
  421. (int)((now - (*mdp)->last_listed) / 3600),
  422. (int)((now - cutoff) / 3600),
  423. held_by_nodes,
  424. smartlist_len(nodes),
  425. networkstatus_age / 3600,
  426. ht_badness);
  427. SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
  428. const char *rs_match = "No RS";
  429. const char *rs_present = "";
  430. if (node->rs) {
  431. if (tor_memeq(node->rs->descriptor_digest,
  432. (*mdp)->digest, DIGEST256_LEN)) {
  433. rs_match = "Microdesc digest in RS matches";
  434. } else {
  435. rs_match = "Microdesc digest in RS does match";
  436. }
  437. if (ns) {
  438. /* This should be impossible, but let's see! */
  439. rs_present = " RS not present in networkstatus.";
  440. SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
  441. if (rs == node->rs) {
  442. rs_present = " RS okay in networkstatus.";
  443. }
  444. });
  445. }
  446. }
  447. log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
  448. node_sl_idx,
  449. hex_str(node->identity, DIGEST_LEN),
  450. node->md, node->rs, node->ri, rs_match, rs_present);
  451. } SMARTLIST_FOREACH_END(node);
  452. smartlist_free(nodes);
  453. (*mdp)->last_listed = now;
  454. }
  455. ++kept;
  456. mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
  457. }
  458. }
  459. if (dropped) {
  460. log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
  461. dropped,dropped+kept);
  462. cache->bytes_dropped += bytes_dropped;
  463. }
  464. }
  465. static int
  466. should_rebuild_md_cache(microdesc_cache_t *cache)
  467. {
  468. const size_t old_len =
  469. cache->cache_content ? cache->cache_content->size : 0;
  470. const size_t journal_len = cache->journal_len;
  471. const size_t dropped = cache->bytes_dropped;
  472. if (journal_len < 16384)
  473. return 0; /* Don't bother, not enough has happened yet. */
  474. if (dropped > (journal_len + old_len) / 3)
  475. return 1; /* We could save 1/3 or more of the currently used space. */
  476. if (journal_len > old_len / 2)
  477. return 1; /* We should append to the regular file */
  478. return 0;
  479. }
  480. /**
  481. * Mark <b>md</b> as having no body, and release any storage previously held
  482. * by its body.
  483. */
  484. static void
  485. microdesc_wipe_body(microdesc_t *md)
  486. {
  487. if (!md)
  488. return;
  489. if (md->saved_location != SAVED_IN_CACHE)
  490. tor_free(md->body);
  491. md->off = 0;
  492. md->saved_location = SAVED_NOWHERE;
  493. md->body = NULL;
  494. md->bodylen = 0;
  495. md->no_save = 1;
  496. }
  497. /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  498. * and update every microdesc_t in the cache with pointers to its new
  499. * location. If <b>force</b> is true, do this unconditionally. If
  500. * <b>force</b> is false, do it only if we expect to save space on disk. */
  501. int
  502. microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
  503. {
  504. open_file_t *open_file;
  505. int fd = -1, res;
  506. microdesc_t **mdp;
  507. smartlist_t *wrote;
  508. ssize_t size;
  509. off_t off = 0, off_real;
  510. int orig_size, new_size;
  511. if (cache == NULL) {
  512. cache = the_microdesc_cache;
  513. if (cache == NULL)
  514. return 0;
  515. }
  516. /* Remove dead descriptors */
  517. microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
  518. if (!force && !should_rebuild_md_cache(cache))
  519. return 0;
  520. log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
  521. orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
  522. orig_size += (int)cache->journal_len;
  523. fd = start_writing_to_file(cache->cache_fname,
  524. OPEN_FLAGS_REPLACE|O_BINARY,
  525. 0600, &open_file);
  526. if (fd < 0)
  527. return -1;
  528. wrote = smartlist_new();
  529. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  530. microdesc_t *md = *mdp;
  531. size_t annotation_len;
  532. if (md->no_save || !md->body)
  533. continue;
  534. size = dump_microdescriptor(fd, md, &annotation_len);
  535. if (size < 0) {
  536. microdesc_wipe_body(md);
  537. /* rewind, in case it was a partial write. */
  538. tor_fd_setpos(fd, off);
  539. continue;
  540. }
  541. tor_assert(((size_t)size) == annotation_len + md->bodylen);
  542. md->off = off + annotation_len;
  543. off += size;
  544. off_real = tor_fd_getpos(fd);
  545. if (off_real != off) {
  546. log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
  547. "By my count, I'm at "I64_FORMAT
  548. ", but I should be at "I64_FORMAT,
  549. I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real));
  550. if (off_real >= 0)
  551. off = off_real;
  552. }
  553. if (md->saved_location != SAVED_IN_CACHE) {
  554. tor_free(md->body);
  555. md->saved_location = SAVED_IN_CACHE;
  556. }
  557. smartlist_add(wrote, md);
  558. }
  559. /* We must do this unmap _before_ we call finish_writing_to_file(), or
  560. * windows will not actually replace the file. */
  561. if (cache->cache_content) {
  562. res = tor_munmap_file(cache->cache_content);
  563. if (res != 0) {
  564. log_warn(LD_FS,
  565. "Failed to unmap old microdescriptor cache while rebuilding");
  566. }
  567. cache->cache_content = NULL;
  568. }
  569. if (finish_writing_to_file(open_file) < 0) {
  570. log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
  571. strerror(errno));
  572. /* Okay. Let's prevent from making things worse elsewhere. */
  573. cache->cache_content = NULL;
  574. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  575. microdesc_t *md = *mdp;
  576. if (md->saved_location == SAVED_IN_CACHE) {
  577. microdesc_wipe_body(md);
  578. }
  579. }
  580. smartlist_free(wrote);
  581. return -1;
  582. }
  583. cache->cache_content = tor_mmap_file(cache->cache_fname);
  584. if (!cache->cache_content && smartlist_len(wrote)) {
  585. log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
  586. cache->cache_fname);
  587. smartlist_free(wrote);
  588. return -1;
  589. }
  590. SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
  591. tor_assert(md->saved_location == SAVED_IN_CACHE);
  592. md->body = (char*)cache->cache_content->data + md->off;
  593. if (PREDICT_UNLIKELY(
  594. md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
  595. /* XXXX once bug 2022 is solved, we can kill this block and turn it
  596. * into just the tor_assert(fast_memeq) */
  597. off_t avail = cache->cache_content->size - md->off;
  598. char *bad_str;
  599. tor_assert(avail >= 0);
  600. bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
  601. log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
  602. " At offset %d, I expected to find a microdescriptor starting "
  603. " with \"onion-key\". Instead I got %s.",
  604. (int)md->off, escaped(bad_str));
  605. tor_free(bad_str);
  606. tor_assert(fast_memeq(md->body, "onion-key", 9));
  607. }
  608. } SMARTLIST_FOREACH_END(md);
  609. smartlist_free(wrote);
  610. write_str_to_file(cache->journal_fname, "", 1);
  611. cache->journal_len = 0;
  612. cache->bytes_dropped = 0;
  613. new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
  614. log_info(LD_DIR, "Done rebuilding microdesc cache. "
  615. "Saved %d bytes; %d still used.",
  616. orig_size-new_size, new_size);
  617. return 0;
  618. }
  619. /** Make sure that the reference count of every microdescriptor in cache is
  620. * accurate. */
  621. void
  622. microdesc_check_counts(void)
  623. {
  624. microdesc_t **mdp;
  625. if (!the_microdesc_cache)
  626. return;
  627. HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
  628. microdesc_t *md = *mdp;
  629. unsigned int found=0;
  630. const smartlist_t *nodes = nodelist_get_list();
  631. SMARTLIST_FOREACH(nodes, node_t *, node, {
  632. if (node->md == md) {
  633. ++found;
  634. }
  635. });
  636. tor_assert(found == md->held_by_nodes);
  637. }
  638. }
  639. /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
  640. * previously been removed from the cache if it had ever been inserted. */
  641. void
  642. microdesc_free_(microdesc_t *md, const char *fname, int lineno)
  643. {
  644. if (!md)
  645. return;
  646. /* Make sure that the microdesc was really removed from the appropriate data
  647. structures. */
  648. if (md->held_in_map) {
  649. microdesc_cache_t *cache = get_microdesc_cache_noload();
  650. microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
  651. if (md2 == md) {
  652. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  653. "in microdesc_map", fname, lineno);
  654. HT_REMOVE(microdesc_map, &cache->map, md);
  655. } else {
  656. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
  657. "set, but microdesc was not in the map.", fname, lineno);
  658. }
  659. tor_fragile_assert();
  660. }
  661. if (md->held_by_nodes) {
  662. microdesc_cache_t *cache = get_microdesc_cache_noload();
  663. int found=0;
  664. const smartlist_t *nodes = nodelist_get_list();
  665. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  666. SMARTLIST_FOREACH(nodes, node_t *, node, {
  667. if (node->md == md) {
  668. ++found;
  669. node->md = NULL;
  670. }
  671. });
  672. if (found) {
  673. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  674. "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
  675. fname, lineno, found, md->held_by_nodes, ht_badness);
  676. } else {
  677. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
  678. "set to %u, but md was not referenced by any nodes. "
  679. "ht_badness == %d",
  680. fname, lineno, md->held_by_nodes, ht_badness);
  681. }
  682. tor_fragile_assert();
  683. }
  684. //tor_assert(md->held_in_map == 0);
  685. //tor_assert(md->held_by_nodes == 0);
  686. if (md->onion_pkey)
  687. crypto_pk_free(md->onion_pkey);
  688. tor_free(md->onion_curve25519_pkey);
  689. tor_free(md->ed25519_identity_pkey);
  690. if (md->body && md->saved_location != SAVED_IN_CACHE)
  691. tor_free(md->body);
  692. if (md->family) {
  693. SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
  694. smartlist_free(md->family);
  695. }
  696. short_policy_free(md->exit_policy);
  697. short_policy_free(md->ipv6_exit_policy);
  698. tor_free(md);
  699. }
  700. /** Free all storage held in the microdesc.c module. */
  701. void
  702. microdesc_free_all(void)
  703. {
  704. if (the_microdesc_cache) {
  705. microdesc_cache_clear(the_microdesc_cache);
  706. tor_free(the_microdesc_cache->cache_fname);
  707. tor_free(the_microdesc_cache->journal_fname);
  708. tor_free(the_microdesc_cache);
  709. }
  710. }
  711. /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
  712. * <b>d</b>, return it. Otherwise return NULL. */
  713. microdesc_t *
  714. microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
  715. {
  716. microdesc_t *md, search;
  717. if (!cache)
  718. cache = get_microdesc_cache();
  719. memcpy(search.digest, d, DIGEST256_LEN);
  720. md = HT_FIND(microdesc_map, &cache->map, &search);
  721. return md;
  722. }
  723. /** Return the mean size of decriptors added to <b>cache</b> since it was last
  724. * cleared. Used to estimate the size of large downloads. */
  725. size_t
  726. microdesc_average_size(microdesc_cache_t *cache)
  727. {
  728. if (!cache)
  729. cache = get_microdesc_cache();
  730. if (!cache->n_seen)
  731. return 512;
  732. return (size_t)(cache->total_len_seen / cache->n_seen);
  733. }
  734. /** Return a smartlist of all the sha256 digest of the microdescriptors that
  735. * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  736. * to internals of <b>ns</b>; you should not free the members of the resulting
  737. * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
  738. smartlist_t *
  739. microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
  740. int downloadable_only, digest256map_t *skip)
  741. {
  742. smartlist_t *result = smartlist_new();
  743. time_t now = time(NULL);
  744. tor_assert(ns->flavor == FLAV_MICRODESC);
  745. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  746. if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
  747. continue;
  748. if (downloadable_only &&
  749. !download_status_is_ready(&rs->dl_status, now,
  750. get_options()->TestingMicrodescMaxDownloadTries))
  751. continue;
  752. if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
  753. continue;
  754. if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
  755. continue;
  756. /* XXXX Also skip if we're a noncache and wouldn't use this router.
  757. * XXXX NM Microdesc
  758. */
  759. smartlist_add(result, rs->descriptor_digest);
  760. } SMARTLIST_FOREACH_END(rs);
  761. return result;
  762. }
  763. /** Launch download requests for microdescriptors as appropriate.
  764. *
  765. * Specifically, we should launch download requests if we are configured to
  766. * download mirodescriptors, and there are some microdescriptors listed in the
  767. * current microdesc consensus that we don't have, and either we never asked
  768. * for them, or we failed to download them but we're willing to retry.
  769. */
  770. void
  771. update_microdesc_downloads(time_t now)
  772. {
  773. const or_options_t *options = get_options();
  774. networkstatus_t *consensus;
  775. smartlist_t *missing;
  776. digest256map_t *pending;
  777. if (should_delay_dir_fetches(options, NULL))
  778. return;
  779. if (directory_too_idle_to_fetch_descriptors(options, now))
  780. return;
  781. consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  782. if (!consensus)
  783. return;
  784. if (!we_fetch_microdescriptors(options))
  785. return;
  786. pending = digest256map_new();
  787. list_pending_microdesc_downloads(pending);
  788. missing = microdesc_list_missing_digest256(consensus,
  789. get_microdesc_cache(),
  790. 1,
  791. pending);
  792. digest256map_free(pending, NULL);
  793. launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
  794. missing, NULL, now);
  795. smartlist_free(missing);
  796. }
  797. /** For every microdescriptor listed in the current microdecriptor consensus,
  798. * update its last_listed field to be at least as recent as the publication
  799. * time of the current microdescriptor consensus.
  800. */
  801. void
  802. update_microdescs_from_networkstatus(time_t now)
  803. {
  804. microdesc_cache_t *cache = get_microdesc_cache();
  805. microdesc_t *md;
  806. networkstatus_t *ns =
  807. networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  808. if (! ns)
  809. return;
  810. tor_assert(ns->flavor == FLAV_MICRODESC);
  811. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  812. md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
  813. if (md && ns->valid_after > md->last_listed)
  814. md->last_listed = ns->valid_after;
  815. } SMARTLIST_FOREACH_END(rs);
  816. }
  817. /** Return true iff we should prefer to use microdescriptors rather than
  818. * routerdescs for building circuits. */
  819. int
  820. we_use_microdescriptors_for_circuits(const or_options_t *options)
  821. {
  822. int ret = options->UseMicrodescriptors;
  823. if (ret == -1) {
  824. /* UseMicrodescriptors is "auto"; we need to decide: */
  825. /* If we are configured to use bridges and none of our bridges
  826. * know what a microdescriptor is, the answer is no. */
  827. if (options->UseBridges && !any_bridge_supports_microdescriptors())
  828. return 0;
  829. /* Otherwise, we decide that we'll use microdescriptors iff we are
  830. * not a server, and we're not autofetching everything. */
  831. /* XXX023 what does not being a server have to do with it? also there's
  832. * a partitioning issue here where bridges differ from clients. */
  833. ret = !server_mode(options) && !options->FetchUselessDescriptors;
  834. }
  835. return ret;
  836. }
  837. /** Return true iff we should try to download microdescriptors at all. */
  838. int
  839. we_fetch_microdescriptors(const or_options_t *options)
  840. {
  841. if (directory_caches_dir_info(options))
  842. return 1;
  843. if (options->FetchUselessDescriptors)
  844. return 1;
  845. return we_use_microdescriptors_for_circuits(options);
  846. }
  847. /** Return true iff we should try to download router descriptors at all. */
  848. int
  849. we_fetch_router_descriptors(const or_options_t *options)
  850. {
  851. if (directory_caches_dir_info(options))
  852. return 1;
  853. if (options->FetchUselessDescriptors)
  854. return 1;
  855. return ! we_use_microdescriptors_for_circuits(options);
  856. }
  857. /** Return the consensus flavor we actually want to use to build circuits. */
  858. MOCK_IMPL(int,
  859. usable_consensus_flavor,(void))
  860. {
  861. if (we_use_microdescriptors_for_circuits(get_options())) {
  862. return FLAV_MICRODESC;
  863. } else {
  864. return FLAV_NS;
  865. }
  866. }