microdesc.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. /* Copyright (c) 2009-2016, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file microdesc.c
  5. *
  6. * \brief Implements microdescriptors -- an abbreviated description of
  7. * less-frequently-changing router information.
  8. */
  9. #include "or.h"
  10. #include "circuitbuild.h"
  11. #include "config.h"
  12. #include "directory.h"
  13. #include "dirserv.h"
  14. #include "entrynodes.h"
  15. #include "microdesc.h"
  16. #include "networkstatus.h"
  17. #include "nodelist.h"
  18. #include "policies.h"
  19. #include "router.h"
  20. #include "routerlist.h"
  21. #include "routerparse.h"
  22. /** A data structure to hold a bunch of cached microdescriptors. There are
  23. * two active files in the cache: a "cache file" that we mmap, and a "journal
  24. * file" that we append to. Periodically, we rebuild the cache file to hold
  25. * only the microdescriptors that we want to keep */
  26. struct microdesc_cache_t {
  27. /** Map from sha256-digest to microdesc_t for every microdesc_t in the
  28. * cache. */
  29. HT_HEAD(microdesc_map, microdesc_t) map;
  30. /** Name of the cache file. */
  31. char *cache_fname;
  32. /** Name of the journal file. */
  33. char *journal_fname;
  34. /** Mmap'd contents of the cache file, or NULL if there is none. */
  35. tor_mmap_t *cache_content;
  36. /** Number of bytes used in the journal file. */
  37. size_t journal_len;
  38. /** Number of bytes in descriptors removed as too old. */
  39. size_t bytes_dropped;
  40. /** Total bytes of microdescriptor bodies we have added to this cache */
  41. uint64_t total_len_seen;
  42. /** Total number of microdescriptors we have added to this cache */
  43. unsigned n_seen;
  44. /** True iff we have loaded this cache from disk ever. */
  45. int is_loaded;
  46. };
  47. static microdesc_cache_t *get_microdesc_cache_noload(void);
  48. /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
  49. static inline unsigned int
  50. microdesc_hash_(microdesc_t *md)
  51. {
  52. return (unsigned) siphash24g(md->digest, sizeof(md->digest));
  53. }
  54. /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
  55. * purposes. */
  56. static inline int
  57. microdesc_eq_(microdesc_t *a, microdesc_t *b)
  58. {
  59. return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
  60. }
  61. HT_PROTOTYPE(microdesc_map, microdesc_t, node,
  62. microdesc_hash_, microdesc_eq_)
  63. HT_GENERATE2(microdesc_map, microdesc_t, node,
  64. microdesc_hash_, microdesc_eq_, 0.6,
  65. tor_reallocarray_, tor_free_)
  66. /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  67. * On success, return the total number of bytes written, and set
  68. * *<b>annotation_len_out</b> to the number of bytes written as
  69. * annotations. */
  70. static ssize_t
  71. dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
  72. {
  73. ssize_t r = 0;
  74. ssize_t written;
  75. if (md->body == NULL) {
  76. *annotation_len_out = 0;
  77. return 0;
  78. }
  79. /* XXXX drops unknown annotations. */
  80. if (md->last_listed) {
  81. char buf[ISO_TIME_LEN+1];
  82. char annotation[ISO_TIME_LEN+32];
  83. format_iso_time(buf, md->last_listed);
  84. tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
  85. if (write_all(fd, annotation, strlen(annotation), 0) < 0) {
  86. log_warn(LD_DIR,
  87. "Couldn't write microdescriptor annotation: %s",
  88. strerror(errno));
  89. return -1;
  90. }
  91. r += strlen(annotation);
  92. *annotation_len_out = r;
  93. } else {
  94. *annotation_len_out = 0;
  95. }
  96. md->off = tor_fd_getpos(fd);
  97. written = write_all(fd, md->body, md->bodylen, 0);
  98. if (written != (ssize_t)md->bodylen) {
  99. written = written < 0 ? 0 : written;
  100. log_warn(LD_DIR,
  101. "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
  102. (long)written, (unsigned long)md->bodylen,
  103. strerror(errno));
  104. return -1;
  105. }
  106. r += md->bodylen;
  107. return r;
  108. }
  109. /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
  110. * such object has been allocated. */
  111. static microdesc_cache_t *the_microdesc_cache = NULL;
  112. /** Return a pointer to the microdescriptor cache, loading it if necessary. */
  113. microdesc_cache_t *
  114. get_microdesc_cache(void)
  115. {
  116. microdesc_cache_t *cache = get_microdesc_cache_noload();
  117. if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
  118. microdesc_cache_reload(cache);
  119. }
  120. return cache;
  121. }
  122. /** Return a pointer to the microdescriptor cache, creating (but not loading)
  123. * it if necessary. */
  124. static microdesc_cache_t *
  125. get_microdesc_cache_noload(void)
  126. {
  127. if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
  128. microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
  129. HT_INIT(microdesc_map, &cache->map);
  130. cache->cache_fname = get_datadir_fname("cached-microdescs");
  131. cache->journal_fname = get_datadir_fname("cached-microdescs.new");
  132. the_microdesc_cache = cache;
  133. }
  134. return the_microdesc_cache;
  135. }
  136. /* There are three sources of microdescriptors:
  137. 1) Generated by us while acting as a directory authority.
  138. 2) Loaded from the cache on disk.
  139. 3) Downloaded.
  140. */
  141. /** Decode the microdescriptors from the string starting at <b>s</b> and
  142. * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
  143. * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
  144. * leave their bodies as pointers to the mmap'd cache. If where is
  145. * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
  146. * set the last_listed field of every microdesc to listed_at. If
  147. * requested_digests is non-null, then it contains a list of digests we mean
  148. * to allow, so we should reject any non-requested microdesc with a different
  149. * digest, and alter the list to contain only the digests of those microdescs
  150. * we didn't find.
  151. * Return a newly allocated list of the added microdescriptors, or NULL */
  152. smartlist_t *
  153. microdescs_add_to_cache(microdesc_cache_t *cache,
  154. const char *s, const char *eos, saved_location_t where,
  155. int no_save, time_t listed_at,
  156. smartlist_t *requested_digests256)
  157. {
  158. void * const DIGEST_REQUESTED = (void*)1;
  159. void * const DIGEST_RECEIVED = (void*)2;
  160. void * const DIGEST_INVALID = (void*)3;
  161. smartlist_t *descriptors, *added;
  162. const int allow_annotations = (where != SAVED_NOWHERE);
  163. smartlist_t *invalid_digests = smartlist_new();
  164. descriptors = microdescs_parse_from_string(s, eos,
  165. allow_annotations,
  166. where, invalid_digests);
  167. if (listed_at != (time_t)-1) {
  168. SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
  169. md->last_listed = listed_at);
  170. }
  171. if (requested_digests256) {
  172. digest256map_t *requested;
  173. requested = digest256map_new();
  174. /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
  175. SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
  176. digest256map_set(requested, cp, DIGEST_REQUESTED));
  177. /* Set requested[d] to DIGEST_INVALID for every md we requested which we
  178. * will never be able to parse. Remove the ones we didn't request from
  179. * invalid_digests.
  180. */
  181. SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
  182. if (digest256map_get(requested, cp)) {
  183. digest256map_set(requested, cp, DIGEST_INVALID);
  184. } else {
  185. tor_free(cp);
  186. SMARTLIST_DEL_CURRENT(invalid_digests, cp);
  187. }
  188. } SMARTLIST_FOREACH_END(cp);
  189. /* Update requested[d] to 2 for the mds we asked for and got. Delete the
  190. * ones we never requested from the 'descriptors' smartlist.
  191. */
  192. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  193. if (digest256map_get(requested, (const uint8_t*)md->digest)) {
  194. digest256map_set(requested, (const uint8_t*)md->digest,
  195. DIGEST_RECEIVED);
  196. } else {
  197. log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
  198. microdesc_free(md);
  199. SMARTLIST_DEL_CURRENT(descriptors, md);
  200. }
  201. } SMARTLIST_FOREACH_END(md);
  202. /* Remove the ones we got or the invalid ones from requested_digests256.
  203. */
  204. SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
  205. void *status = digest256map_get(requested, cp);
  206. if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
  207. tor_free(cp);
  208. SMARTLIST_DEL_CURRENT(requested_digests256, cp);
  209. }
  210. } SMARTLIST_FOREACH_END(cp);
  211. digest256map_free(requested, NULL);
  212. }
  213. /* For every requested microdescriptor that was unparseable, mark it
  214. * as not to be retried. */
  215. if (smartlist_len(invalid_digests)) {
  216. networkstatus_t *ns =
  217. networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
  218. if (ns) {
  219. SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
  220. routerstatus_t *rs =
  221. router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
  222. if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
  223. download_status_mark_impossible(&rs->dl_status);
  224. }
  225. } SMARTLIST_FOREACH_END(d);
  226. }
  227. }
  228. SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
  229. smartlist_free(invalid_digests);
  230. added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
  231. smartlist_free(descriptors);
  232. return added;
  233. }
  234. /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
  235. * a string to decode. Frees any members of <b>descriptors</b> that it does
  236. * not add. */
  237. smartlist_t *
  238. microdescs_add_list_to_cache(microdesc_cache_t *cache,
  239. smartlist_t *descriptors, saved_location_t where,
  240. int no_save)
  241. {
  242. smartlist_t *added;
  243. open_file_t *open_file = NULL;
  244. int fd = -1;
  245. // int n_added = 0;
  246. ssize_t size = 0;
  247. if (where == SAVED_NOWHERE && !no_save) {
  248. fd = start_writing_to_file(cache->journal_fname,
  249. OPEN_FLAGS_APPEND|O_BINARY,
  250. 0600, &open_file);
  251. if (fd < 0) {
  252. log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
  253. cache->journal_fname, strerror(errno));
  254. }
  255. }
  256. added = smartlist_new();
  257. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  258. microdesc_t *md2;
  259. md2 = HT_FIND(microdesc_map, &cache->map, md);
  260. if (md2) {
  261. /* We already had this one. */
  262. if (md2->last_listed < md->last_listed)
  263. md2->last_listed = md->last_listed;
  264. microdesc_free(md);
  265. if (where != SAVED_NOWHERE)
  266. cache->bytes_dropped += size;
  267. continue;
  268. }
  269. /* Okay, it's a new one. */
  270. if (fd >= 0) {
  271. size_t annotation_len;
  272. size = dump_microdescriptor(fd, md, &annotation_len);
  273. if (size < 0) {
  274. /* we already warned in dump_microdescriptor */
  275. abort_writing_to_file(open_file);
  276. fd = -1;
  277. } else {
  278. md->saved_location = SAVED_IN_JOURNAL;
  279. cache->journal_len += size;
  280. }
  281. } else {
  282. md->saved_location = where;
  283. }
  284. md->no_save = no_save;
  285. HT_INSERT(microdesc_map, &cache->map, md);
  286. md->held_in_map = 1;
  287. smartlist_add(added, md);
  288. ++cache->n_seen;
  289. cache->total_len_seen += md->bodylen;
  290. } SMARTLIST_FOREACH_END(md);
  291. if (fd >= 0) {
  292. if (finish_writing_to_file(open_file) < 0) {
  293. log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
  294. strerror(errno));
  295. smartlist_clear(added);
  296. return added;
  297. }
  298. }
  299. {
  300. networkstatus_t *ns = networkstatus_get_latest_consensus();
  301. if (ns && ns->flavor == FLAV_MICRODESC)
  302. SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
  303. }
  304. if (smartlist_len(added))
  305. router_dir_info_changed();
  306. return added;
  307. }
  308. /** Remove every microdescriptor in <b>cache</b>. */
  309. void
  310. microdesc_cache_clear(microdesc_cache_t *cache)
  311. {
  312. microdesc_t **entry, **next;
  313. for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
  314. microdesc_t *md = *entry;
  315. next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
  316. md->held_in_map = 0;
  317. microdesc_free(md);
  318. }
  319. HT_CLEAR(microdesc_map, &cache->map);
  320. if (cache->cache_content) {
  321. int res = tor_munmap_file(cache->cache_content);
  322. if (res != 0) {
  323. log_warn(LD_FS,
  324. "tor_munmap_file() failed clearing microdesc cache; "
  325. "we are probably about to leak memory.");
  326. /* TODO something smarter? */
  327. }
  328. cache->cache_content = NULL;
  329. }
  330. cache->total_len_seen = 0;
  331. cache->n_seen = 0;
  332. cache->bytes_dropped = 0;
  333. }
  334. /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
  335. * for the first time. Return 0 on success, -1 on failure. */
  336. int
  337. microdesc_cache_reload(microdesc_cache_t *cache)
  338. {
  339. struct stat st;
  340. char *journal_content;
  341. smartlist_t *added;
  342. tor_mmap_t *mm;
  343. int total = 0;
  344. microdesc_cache_clear(cache);
  345. cache->is_loaded = 1;
  346. mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
  347. if (mm) {
  348. added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
  349. SAVED_IN_CACHE, 0, -1, NULL);
  350. if (added) {
  351. total += smartlist_len(added);
  352. smartlist_free(added);
  353. }
  354. }
  355. journal_content = read_file_to_str(cache->journal_fname,
  356. RFTS_IGNORE_MISSING, &st);
  357. if (journal_content) {
  358. cache->journal_len = (size_t) st.st_size;
  359. added = microdescs_add_to_cache(cache, journal_content,
  360. journal_content+st.st_size,
  361. SAVED_IN_JOURNAL, 0, -1, NULL);
  362. if (added) {
  363. total += smartlist_len(added);
  364. smartlist_free(added);
  365. }
  366. tor_free(journal_content);
  367. }
  368. log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
  369. total);
  370. microdesc_cache_rebuild(cache, 0 /* don't force */);
  371. return 0;
  372. }
  373. /** By default, we remove any microdescriptors that have gone at least this
  374. * long without appearing in a current consensus. */
  375. #define TOLERATE_MICRODESC_AGE (7*24*60*60)
  376. /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
  377. * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
  378. * positive, specifically remove microdescriptors that have been unlisted
  379. * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
  380. * if we have no current live microdescriptor consensus.
  381. */
  382. void
  383. microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
  384. {
  385. microdesc_t **mdp, *victim;
  386. int dropped=0, kept=0;
  387. size_t bytes_dropped = 0;
  388. time_t now = time(NULL);
  389. /* If we don't know a live consensus, don't believe last_listed values: we
  390. * might be starting up after being down for a while. */
  391. if (! force &&
  392. ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
  393. return;
  394. if (cutoff <= 0)
  395. cutoff = now - TOLERATE_MICRODESC_AGE;
  396. for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
  397. const int is_old = (*mdp)->last_listed < cutoff;
  398. const unsigned held_by_nodes = (*mdp)->held_by_nodes;
  399. if (is_old && !held_by_nodes) {
  400. ++dropped;
  401. victim = *mdp;
  402. mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
  403. victim->held_in_map = 0;
  404. bytes_dropped += victim->bodylen;
  405. microdesc_free(victim);
  406. } else {
  407. if (is_old) {
  408. /* It's old, but it has held_by_nodes set. That's not okay. */
  409. /* Let's try to diagnose and fix #7164 . */
  410. smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
  411. const networkstatus_t *ns = networkstatus_get_latest_consensus();
  412. long networkstatus_age = -1;
  413. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  414. if (ns) {
  415. networkstatus_age = now - ns->valid_after;
  416. }
  417. log_warn(LD_BUG, "Microdescriptor seemed very old "
  418. "(last listed %d hours ago vs %d hour cutoff), but is still "
  419. "marked as being held by %d node(s). I found %d node(s) "
  420. "holding it. Current networkstatus is %ld hours old. "
  421. "Hashtable badness is %d.",
  422. (int)((now - (*mdp)->last_listed) / 3600),
  423. (int)((now - cutoff) / 3600),
  424. held_by_nodes,
  425. smartlist_len(nodes),
  426. networkstatus_age / 3600,
  427. ht_badness);
  428. SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
  429. const char *rs_match = "No RS";
  430. const char *rs_present = "";
  431. if (node->rs) {
  432. if (tor_memeq(node->rs->descriptor_digest,
  433. (*mdp)->digest, DIGEST256_LEN)) {
  434. rs_match = "Microdesc digest in RS matches";
  435. } else {
  436. rs_match = "Microdesc digest in RS does match";
  437. }
  438. if (ns) {
  439. /* This should be impossible, but let's see! */
  440. rs_present = " RS not present in networkstatus.";
  441. SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
  442. if (rs == node->rs) {
  443. rs_present = " RS okay in networkstatus.";
  444. }
  445. });
  446. }
  447. }
  448. log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
  449. node_sl_idx,
  450. hex_str(node->identity, DIGEST_LEN),
  451. node->md, node->rs, node->ri, rs_match, rs_present);
  452. } SMARTLIST_FOREACH_END(node);
  453. smartlist_free(nodes);
  454. (*mdp)->last_listed = now;
  455. }
  456. ++kept;
  457. mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
  458. }
  459. }
  460. if (dropped) {
  461. log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
  462. dropped,dropped+kept);
  463. cache->bytes_dropped += bytes_dropped;
  464. }
  465. }
  466. static int
  467. should_rebuild_md_cache(microdesc_cache_t *cache)
  468. {
  469. const size_t old_len =
  470. cache->cache_content ? cache->cache_content->size : 0;
  471. const size_t journal_len = cache->journal_len;
  472. const size_t dropped = cache->bytes_dropped;
  473. if (journal_len < 16384)
  474. return 0; /* Don't bother, not enough has happened yet. */
  475. if (dropped > (journal_len + old_len) / 3)
  476. return 1; /* We could save 1/3 or more of the currently used space. */
  477. if (journal_len > old_len / 2)
  478. return 1; /* We should append to the regular file */
  479. return 0;
  480. }
  481. /**
  482. * Mark <b>md</b> as having no body, and release any storage previously held
  483. * by its body.
  484. */
  485. static void
  486. microdesc_wipe_body(microdesc_t *md)
  487. {
  488. if (!md)
  489. return;
  490. if (md->saved_location != SAVED_IN_CACHE)
  491. tor_free(md->body);
  492. md->off = 0;
  493. md->saved_location = SAVED_NOWHERE;
  494. md->body = NULL;
  495. md->bodylen = 0;
  496. md->no_save = 1;
  497. }
  498. /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  499. * and update every microdesc_t in the cache with pointers to its new
  500. * location. If <b>force</b> is true, do this unconditionally. If
  501. * <b>force</b> is false, do it only if we expect to save space on disk. */
  502. int
  503. microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
  504. {
  505. open_file_t *open_file;
  506. int fd = -1, res;
  507. microdesc_t **mdp;
  508. smartlist_t *wrote;
  509. ssize_t size;
  510. off_t off = 0, off_real;
  511. int orig_size, new_size;
  512. if (cache == NULL) {
  513. cache = the_microdesc_cache;
  514. if (cache == NULL)
  515. return 0;
  516. }
  517. /* Remove dead descriptors */
  518. microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
  519. if (!force && !should_rebuild_md_cache(cache))
  520. return 0;
  521. log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
  522. orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
  523. orig_size += (int)cache->journal_len;
  524. fd = start_writing_to_file(cache->cache_fname,
  525. OPEN_FLAGS_REPLACE|O_BINARY,
  526. 0600, &open_file);
  527. if (fd < 0)
  528. return -1;
  529. wrote = smartlist_new();
  530. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  531. microdesc_t *md = *mdp;
  532. size_t annotation_len;
  533. if (md->no_save || !md->body)
  534. continue;
  535. size = dump_microdescriptor(fd, md, &annotation_len);
  536. if (size < 0) {
  537. microdesc_wipe_body(md);
  538. /* rewind, in case it was a partial write. */
  539. tor_fd_setpos(fd, off);
  540. continue;
  541. }
  542. tor_assert(((size_t)size) == annotation_len + md->bodylen);
  543. md->off = off + annotation_len;
  544. off += size;
  545. off_real = tor_fd_getpos(fd);
  546. if (off_real != off) {
  547. log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
  548. "By my count, I'm at "I64_FORMAT
  549. ", but I should be at "I64_FORMAT,
  550. I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real));
  551. if (off_real >= 0)
  552. off = off_real;
  553. }
  554. if (md->saved_location != SAVED_IN_CACHE) {
  555. tor_free(md->body);
  556. md->saved_location = SAVED_IN_CACHE;
  557. }
  558. smartlist_add(wrote, md);
  559. }
  560. /* We must do this unmap _before_ we call finish_writing_to_file(), or
  561. * windows will not actually replace the file. */
  562. if (cache->cache_content) {
  563. res = tor_munmap_file(cache->cache_content);
  564. if (res != 0) {
  565. log_warn(LD_FS,
  566. "Failed to unmap old microdescriptor cache while rebuilding");
  567. }
  568. cache->cache_content = NULL;
  569. }
  570. if (finish_writing_to_file(open_file) < 0) {
  571. log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
  572. strerror(errno));
  573. /* Okay. Let's prevent from making things worse elsewhere. */
  574. cache->cache_content = NULL;
  575. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  576. microdesc_t *md = *mdp;
  577. if (md->saved_location == SAVED_IN_CACHE) {
  578. microdesc_wipe_body(md);
  579. }
  580. }
  581. smartlist_free(wrote);
  582. return -1;
  583. }
  584. cache->cache_content = tor_mmap_file(cache->cache_fname);
  585. if (!cache->cache_content && smartlist_len(wrote)) {
  586. log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
  587. cache->cache_fname);
  588. smartlist_free(wrote);
  589. return -1;
  590. }
  591. SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
  592. tor_assert(md->saved_location == SAVED_IN_CACHE);
  593. md->body = (char*)cache->cache_content->data + md->off;
  594. if (PREDICT_UNLIKELY(
  595. md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
  596. /* XXXX once bug 2022 is solved, we can kill this block and turn it
  597. * into just the tor_assert(fast_memeq) */
  598. off_t avail = cache->cache_content->size - md->off;
  599. char *bad_str;
  600. tor_assert(avail >= 0);
  601. bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
  602. log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
  603. " At offset %d, I expected to find a microdescriptor starting "
  604. " with \"onion-key\". Instead I got %s.",
  605. (int)md->off, escaped(bad_str));
  606. tor_free(bad_str);
  607. tor_assert(fast_memeq(md->body, "onion-key", 9));
  608. }
  609. } SMARTLIST_FOREACH_END(md);
  610. smartlist_free(wrote);
  611. write_str_to_file(cache->journal_fname, "", 1);
  612. cache->journal_len = 0;
  613. cache->bytes_dropped = 0;
  614. new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
  615. log_info(LD_DIR, "Done rebuilding microdesc cache. "
  616. "Saved %d bytes; %d still used.",
  617. orig_size-new_size, new_size);
  618. return 0;
  619. }
  620. /** Make sure that the reference count of every microdescriptor in cache is
  621. * accurate. */
  622. void
  623. microdesc_check_counts(void)
  624. {
  625. microdesc_t **mdp;
  626. if (!the_microdesc_cache)
  627. return;
  628. HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
  629. microdesc_t *md = *mdp;
  630. unsigned int found=0;
  631. const smartlist_t *nodes = nodelist_get_list();
  632. SMARTLIST_FOREACH(nodes, node_t *, node, {
  633. if (node->md == md) {
  634. ++found;
  635. }
  636. });
  637. tor_assert(found == md->held_by_nodes);
  638. }
  639. }
  640. /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
  641. * previously been removed from the cache if it had ever been inserted. */
  642. void
  643. microdesc_free_(microdesc_t *md, const char *fname, int lineno)
  644. {
  645. if (!md)
  646. return;
  647. /* Make sure that the microdesc was really removed from the appropriate data
  648. structures. */
  649. if (md->held_in_map) {
  650. microdesc_cache_t *cache = get_microdesc_cache_noload();
  651. microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
  652. if (md2 == md) {
  653. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  654. "in microdesc_map", fname, lineno);
  655. HT_REMOVE(microdesc_map, &cache->map, md);
  656. } else {
  657. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
  658. "set, but microdesc was not in the map.", fname, lineno);
  659. }
  660. tor_fragile_assert();
  661. }
  662. if (md->held_by_nodes) {
  663. microdesc_cache_t *cache = get_microdesc_cache_noload();
  664. int found=0;
  665. const smartlist_t *nodes = nodelist_get_list();
  666. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  667. SMARTLIST_FOREACH(nodes, node_t *, node, {
  668. if (node->md == md) {
  669. ++found;
  670. node->md = NULL;
  671. }
  672. });
  673. if (found) {
  674. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  675. "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
  676. fname, lineno, found, md->held_by_nodes, ht_badness);
  677. } else {
  678. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
  679. "set to %u, but md was not referenced by any nodes. "
  680. "ht_badness == %d",
  681. fname, lineno, md->held_by_nodes, ht_badness);
  682. }
  683. tor_fragile_assert();
  684. }
  685. //tor_assert(md->held_in_map == 0);
  686. //tor_assert(md->held_by_nodes == 0);
  687. if (md->onion_pkey)
  688. crypto_pk_free(md->onion_pkey);
  689. tor_free(md->onion_curve25519_pkey);
  690. tor_free(md->ed25519_identity_pkey);
  691. if (md->body && md->saved_location != SAVED_IN_CACHE)
  692. tor_free(md->body);
  693. if (md->family) {
  694. SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
  695. smartlist_free(md->family);
  696. }
  697. short_policy_free(md->exit_policy);
  698. short_policy_free(md->ipv6_exit_policy);
  699. tor_free(md);
  700. }
  701. /** Free all storage held in the microdesc.c module. */
  702. void
  703. microdesc_free_all(void)
  704. {
  705. if (the_microdesc_cache) {
  706. microdesc_cache_clear(the_microdesc_cache);
  707. tor_free(the_microdesc_cache->cache_fname);
  708. tor_free(the_microdesc_cache->journal_fname);
  709. tor_free(the_microdesc_cache);
  710. }
  711. }
  712. /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
  713. * <b>d</b>, return it. Otherwise return NULL. */
  714. microdesc_t *
  715. microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
  716. {
  717. microdesc_t *md, search;
  718. if (!cache)
  719. cache = get_microdesc_cache();
  720. memcpy(search.digest, d, DIGEST256_LEN);
  721. md = HT_FIND(microdesc_map, &cache->map, &search);
  722. return md;
  723. }
  724. /** Return the mean size of decriptors added to <b>cache</b> since it was last
  725. * cleared. Used to estimate the size of large downloads. */
  726. size_t
  727. microdesc_average_size(microdesc_cache_t *cache)
  728. {
  729. if (!cache)
  730. cache = get_microdesc_cache();
  731. if (!cache->n_seen)
  732. return 512;
  733. return (size_t)(cache->total_len_seen / cache->n_seen);
  734. }
  735. /** Return a smartlist of all the sha256 digest of the microdescriptors that
  736. * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  737. * to internals of <b>ns</b>; you should not free the members of the resulting
  738. * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
  739. smartlist_t *
  740. microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
  741. int downloadable_only, digest256map_t *skip)
  742. {
  743. smartlist_t *result = smartlist_new();
  744. time_t now = time(NULL);
  745. tor_assert(ns->flavor == FLAV_MICRODESC);
  746. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  747. if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
  748. continue;
  749. if (downloadable_only &&
  750. !download_status_is_ready(&rs->dl_status, now,
  751. get_options()->TestingMicrodescMaxDownloadTries))
  752. continue;
  753. if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
  754. continue;
  755. if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
  756. continue;
  757. /* XXXX Also skip if we're a noncache and wouldn't use this router.
  758. * XXXX NM Microdesc
  759. */
  760. smartlist_add(result, rs->descriptor_digest);
  761. } SMARTLIST_FOREACH_END(rs);
  762. return result;
  763. }
  764. /** Launch download requests for microdescriptors as appropriate.
  765. *
  766. * Specifically, we should launch download requests if we are configured to
  767. * download mirodescriptors, and there are some microdescriptors listed in the
  768. * current microdesc consensus that we don't have, and either we never asked
  769. * for them, or we failed to download them but we're willing to retry.
  770. */
  771. void
  772. update_microdesc_downloads(time_t now)
  773. {
  774. const or_options_t *options = get_options();
  775. networkstatus_t *consensus;
  776. smartlist_t *missing;
  777. digest256map_t *pending;
  778. if (should_delay_dir_fetches(options, NULL))
  779. return;
  780. if (directory_too_idle_to_fetch_descriptors(options, now))
  781. return;
  782. consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  783. if (!consensus)
  784. return;
  785. if (!we_fetch_microdescriptors(options))
  786. return;
  787. pending = digest256map_new();
  788. list_pending_microdesc_downloads(pending);
  789. missing = microdesc_list_missing_digest256(consensus,
  790. get_microdesc_cache(),
  791. 1,
  792. pending);
  793. digest256map_free(pending, NULL);
  794. launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
  795. missing, NULL, now);
  796. smartlist_free(missing);
  797. }
  798. /** For every microdescriptor listed in the current microdecriptor consensus,
  799. * update its last_listed field to be at least as recent as the publication
  800. * time of the current microdescriptor consensus.
  801. */
  802. void
  803. update_microdescs_from_networkstatus(time_t now)
  804. {
  805. microdesc_cache_t *cache = get_microdesc_cache();
  806. microdesc_t *md;
  807. networkstatus_t *ns =
  808. networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  809. if (! ns)
  810. return;
  811. tor_assert(ns->flavor == FLAV_MICRODESC);
  812. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  813. md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
  814. if (md && ns->valid_after > md->last_listed)
  815. md->last_listed = ns->valid_after;
  816. } SMARTLIST_FOREACH_END(rs);
  817. }
  818. /** Return true iff we should prefer to use microdescriptors rather than
  819. * routerdescs for building circuits. */
  820. int
  821. we_use_microdescriptors_for_circuits(const or_options_t *options)
  822. {
  823. if (options->UseMicrodescriptors == 0)
  824. return 0; /* the user explicitly picked no */
  825. return 1; /* yes and auto both mean yes */
  826. }
  827. /** Return true iff we should try to download microdescriptors at all. */
  828. int
  829. we_fetch_microdescriptors(const or_options_t *options)
  830. {
  831. if (directory_caches_dir_info(options))
  832. return 1;
  833. if (options->FetchUselessDescriptors)
  834. return 1;
  835. return we_use_microdescriptors_for_circuits(options);
  836. }
  837. /** Return true iff we should try to download router descriptors at all. */
  838. int
  839. we_fetch_router_descriptors(const or_options_t *options)
  840. {
  841. if (directory_caches_dir_info(options))
  842. return 1;
  843. if (options->FetchUselessDescriptors)
  844. return 1;
  845. return ! we_use_microdescriptors_for_circuits(options);
  846. }
  847. /** Return the consensus flavor we actually want to use to build circuits. */
  848. MOCK_IMPL(int,
  849. usable_consensus_flavor,(void))
  850. {
  851. if (we_use_microdescriptors_for_circuits(get_options())) {
  852. return FLAV_MICRODESC;
  853. } else {
  854. return FLAV_NS;
  855. }
  856. }