microdesc.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959
  1. /* Copyright (c) 2009-2015, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. #include "or.h"
  4. #include "circuitbuild.h"
  5. #include "config.h"
  6. #include "directory.h"
  7. #include "dirserv.h"
  8. #include "entrynodes.h"
  9. #include "microdesc.h"
  10. #include "networkstatus.h"
  11. #include "nodelist.h"
  12. #include "policies.h"
  13. #include "router.h"
  14. #include "routerlist.h"
  15. #include "routerparse.h"
  16. /** A data structure to hold a bunch of cached microdescriptors. There are
  17. * two active files in the cache: a "cache file" that we mmap, and a "journal
  18. * file" that we append to. Periodically, we rebuild the cache file to hold
  19. * only the microdescriptors that we want to keep */
  20. struct microdesc_cache_t {
  21. /** Map from sha256-digest to microdesc_t for every microdesc_t in the
  22. * cache. */
  23. HT_HEAD(microdesc_map, microdesc_t) map;
  24. /** Name of the cache file. */
  25. char *cache_fname;
  26. /** Name of the journal file. */
  27. char *journal_fname;
  28. /** Mmap'd contents of the cache file, or NULL if there is none. */
  29. tor_mmap_t *cache_content;
  30. /** Number of bytes used in the journal file. */
  31. size_t journal_len;
  32. /** Number of bytes in descriptors removed as too old. */
  33. size_t bytes_dropped;
  34. /** Total bytes of microdescriptor bodies we have added to this cache */
  35. uint64_t total_len_seen;
  36. /** Total number of microdescriptors we have added to this cache */
  37. unsigned n_seen;
  38. /** True iff we have loaded this cache from disk ever. */
  39. int is_loaded;
  40. };
  41. static microdesc_cache_t *get_microdesc_cache_noload(void);
  42. /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
  43. static INLINE unsigned int
  44. microdesc_hash_(microdesc_t *md)
  45. {
  46. return (unsigned) siphash24g(md->digest, sizeof(md->digest));
  47. }
  48. /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */
  49. static INLINE int
  50. microdesc_eq_(microdesc_t *a, microdesc_t *b)
  51. {
  52. return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
  53. }
  54. HT_PROTOTYPE(microdesc_map, microdesc_t, node,
  55. microdesc_hash_, microdesc_eq_);
  56. HT_GENERATE2(microdesc_map, microdesc_t, node,
  57. microdesc_hash_, microdesc_eq_, 0.6,
  58. tor_reallocarray_, tor_free_)
  59. /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  60. * On success, return the total number of bytes written, and set
  61. * *<b>annotation_len_out</b> to the number of bytes written as
  62. * annotations. */
  63. static ssize_t
  64. dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
  65. {
  66. ssize_t r = 0;
  67. ssize_t written;
  68. if (md->body == NULL) {
  69. *annotation_len_out = 0;
  70. return 0;
  71. }
  72. /* XXXX drops unknown annotations. */
  73. if (md->last_listed) {
  74. char buf[ISO_TIME_LEN+1];
  75. char annotation[ISO_TIME_LEN+32];
  76. format_iso_time(buf, md->last_listed);
  77. tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
  78. if (write_all(fd, annotation, strlen(annotation), 0) < 0) {
  79. log_warn(LD_DIR,
  80. "Couldn't write microdescriptor annotation: %s",
  81. strerror(errno));
  82. return -1;
  83. }
  84. r += strlen(annotation);
  85. *annotation_len_out = r;
  86. } else {
  87. *annotation_len_out = 0;
  88. }
  89. md->off = tor_fd_getpos(fd);
  90. written = write_all(fd, md->body, md->bodylen, 0);
  91. if (written != (ssize_t)md->bodylen) {
  92. log_warn(LD_DIR,
  93. "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
  94. (long)written, (unsigned long)md->bodylen,
  95. strerror(errno));
  96. return -1;
  97. }
  98. r += md->bodylen;
  99. return r;
  100. }
  101. /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
  102. * such object has been allocated. */
  103. static microdesc_cache_t *the_microdesc_cache = NULL;
  104. /** Return a pointer to the microdescriptor cache, loading it if necessary. */
  105. microdesc_cache_t *
  106. get_microdesc_cache(void)
  107. {
  108. microdesc_cache_t *cache = get_microdesc_cache_noload();
  109. if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
  110. microdesc_cache_reload(cache);
  111. }
  112. return cache;
  113. }
  114. /** Return a pointer to the microdescriptor cache, creating (but not loading)
  115. * it if necessary. */
  116. static microdesc_cache_t *
  117. get_microdesc_cache_noload(void)
  118. {
  119. if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
  120. microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
  121. HT_INIT(microdesc_map, &cache->map);
  122. cache->cache_fname = get_datadir_fname("cached-microdescs");
  123. cache->journal_fname = get_datadir_fname("cached-microdescs.new");
  124. the_microdesc_cache = cache;
  125. }
  126. return the_microdesc_cache;
  127. }
  128. /* There are three sources of microdescriptors:
  129. 1) Generated by us while acting as a directory authority.
  130. 2) Loaded from the cache on disk.
  131. 3) Downloaded.
  132. */
  133. /** Decode the microdescriptors from the string starting at <b>s</b> and
  134. * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
  135. * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
  136. * leave their bodies as pointers to the mmap'd cache. If where is
  137. * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
  138. * set the last_listed field of every microdesc to listed_at. If
  139. * requested_digests is non-null, then it contains a list of digests we mean
  140. * to allow, so we should reject any non-requested microdesc with a different
  141. * digest, and alter the list to contain only the digests of those microdescs
  142. * we didn't find.
  143. * Return a newly allocated list of the added microdescriptors, or NULL */
  144. smartlist_t *
  145. microdescs_add_to_cache(microdesc_cache_t *cache,
  146. const char *s, const char *eos, saved_location_t where,
  147. int no_save, time_t listed_at,
  148. smartlist_t *requested_digests256)
  149. {
  150. void * const DIGEST_REQUESTED = (void*)1;
  151. void * const DIGEST_RECEIVED = (void*)2;
  152. void * const DIGEST_INVALID = (void*)3;
  153. smartlist_t *descriptors, *added;
  154. const int allow_annotations = (where != SAVED_NOWHERE);
  155. smartlist_t *invalid_digests = smartlist_new();
  156. descriptors = microdescs_parse_from_string(s, eos,
  157. allow_annotations,
  158. where, invalid_digests);
  159. if (listed_at != (time_t)-1) {
  160. SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
  161. md->last_listed = listed_at);
  162. }
  163. if (requested_digests256) {
  164. digest256map_t *requested;
  165. requested = digest256map_new();
  166. /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
  167. SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
  168. digest256map_set(requested, cp, DIGEST_REQUESTED));
  169. /* Set requested[d] to DIGEST_INVALID for every md we requested which we
  170. * will never be able to parse. Remove the ones we didn't request from
  171. * invalid_digests.
  172. */
  173. SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
  174. if (digest256map_get(requested, cp)) {
  175. digest256map_set(requested, cp, DIGEST_INVALID);
  176. } else {
  177. tor_free(cp);
  178. SMARTLIST_DEL_CURRENT(invalid_digests, cp);
  179. }
  180. } SMARTLIST_FOREACH_END(cp);
  181. /* Update requested[d] to 2 for the mds we asked for and got. Delete the
  182. * ones we never requested from the 'descriptors' smartlist.
  183. */
  184. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  185. if (digest256map_get(requested, (const uint8_t*)md->digest)) {
  186. digest256map_set(requested, (const uint8_t*)md->digest,
  187. DIGEST_RECEIVED);
  188. } else {
  189. log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
  190. microdesc_free(md);
  191. SMARTLIST_DEL_CURRENT(descriptors, md);
  192. }
  193. } SMARTLIST_FOREACH_END(md);
  194. /* Remove the ones we got or the invalid ones from requested_digests256.
  195. */
  196. SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
  197. void *status = digest256map_get(requested, cp);
  198. if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
  199. tor_free(cp);
  200. SMARTLIST_DEL_CURRENT(requested_digests256, cp);
  201. }
  202. } SMARTLIST_FOREACH_END(cp);
  203. digest256map_free(requested, NULL);
  204. }
  205. /* For every requested microdescriptor that was unparseable, mark it
  206. * as not to be retried. */
  207. if (smartlist_len(invalid_digests)) {
  208. networkstatus_t *ns =
  209. networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
  210. if (ns) {
  211. SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
  212. routerstatus_t *rs =
  213. router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
  214. if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
  215. download_status_mark_impossible(&rs->dl_status);
  216. }
  217. } SMARTLIST_FOREACH_END(d);
  218. }
  219. }
  220. SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
  221. smartlist_free(invalid_digests);
  222. added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
  223. smartlist_free(descriptors);
  224. return added;
  225. }
  226. /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
  227. * a string to decode. Frees any members of <b>descriptors</b> that it does
  228. * not add. */
  229. smartlist_t *
  230. microdescs_add_list_to_cache(microdesc_cache_t *cache,
  231. smartlist_t *descriptors, saved_location_t where,
  232. int no_save)
  233. {
  234. smartlist_t *added;
  235. open_file_t *open_file = NULL;
  236. int fd = -1;
  237. // int n_added = 0;
  238. ssize_t size = 0;
  239. if (where == SAVED_NOWHERE && !no_save) {
  240. fd = start_writing_to_file(cache->journal_fname,
  241. OPEN_FLAGS_APPEND|O_BINARY,
  242. 0600, &open_file);
  243. if (fd < 0) {
  244. log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
  245. cache->journal_fname, strerror(errno));
  246. }
  247. }
  248. added = smartlist_new();
  249. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  250. microdesc_t *md2;
  251. md2 = HT_FIND(microdesc_map, &cache->map, md);
  252. if (md2) {
  253. /* We already had this one. */
  254. if (md2->last_listed < md->last_listed)
  255. md2->last_listed = md->last_listed;
  256. microdesc_free(md);
  257. if (where != SAVED_NOWHERE)
  258. cache->bytes_dropped += size;
  259. continue;
  260. }
  261. /* Okay, it's a new one. */
  262. if (fd >= 0) {
  263. size_t annotation_len;
  264. size = dump_microdescriptor(fd, md, &annotation_len);
  265. if (size < 0) {
  266. /* we already warned in dump_microdescriptor */
  267. abort_writing_to_file(open_file);
  268. fd = -1;
  269. } else {
  270. md->saved_location = SAVED_IN_JOURNAL;
  271. cache->journal_len += size;
  272. }
  273. } else {
  274. md->saved_location = where;
  275. }
  276. md->no_save = no_save;
  277. HT_INSERT(microdesc_map, &cache->map, md);
  278. md->held_in_map = 1;
  279. smartlist_add(added, md);
  280. ++cache->n_seen;
  281. cache->total_len_seen += md->bodylen;
  282. } SMARTLIST_FOREACH_END(md);
  283. if (fd >= 0) {
  284. if (finish_writing_to_file(open_file) < 0) {
  285. log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
  286. strerror(errno));
  287. smartlist_clear(added);
  288. return added;
  289. }
  290. }
  291. {
  292. networkstatus_t *ns = networkstatus_get_latest_consensus();
  293. if (ns && ns->flavor == FLAV_MICRODESC)
  294. SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
  295. }
  296. if (smartlist_len(added))
  297. router_dir_info_changed();
  298. return added;
  299. }
  300. /** Remove every microdescriptor in <b>cache</b>. */
  301. void
  302. microdesc_cache_clear(microdesc_cache_t *cache)
  303. {
  304. microdesc_t **entry, **next;
  305. for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
  306. microdesc_t *md = *entry;
  307. next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
  308. md->held_in_map = 0;
  309. microdesc_free(md);
  310. }
  311. HT_CLEAR(microdesc_map, &cache->map);
  312. if (cache->cache_content) {
  313. int res = tor_munmap_file(cache->cache_content);
  314. if (res != 0) {
  315. log_warn(LD_FS,
  316. "tor_munmap_file() failed clearing microdesc cache; "
  317. "we are probably about to leak memory.");
  318. /* TODO something smarter? */
  319. }
  320. cache->cache_content = NULL;
  321. }
  322. cache->total_len_seen = 0;
  323. cache->n_seen = 0;
  324. cache->bytes_dropped = 0;
  325. }
  326. /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
  327. * for the first time. Return 0 on success, -1 on failure. */
  328. int
  329. microdesc_cache_reload(microdesc_cache_t *cache)
  330. {
  331. struct stat st;
  332. char *journal_content;
  333. smartlist_t *added;
  334. tor_mmap_t *mm;
  335. int total = 0;
  336. microdesc_cache_clear(cache);
  337. cache->is_loaded = 1;
  338. mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
  339. if (mm) {
  340. added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
  341. SAVED_IN_CACHE, 0, -1, NULL);
  342. if (added) {
  343. total += smartlist_len(added);
  344. smartlist_free(added);
  345. }
  346. }
  347. journal_content = read_file_to_str(cache->journal_fname,
  348. RFTS_IGNORE_MISSING, &st);
  349. if (journal_content) {
  350. cache->journal_len = (size_t) st.st_size;
  351. added = microdescs_add_to_cache(cache, journal_content,
  352. journal_content+st.st_size,
  353. SAVED_IN_JOURNAL, 0, -1, NULL);
  354. if (added) {
  355. total += smartlist_len(added);
  356. smartlist_free(added);
  357. }
  358. tor_free(journal_content);
  359. }
  360. log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
  361. total);
  362. microdesc_cache_rebuild(cache, 0 /* don't force */);
  363. return 0;
  364. }
  365. /** By default, we remove any microdescriptors that have gone at least this
  366. * long without appearing in a current consensus. */
  367. #define TOLERATE_MICRODESC_AGE (7*24*60*60)
  368. /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
  369. * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
  370. * positive, specifically remove microdescriptors that have been unlisted
  371. * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
  372. * if we have no current live microdescriptor consensus.
  373. */
  374. void
  375. microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
  376. {
  377. microdesc_t **mdp, *victim;
  378. int dropped=0, kept=0;
  379. size_t bytes_dropped = 0;
  380. time_t now = time(NULL);
  381. /* If we don't know a live consensus, don't believe last_listed values: we
  382. * might be starting up after being down for a while. */
  383. if (! force &&
  384. ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
  385. return;
  386. if (cutoff <= 0)
  387. cutoff = now - TOLERATE_MICRODESC_AGE;
  388. for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
  389. const int is_old = (*mdp)->last_listed < cutoff;
  390. const unsigned held_by_nodes = (*mdp)->held_by_nodes;
  391. if (is_old && !held_by_nodes) {
  392. ++dropped;
  393. victim = *mdp;
  394. mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
  395. victim->held_in_map = 0;
  396. bytes_dropped += victim->bodylen;
  397. microdesc_free(victim);
  398. } else {
  399. if (is_old) {
  400. /* It's old, but it has held_by_nodes set. That's not okay. */
  401. /* Let's try to diagnose and fix #7164 . */
  402. smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
  403. const networkstatus_t *ns = networkstatus_get_latest_consensus();
  404. long networkstatus_age = -1;
  405. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  406. if (ns) {
  407. networkstatus_age = now - ns->valid_after;
  408. }
  409. log_warn(LD_BUG, "Microdescriptor seemed very old "
  410. "(last listed %d hours ago vs %d hour cutoff), but is still "
  411. "marked as being held by %d node(s). I found %d node(s) "
  412. "holding it. Current networkstatus is %ld hours old. "
  413. "Hashtable badness is %d.",
  414. (int)((now - (*mdp)->last_listed) / 3600),
  415. (int)((now - cutoff) / 3600),
  416. held_by_nodes,
  417. smartlist_len(nodes),
  418. networkstatus_age / 3600,
  419. ht_badness);
  420. SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
  421. const char *rs_match = "No RS";
  422. const char *rs_present = "";
  423. if (node->rs) {
  424. if (tor_memeq(node->rs->descriptor_digest,
  425. (*mdp)->digest, DIGEST256_LEN)) {
  426. rs_match = "Microdesc digest in RS matches";
  427. } else {
  428. rs_match = "Microdesc digest in RS does match";
  429. }
  430. if (ns) {
  431. /* This should be impossible, but let's see! */
  432. rs_present = " RS not present in networkstatus.";
  433. SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
  434. if (rs == node->rs) {
  435. rs_present = " RS okay in networkstatus.";
  436. }
  437. });
  438. }
  439. }
  440. log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
  441. node_sl_idx,
  442. hex_str(node->identity, DIGEST_LEN),
  443. node->md, node->rs, node->ri, rs_match, rs_present);
  444. } SMARTLIST_FOREACH_END(node);
  445. smartlist_free(nodes);
  446. (*mdp)->last_listed = now;
  447. }
  448. ++kept;
  449. mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
  450. }
  451. }
  452. if (dropped) {
  453. log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
  454. dropped,dropped+kept);
  455. cache->bytes_dropped += bytes_dropped;
  456. }
  457. }
  458. static int
  459. should_rebuild_md_cache(microdesc_cache_t *cache)
  460. {
  461. const size_t old_len =
  462. cache->cache_content ? cache->cache_content->size : 0;
  463. const size_t journal_len = cache->journal_len;
  464. const size_t dropped = cache->bytes_dropped;
  465. if (journal_len < 16384)
  466. return 0; /* Don't bother, not enough has happened yet. */
  467. if (dropped > (journal_len + old_len) / 3)
  468. return 1; /* We could save 1/3 or more of the currently used space. */
  469. if (journal_len > old_len / 2)
  470. return 1; /* We should append to the regular file */
  471. return 0;
  472. }
  473. /**
  474. * Mark <b>md</b> as having no body, and release any storage previously held
  475. * by its body.
  476. */
  477. static void
  478. microdesc_wipe_body(microdesc_t *md)
  479. {
  480. if (!md)
  481. return;
  482. if (md->saved_location != SAVED_IN_CACHE)
  483. tor_free(md->body);
  484. md->off = 0;
  485. md->saved_location = SAVED_NOWHERE;
  486. md->body = NULL;
  487. md->bodylen = 0;
  488. md->no_save = 1;
  489. }
  490. /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  491. * and update every microdesc_t in the cache with pointers to its new
  492. * location. If <b>force</b> is true, do this unconditionally. If
  493. * <b>force</b> is false, do it only if we expect to save space on disk. */
  494. int
  495. microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
  496. {
  497. open_file_t *open_file;
  498. int fd = -1, res;
  499. microdesc_t **mdp;
  500. smartlist_t *wrote;
  501. ssize_t size;
  502. off_t off = 0, off_real;
  503. int orig_size, new_size;
  504. if (cache == NULL) {
  505. cache = the_microdesc_cache;
  506. if (cache == NULL)
  507. return 0;
  508. }
  509. /* Remove dead descriptors */
  510. microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
  511. if (!force && !should_rebuild_md_cache(cache))
  512. return 0;
  513. log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
  514. orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
  515. orig_size += (int)cache->journal_len;
  516. fd = start_writing_to_file(cache->cache_fname,
  517. OPEN_FLAGS_REPLACE|O_BINARY,
  518. 0600, &open_file);
  519. if (fd < 0)
  520. return -1;
  521. wrote = smartlist_new();
  522. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  523. microdesc_t *md = *mdp;
  524. size_t annotation_len;
  525. if (md->no_save || !md->body)
  526. continue;
  527. size = dump_microdescriptor(fd, md, &annotation_len);
  528. if (size < 0) {
  529. microdesc_wipe_body(md);
  530. /* rewind, in case it was a partial write. */
  531. tor_fd_setpos(fd, off);
  532. continue;
  533. }
  534. tor_assert(((size_t)size) == annotation_len + md->bodylen);
  535. md->off = off + annotation_len;
  536. off += size;
  537. off_real = tor_fd_getpos(fd);
  538. if (off_real != off) {
  539. log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
  540. "By my count, I'm at "I64_FORMAT
  541. ", but I should be at "I64_FORMAT,
  542. I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real));
  543. if (off_real >= 0)
  544. off = off_real;
  545. }
  546. if (md->saved_location != SAVED_IN_CACHE) {
  547. tor_free(md->body);
  548. md->saved_location = SAVED_IN_CACHE;
  549. }
  550. smartlist_add(wrote, md);
  551. }
  552. /* We must do this unmap _before_ we call finish_writing_to_file(), or
  553. * windows will not actually replace the file. */
  554. if (cache->cache_content) {
  555. res = tor_munmap_file(cache->cache_content);
  556. if (res != 0) {
  557. log_warn(LD_FS,
  558. "Failed to unmap old microdescriptor cache while rebuilding");
  559. }
  560. cache->cache_content = NULL;
  561. }
  562. if (finish_writing_to_file(open_file) < 0) {
  563. log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
  564. strerror(errno));
  565. /* Okay. Let's prevent from making things worse elsewhere. */
  566. cache->cache_content = NULL;
  567. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  568. microdesc_t *md = *mdp;
  569. if (md->saved_location == SAVED_IN_CACHE) {
  570. microdesc_wipe_body(md);
  571. }
  572. }
  573. smartlist_free(wrote);
  574. return -1;
  575. }
  576. cache->cache_content = tor_mmap_file(cache->cache_fname);
  577. if (!cache->cache_content && smartlist_len(wrote)) {
  578. log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
  579. cache->cache_fname);
  580. smartlist_free(wrote);
  581. return -1;
  582. }
  583. SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
  584. tor_assert(md->saved_location == SAVED_IN_CACHE);
  585. md->body = (char*)cache->cache_content->data + md->off;
  586. if (PREDICT_UNLIKELY(
  587. md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
  588. /* XXXX once bug 2022 is solved, we can kill this block and turn it
  589. * into just the tor_assert(fast_memeq) */
  590. off_t avail = cache->cache_content->size - md->off;
  591. char *bad_str;
  592. tor_assert(avail >= 0);
  593. bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
  594. log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
  595. " At offset %d, I expected to find a microdescriptor starting "
  596. " with \"onion-key\". Instead I got %s.",
  597. (int)md->off, escaped(bad_str));
  598. tor_free(bad_str);
  599. tor_assert(fast_memeq(md->body, "onion-key", 9));
  600. }
  601. } SMARTLIST_FOREACH_END(md);
  602. smartlist_free(wrote);
  603. write_str_to_file(cache->journal_fname, "", 1);
  604. cache->journal_len = 0;
  605. cache->bytes_dropped = 0;
  606. new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
  607. log_info(LD_DIR, "Done rebuilding microdesc cache. "
  608. "Saved %d bytes; %d still used.",
  609. orig_size-new_size, new_size);
  610. return 0;
  611. }
  612. /** Make sure that the reference count of every microdescriptor in cache is
  613. * accurate. */
  614. void
  615. microdesc_check_counts(void)
  616. {
  617. microdesc_t **mdp;
  618. if (!the_microdesc_cache)
  619. return;
  620. HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
  621. microdesc_t *md = *mdp;
  622. unsigned int found=0;
  623. const smartlist_t *nodes = nodelist_get_list();
  624. SMARTLIST_FOREACH(nodes, node_t *, node, {
  625. if (node->md == md) {
  626. ++found;
  627. }
  628. });
  629. tor_assert(found == md->held_by_nodes);
  630. }
  631. }
  632. /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
  633. * previously been removed from the cache if it had ever been inserted. */
  634. void
  635. microdesc_free_(microdesc_t *md, const char *fname, int lineno)
  636. {
  637. if (!md)
  638. return;
  639. /* Make sure that the microdesc was really removed from the appropriate data
  640. structures. */
  641. if (md->held_in_map) {
  642. microdesc_cache_t *cache = get_microdesc_cache_noload();
  643. microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
  644. if (md2 == md) {
  645. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  646. "in microdesc_map", fname, lineno);
  647. HT_REMOVE(microdesc_map, &cache->map, md);
  648. } else {
  649. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
  650. "set, but microdesc was not in the map.", fname, lineno);
  651. }
  652. tor_fragile_assert();
  653. }
  654. if (md->held_by_nodes) {
  655. microdesc_cache_t *cache = get_microdesc_cache_noload();
  656. int found=0;
  657. const smartlist_t *nodes = nodelist_get_list();
  658. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  659. SMARTLIST_FOREACH(nodes, node_t *, node, {
  660. if (node->md == md) {
  661. ++found;
  662. node->md = NULL;
  663. }
  664. });
  665. if (found) {
  666. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  667. "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
  668. fname, lineno, found, md->held_by_nodes, ht_badness);
  669. } else {
  670. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
  671. "set to %u, but md was not referenced by any nodes. "
  672. "ht_badness == %d",
  673. fname, lineno, md->held_by_nodes, ht_badness);
  674. }
  675. tor_fragile_assert();
  676. }
  677. //tor_assert(md->held_in_map == 0);
  678. //tor_assert(md->held_by_nodes == 0);
  679. if (md->onion_pkey)
  680. crypto_pk_free(md->onion_pkey);
  681. tor_free(md->onion_curve25519_pkey);
  682. tor_free(md->ed25519_identity_pkey);
  683. if (md->body && md->saved_location != SAVED_IN_CACHE)
  684. tor_free(md->body);
  685. if (md->family) {
  686. SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
  687. smartlist_free(md->family);
  688. }
  689. short_policy_free(md->exit_policy);
  690. short_policy_free(md->ipv6_exit_policy);
  691. tor_free(md);
  692. }
  693. /** Free all storage held in the microdesc.c module. */
  694. void
  695. microdesc_free_all(void)
  696. {
  697. if (the_microdesc_cache) {
  698. microdesc_cache_clear(the_microdesc_cache);
  699. tor_free(the_microdesc_cache->cache_fname);
  700. tor_free(the_microdesc_cache->journal_fname);
  701. tor_free(the_microdesc_cache);
  702. }
  703. }
  704. /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
  705. * <b>d</b>, return it. Otherwise return NULL. */
  706. microdesc_t *
  707. microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
  708. {
  709. microdesc_t *md, search;
  710. if (!cache)
  711. cache = get_microdesc_cache();
  712. memcpy(search.digest, d, DIGEST256_LEN);
  713. md = HT_FIND(microdesc_map, &cache->map, &search);
  714. return md;
  715. }
  716. /** Return the mean size of decriptors added to <b>cache</b> since it was last
  717. * cleared. Used to estimate the size of large downloads. */
  718. size_t
  719. microdesc_average_size(microdesc_cache_t *cache)
  720. {
  721. if (!cache)
  722. cache = get_microdesc_cache();
  723. if (!cache->n_seen)
  724. return 512;
  725. return (size_t)(cache->total_len_seen / cache->n_seen);
  726. }
  727. /** Return a smartlist of all the sha256 digest of the microdescriptors that
  728. * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  729. * to internals of <b>ns</b>; you should not free the members of the resulting
  730. * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
  731. smartlist_t *
  732. microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
  733. int downloadable_only, digest256map_t *skip)
  734. {
  735. smartlist_t *result = smartlist_new();
  736. time_t now = time(NULL);
  737. tor_assert(ns->flavor == FLAV_MICRODESC);
  738. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  739. if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
  740. continue;
  741. if (downloadable_only &&
  742. !download_status_is_ready(&rs->dl_status, now,
  743. get_options()->TestingMicrodescMaxDownloadTries))
  744. continue;
  745. if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
  746. continue;
  747. if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
  748. continue;
  749. /* XXXX Also skip if we're a noncache and wouldn't use this router.
  750. * XXXX NM Microdesc
  751. */
  752. smartlist_add(result, rs->descriptor_digest);
  753. } SMARTLIST_FOREACH_END(rs);
  754. return result;
  755. }
  756. /** Launch download requests for microdescriptors as appropriate.
  757. *
  758. * Specifically, we should launch download requests if we are configured to
  759. * download mirodescriptors, and there are some microdescriptors listed the
  760. * current microdesc consensus that we don't have, and either we never asked
  761. * for them, or we failed to download them but we're willing to retry.
  762. */
  763. void
  764. update_microdesc_downloads(time_t now)
  765. {
  766. const or_options_t *options = get_options();
  767. networkstatus_t *consensus;
  768. smartlist_t *missing;
  769. digest256map_t *pending;
  770. if (should_delay_dir_fetches(options, NULL))
  771. return;
  772. if (directory_too_idle_to_fetch_descriptors(options, now))
  773. return;
  774. consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  775. if (!consensus)
  776. return;
  777. if (!we_fetch_microdescriptors(options))
  778. return;
  779. pending = digest256map_new();
  780. list_pending_microdesc_downloads(pending);
  781. missing = microdesc_list_missing_digest256(consensus,
  782. get_microdesc_cache(),
  783. 1,
  784. pending);
  785. digest256map_free(pending, NULL);
  786. launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
  787. missing, NULL, now);
  788. smartlist_free(missing);
  789. }
  790. /** For every microdescriptor listed in the current microdecriptor consensus,
  791. * update its last_listed field to be at least as recent as the publication
  792. * time of the current microdescriptor consensus.
  793. */
  794. void
  795. update_microdescs_from_networkstatus(time_t now)
  796. {
  797. microdesc_cache_t *cache = get_microdesc_cache();
  798. microdesc_t *md;
  799. networkstatus_t *ns =
  800. networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  801. if (! ns)
  802. return;
  803. tor_assert(ns->flavor == FLAV_MICRODESC);
  804. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  805. md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
  806. if (md && ns->valid_after > md->last_listed)
  807. md->last_listed = ns->valid_after;
  808. } SMARTLIST_FOREACH_END(rs);
  809. }
  810. /** Return true iff we should prefer to use microdescriptors rather than
  811. * routerdescs for building circuits. */
  812. int
  813. we_use_microdescriptors_for_circuits(const or_options_t *options)
  814. {
  815. int ret = options->UseMicrodescriptors;
  816. if (ret == -1) {
  817. /* UseMicrodescriptors is "auto"; we need to decide: */
  818. /* If we are configured to use bridges and none of our bridges
  819. * know what a microdescriptor is, the answer is no. */
  820. if (options->UseBridges && !any_bridge_supports_microdescriptors())
  821. return 0;
  822. /* Otherwise, we decide that we'll use microdescriptors iff we are
  823. * not a server, and we're not autofetching everything. */
  824. /* XXX023 what does not being a server have to do with it? also there's
  825. * a partitioning issue here where bridges differ from clients. */
  826. ret = !server_mode(options) && !options->FetchUselessDescriptors;
  827. }
  828. return ret;
  829. }
  830. /** Return true iff we should try to download microdescriptors at all. */
  831. int
  832. we_fetch_microdescriptors(const or_options_t *options)
  833. {
  834. if (directory_caches_dir_info(options))
  835. return 1;
  836. if (options->FetchUselessDescriptors)
  837. return 1;
  838. return we_use_microdescriptors_for_circuits(options);
  839. }
  840. /** Return true iff we should try to download router descriptors at all. */
  841. int
  842. we_fetch_router_descriptors(const or_options_t *options)
  843. {
  844. if (directory_caches_dir_info(options))
  845. return 1;
  846. if (options->FetchUselessDescriptors)
  847. return 1;
  848. return ! we_use_microdescriptors_for_circuits(options);
  849. }
  850. /** Return the consensus flavor we actually want to use to build circuits. */
  851. int
  852. usable_consensus_flavor(void)
  853. {
  854. if (we_use_microdescriptors_for_circuits(get_options())) {
  855. return FLAV_MICRODESC;
  856. } else {
  857. return FLAV_NS;
  858. }
  859. }