microdesc.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777
  1. /* Copyright (c) 2009-2012, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. #include "or.h"
  4. #include "circuitbuild.h"
  5. #include "config.h"
  6. #include "directory.h"
  7. #include "dirserv.h"
  8. #include "entrynodes.h"
  9. #include "microdesc.h"
  10. #include "networkstatus.h"
  11. #include "nodelist.h"
  12. #include "policies.h"
  13. #include "router.h"
  14. #include "routerlist.h"
  15. #include "routerparse.h"
  16. /** A data structure to hold a bunch of cached microdescriptors. There are
  17. * two active files in the cache: a "cache file" that we mmap, and a "journal
  18. * file" that we append to. Periodically, we rebuild the cache file to hold
  19. * only the microdescriptors that we want to keep */
  20. struct microdesc_cache_t {
  21. /** Map from sha256-digest to microdesc_t for every microdesc_t in the
  22. * cache. */
  23. HT_HEAD(microdesc_map, microdesc_t) map;
  24. /** Name of the cache file. */
  25. char *cache_fname;
  26. /** Name of the journal file. */
  27. char *journal_fname;
  28. /** Mmap'd contents of the cache file, or NULL if there is none. */
  29. tor_mmap_t *cache_content;
  30. /** Number of bytes used in the journal file. */
  31. size_t journal_len;
  32. /** Number of bytes in descriptors removed as too old. */
  33. size_t bytes_dropped;
  34. /** Total bytes of microdescriptor bodies we have added to this cache */
  35. uint64_t total_len_seen;
  36. /** Total number of microdescriptors we have added to this cache */
  37. unsigned n_seen;
  38. };
  39. /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
  40. static INLINE unsigned int
  41. microdesc_hash_(microdesc_t *md)
  42. {
  43. unsigned *d = (unsigned*)md->digest;
  44. #if SIZEOF_INT == 4
  45. return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7];
  46. #else
  47. return d[0] ^ d[1] ^ d[2] ^ d[3];
  48. #endif
  49. }
  50. /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */
  51. static INLINE int
  52. microdesc_eq_(microdesc_t *a, microdesc_t *b)
  53. {
  54. return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
  55. }
  56. HT_PROTOTYPE(microdesc_map, microdesc_t, node,
  57. microdesc_hash_, microdesc_eq_);
  58. HT_GENERATE(microdesc_map, microdesc_t, node,
  59. microdesc_hash_, microdesc_eq_, 0.6,
  60. malloc, realloc, free);
  61. /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  62. * On success, return the total number of bytes written, and set
  63. * *<b>annotation_len_out</b> to the number of bytes written as
  64. * annotations. */
  65. static ssize_t
  66. dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out)
  67. {
  68. ssize_t r = 0;
  69. size_t written;
  70. /* XXXX drops unkown annotations. */
  71. if (md->last_listed) {
  72. char buf[ISO_TIME_LEN+1];
  73. char annotation[ISO_TIME_LEN+32];
  74. format_iso_time(buf, md->last_listed);
  75. tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
  76. if (fputs(annotation, f) < 0) {
  77. log_warn(LD_DIR,
  78. "Couldn't write microdescriptor annotation: %s",
  79. strerror(ferror(f)));
  80. return -1;
  81. }
  82. r += strlen(annotation);
  83. *annotation_len_out = r;
  84. } else {
  85. *annotation_len_out = 0;
  86. }
  87. md->off = (off_t) ftell(f);
  88. written = fwrite(md->body, 1, md->bodylen, f);
  89. if (written != md->bodylen) {
  90. log_warn(LD_DIR,
  91. "Couldn't dump microdescriptor (wrote %lu out of %lu): %s",
  92. (unsigned long)written, (unsigned long)md->bodylen,
  93. strerror(ferror(f)));
  94. return -1;
  95. }
  96. r += md->bodylen;
  97. return r;
  98. }
  99. /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
  100. * such object has been allocated. */
  101. static microdesc_cache_t *the_microdesc_cache = NULL;
  102. /** Return a pointer to the microdescriptor cache, loading it if necessary. */
  103. microdesc_cache_t *
  104. get_microdesc_cache(void)
  105. {
  106. if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
  107. microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
  108. HT_INIT(microdesc_map, &cache->map);
  109. cache->cache_fname = get_datadir_fname("cached-microdescs");
  110. cache->journal_fname = get_datadir_fname("cached-microdescs.new");
  111. microdesc_cache_reload(cache);
  112. the_microdesc_cache = cache;
  113. }
  114. return the_microdesc_cache;
  115. }
  116. /* There are three sources of microdescriptors:
  117. 1) Generated by us while acting as a directory authority.
  118. 2) Loaded from the cache on disk.
  119. 3) Downloaded.
  120. */
  121. /** Decode the microdescriptors from the string starting at <b>s</b> and
  122. * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>,
  123. * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
  124. * leave their bodies as pointers to the mmap'd cache. If where is
  125. * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive,
  126. * set the last_listed field of every microdesc to listed_at. If
  127. * requested_digests is non-null, then it contains a list of digests we mean
  128. * to allow, so we should reject any non-requested microdesc with a different
  129. * digest, and alter the list to contain only the digests of those microdescs
  130. * we didn't find.
  131. * Return a newly allocated list of the added microdescriptors, or NULL */
  132. smartlist_t *
  133. microdescs_add_to_cache(microdesc_cache_t *cache,
  134. const char *s, const char *eos, saved_location_t where,
  135. int no_save, time_t listed_at,
  136. smartlist_t *requested_digests256)
  137. {
  138. smartlist_t *descriptors, *added;
  139. const int allow_annotations = (where != SAVED_NOWHERE);
  140. const int copy_body = (where != SAVED_IN_CACHE);
  141. descriptors = microdescs_parse_from_string(s, eos,
  142. allow_annotations,
  143. copy_body);
  144. if (listed_at > 0) {
  145. SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
  146. md->last_listed = listed_at);
  147. }
  148. if (requested_digests256) {
  149. digestmap_t *requested; /* XXXX actuqlly we should just use a
  150. digest256map */
  151. requested = digestmap_new();
  152. SMARTLIST_FOREACH(requested_digests256, const char *, cp,
  153. digestmap_set(requested, cp, (void*)1));
  154. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  155. if (digestmap_get(requested, md->digest)) {
  156. digestmap_set(requested, md->digest, (void*)2);
  157. } else {
  158. log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microcdesc");
  159. microdesc_free(md);
  160. SMARTLIST_DEL_CURRENT(descriptors, md);
  161. }
  162. } SMARTLIST_FOREACH_END(md);
  163. SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) {
  164. if (digestmap_get(requested, cp) == (void*)2) {
  165. tor_free(cp);
  166. SMARTLIST_DEL_CURRENT(requested_digests256, cp);
  167. }
  168. } SMARTLIST_FOREACH_END(cp);
  169. digestmap_free(requested, NULL);
  170. }
  171. added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
  172. smartlist_free(descriptors);
  173. return added;
  174. }
  175. /** As microdescs_add_to_cache, but takes a list of micrdescriptors instead of
  176. * a string to decode. Frees any members of <b>descriptors</b> that it does
  177. * not add. */
  178. smartlist_t *
  179. microdescs_add_list_to_cache(microdesc_cache_t *cache,
  180. smartlist_t *descriptors, saved_location_t where,
  181. int no_save)
  182. {
  183. smartlist_t *added;
  184. open_file_t *open_file = NULL;
  185. FILE *f = NULL;
  186. // int n_added = 0;
  187. ssize_t size = 0;
  188. if (where == SAVED_NOWHERE && !no_save) {
  189. f = start_writing_to_stdio_file(cache->journal_fname,
  190. OPEN_FLAGS_APPEND|O_BINARY,
  191. 0600, &open_file);
  192. if (!f) {
  193. log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
  194. cache->journal_fname, strerror(errno));
  195. return NULL;
  196. }
  197. }
  198. added = smartlist_new();
  199. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  200. microdesc_t *md2;
  201. md2 = HT_FIND(microdesc_map, &cache->map, md);
  202. if (md2) {
  203. /* We already had this one. */
  204. if (md2->last_listed < md->last_listed)
  205. md2->last_listed = md->last_listed;
  206. microdesc_free(md);
  207. if (where != SAVED_NOWHERE)
  208. cache->bytes_dropped += size;
  209. continue;
  210. }
  211. /* Okay, it's a new one. */
  212. if (f) {
  213. size_t annotation_len;
  214. size = dump_microdescriptor(f, md, &annotation_len);
  215. if (size < 0) {
  216. /* we already warned in dump_microdescriptor; */
  217. abort_writing_to_file(open_file);
  218. smartlist_clear(added);
  219. return added;
  220. }
  221. md->saved_location = SAVED_IN_JOURNAL;
  222. cache->journal_len += size;
  223. } else {
  224. md->saved_location = where;
  225. }
  226. md->no_save = no_save;
  227. HT_INSERT(microdesc_map, &cache->map, md);
  228. md->held_in_map = 1;
  229. smartlist_add(added, md);
  230. ++cache->n_seen;
  231. cache->total_len_seen += md->bodylen;
  232. } SMARTLIST_FOREACH_END(md);
  233. if (f)
  234. finish_writing_to_file(open_file); /*XXX Check me.*/
  235. {
  236. networkstatus_t *ns = networkstatus_get_latest_consensus();
  237. if (ns && ns->flavor == FLAV_MICRODESC)
  238. SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
  239. }
  240. if (smartlist_len(added))
  241. router_dir_info_changed();
  242. return added;
  243. }
  244. /** Remove every microdescriptor in <b>cache</b>. */
  245. void
  246. microdesc_cache_clear(microdesc_cache_t *cache)
  247. {
  248. microdesc_t **entry, **next;
  249. for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
  250. microdesc_t *md = *entry;
  251. next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
  252. md->held_in_map = 0;
  253. microdesc_free(md);
  254. }
  255. HT_CLEAR(microdesc_map, &cache->map);
  256. if (cache->cache_content) {
  257. tor_munmap_file(cache->cache_content);
  258. cache->cache_content = NULL;
  259. }
  260. cache->total_len_seen = 0;
  261. cache->n_seen = 0;
  262. cache->bytes_dropped = 0;
  263. }
  264. /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
  265. * for the first time. Return 0 on success, -1 on failure. */
  266. int
  267. microdesc_cache_reload(microdesc_cache_t *cache)
  268. {
  269. struct stat st;
  270. char *journal_content;
  271. smartlist_t *added;
  272. tor_mmap_t *mm;
  273. int total = 0;
  274. microdesc_cache_clear(cache);
  275. mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
  276. if (mm) {
  277. added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
  278. SAVED_IN_CACHE, 0, -1, NULL);
  279. if (added) {
  280. total += smartlist_len(added);
  281. smartlist_free(added);
  282. }
  283. }
  284. journal_content = read_file_to_str(cache->journal_fname,
  285. RFTS_IGNORE_MISSING, &st);
  286. if (journal_content) {
  287. cache->journal_len = (size_t) st.st_size;
  288. added = microdescs_add_to_cache(cache, journal_content,
  289. journal_content+st.st_size,
  290. SAVED_IN_JOURNAL, 0, -1, NULL);
  291. if (added) {
  292. total += smartlist_len(added);
  293. smartlist_free(added);
  294. }
  295. tor_free(journal_content);
  296. }
  297. log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
  298. total);
  299. microdesc_cache_rebuild(cache, 0 /* don't force */);
  300. return 0;
  301. }
  302. /** By default, we remove any microdescriptors that have gone at least this
  303. * long without appearing in a current consensus. */
  304. #define TOLERATE_MICRODESC_AGE (7*24*60*60)
  305. /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
  306. * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
  307. * positive, specifically remove microdescriptors that have been unlisted
  308. * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
  309. * if we have no current live microdescriptor consensus.
  310. */
  311. void
  312. microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
  313. {
  314. microdesc_t **mdp, *victim;
  315. int dropped=0, kept=0;
  316. size_t bytes_dropped = 0;
  317. time_t now = time(NULL);
  318. /* If we don't know a live consensus, don't believe last_listed values: we
  319. * might be starting up after being down for a while. */
  320. if (! force &&
  321. ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
  322. return;
  323. if (cutoff <= 0)
  324. cutoff = now - TOLERATE_MICRODESC_AGE;
  325. for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
  326. if ((*mdp)->last_listed < cutoff) {
  327. ++dropped;
  328. victim = *mdp;
  329. mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
  330. victim->held_in_map = 0;
  331. bytes_dropped += victim->bodylen;
  332. microdesc_free(victim);
  333. } else {
  334. ++kept;
  335. mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
  336. }
  337. }
  338. if (dropped) {
  339. log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
  340. dropped,dropped+kept);
  341. cache->bytes_dropped += bytes_dropped;
  342. }
  343. }
  344. static int
  345. should_rebuild_md_cache(microdesc_cache_t *cache)
  346. {
  347. const size_t old_len =
  348. cache->cache_content ? cache->cache_content->size : 0;
  349. const size_t journal_len = cache->journal_len;
  350. const size_t dropped = cache->bytes_dropped;
  351. if (journal_len < 16384)
  352. return 0; /* Don't bother, not enough has happened yet. */
  353. if (dropped > (journal_len + old_len) / 3)
  354. return 1; /* We could save 1/3 or more of the currently used space. */
  355. if (journal_len > old_len / 2)
  356. return 1; /* We should append to the regular file */
  357. return 0;
  358. }
  359. /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  360. * and update every microdesc_t in the cache with pointers to its new
  361. * location. If <b>force</b> is true, do this unconditionally. If
  362. * <b>force</b> is false, do it only if we expect to save space on disk. */
  363. int
  364. microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
  365. {
  366. open_file_t *open_file;
  367. FILE *f;
  368. microdesc_t **mdp;
  369. smartlist_t *wrote;
  370. ssize_t size;
  371. off_t off = 0;
  372. int orig_size, new_size;
  373. if (cache == NULL) {
  374. cache = the_microdesc_cache;
  375. if (cache == NULL)
  376. return 0;
  377. }
  378. /* Remove dead descriptors */
  379. microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
  380. if (!force && !should_rebuild_md_cache(cache))
  381. return 0;
  382. log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
  383. orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
  384. orig_size += (int)cache->journal_len;
  385. f = start_writing_to_stdio_file(cache->cache_fname,
  386. OPEN_FLAGS_REPLACE|O_BINARY,
  387. 0600, &open_file);
  388. if (!f)
  389. return -1;
  390. wrote = smartlist_new();
  391. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  392. microdesc_t *md = *mdp;
  393. size_t annotation_len;
  394. if (md->no_save)
  395. continue;
  396. size = dump_microdescriptor(f, md, &annotation_len);
  397. if (size < 0) {
  398. /* XXX handle errors from dump_microdescriptor() */
  399. /* log? return -1? die? coredump the universe? */
  400. continue;
  401. }
  402. tor_assert(((size_t)size) == annotation_len + md->bodylen);
  403. md->off = off + annotation_len;
  404. off += size;
  405. if (md->saved_location != SAVED_IN_CACHE) {
  406. tor_free(md->body);
  407. md->saved_location = SAVED_IN_CACHE;
  408. }
  409. smartlist_add(wrote, md);
  410. }
  411. if (cache->cache_content)
  412. tor_munmap_file(cache->cache_content);
  413. finish_writing_to_file(open_file); /*XXX Check me.*/
  414. cache->cache_content = tor_mmap_file(cache->cache_fname);
  415. if (!cache->cache_content && smartlist_len(wrote)) {
  416. log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
  417. cache->cache_fname);
  418. smartlist_free(wrote);
  419. return -1;
  420. }
  421. SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
  422. tor_assert(md->saved_location == SAVED_IN_CACHE);
  423. md->body = (char*)cache->cache_content->data + md->off;
  424. if (PREDICT_UNLIKELY(
  425. md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
  426. /* XXXX once bug 2022 is solved, we can kill this block and turn it
  427. * into just the tor_assert(!memcmp) */
  428. off_t avail = cache->cache_content->size - md->off;
  429. char *bad_str;
  430. tor_assert(avail >= 0);
  431. bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
  432. log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
  433. " At offset %d, I expected to find a microdescriptor starting "
  434. " with \"onion-key\". Instead I got %s.",
  435. (int)md->off, escaped(bad_str));
  436. tor_free(bad_str);
  437. tor_assert(fast_memeq(md->body, "onion-key", 9));
  438. }
  439. } SMARTLIST_FOREACH_END(md);
  440. smartlist_free(wrote);
  441. write_str_to_file(cache->journal_fname, "", 1);
  442. cache->journal_len = 0;
  443. cache->bytes_dropped = 0;
  444. new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
  445. log_info(LD_DIR, "Done rebuilding microdesc cache. "
  446. "Saved %d bytes; %d still used.",
  447. orig_size-new_size, new_size);
  448. return 0;
  449. }
  450. /** Make sure that the reference count of every microdescriptor in cache is
  451. * accurate. */
  452. void
  453. microdesc_check_counts(void)
  454. {
  455. microdesc_t **mdp;
  456. if (!the_microdesc_cache)
  457. return;
  458. HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
  459. microdesc_t *md = *mdp;
  460. unsigned int found=0;
  461. const smartlist_t *nodes = nodelist_get_list();
  462. SMARTLIST_FOREACH(nodes, node_t *, node, {
  463. if (node->md == md) {
  464. ++found;
  465. }
  466. });
  467. tor_assert(found == md->held_by_nodes);
  468. }
  469. }
  470. /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
  471. * previously been removed from the cache if it had ever been inserted. */
  472. void
  473. microdesc_free(microdesc_t *md)
  474. {
  475. if (!md)
  476. return;
  477. /* Make sure that the microdesc was really removed from the appropriate data
  478. structures. */
  479. if (md->held_in_map) {
  480. microdesc_cache_t *cache = get_microdesc_cache();
  481. microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
  482. if (md2 == md) {
  483. log_warn(LD_BUG, "microdesc_free() called, but md was still in "
  484. "microdesc_map");
  485. HT_REMOVE(microdesc_map, &cache->map, md);
  486. } else {
  487. log_warn(LD_BUG, "microdesc_free() called with held_in_map set, but "
  488. "microdesc was not in the map.");
  489. }
  490. tor_fragile_assert();
  491. }
  492. if (md->held_by_nodes) {
  493. int found=0;
  494. const smartlist_t *nodes = nodelist_get_list();
  495. SMARTLIST_FOREACH(nodes, node_t *, node, {
  496. if (node->md == md) {
  497. ++found;
  498. node->md = NULL;
  499. }
  500. });
  501. if (found) {
  502. log_warn(LD_BUG, "microdesc_free() called, but md was still referenced "
  503. "%d node(s); held_by_nodes == %u", found, md->held_by_nodes);
  504. } else {
  505. log_warn(LD_BUG, "microdesc_free() called with held_by_nodes set to %u, "
  506. "but md was not referenced by any nodes", md->held_by_nodes);
  507. }
  508. tor_fragile_assert();
  509. }
  510. //tor_assert(md->held_in_map == 0);
  511. //tor_assert(md->held_by_nodes == 0);
  512. if (md->onion_pkey)
  513. crypto_pk_free(md->onion_pkey);
  514. if (md->body && md->saved_location != SAVED_IN_CACHE)
  515. tor_free(md->body);
  516. if (md->family) {
  517. SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
  518. smartlist_free(md->family);
  519. }
  520. short_policy_free(md->exit_policy);
  521. short_policy_free(md->ipv6_exit_policy);
  522. tor_free(md);
  523. }
  524. /** Free all storage held in the microdesc.c module. */
  525. void
  526. microdesc_free_all(void)
  527. {
  528. if (the_microdesc_cache) {
  529. microdesc_cache_clear(the_microdesc_cache);
  530. tor_free(the_microdesc_cache->cache_fname);
  531. tor_free(the_microdesc_cache->journal_fname);
  532. tor_free(the_microdesc_cache);
  533. }
  534. }
  535. /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
  536. * <b>d</b>, return it. Otherwise return NULL. */
  537. microdesc_t *
  538. microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
  539. {
  540. microdesc_t *md, search;
  541. if (!cache)
  542. cache = get_microdesc_cache();
  543. memcpy(search.digest, d, DIGEST256_LEN);
  544. md = HT_FIND(microdesc_map, &cache->map, &search);
  545. return md;
  546. }
  547. /** Return the mean size of decriptors added to <b>cache</b> since it was last
  548. * cleared. Used to estimate the size of large downloads. */
  549. size_t
  550. microdesc_average_size(microdesc_cache_t *cache)
  551. {
  552. if (!cache)
  553. cache = get_microdesc_cache();
  554. if (!cache->n_seen)
  555. return 512;
  556. return (size_t)(cache->total_len_seen / cache->n_seen);
  557. }
  558. /** Return a smartlist of all the sha256 digest of the microdescriptors that
  559. * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  560. * to internals of <b>ns</b>; you should not free the members of the resulting
  561. * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
  562. smartlist_t *
  563. microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
  564. int downloadable_only, digestmap_t *skip)
  565. {
  566. smartlist_t *result = smartlist_new();
  567. time_t now = time(NULL);
  568. tor_assert(ns->flavor == FLAV_MICRODESC);
  569. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  570. if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
  571. continue;
  572. if (downloadable_only &&
  573. !download_status_is_ready(&rs->dl_status, now,
  574. MAX_MICRODESC_DOWNLOAD_FAILURES))
  575. continue;
  576. if (skip && digestmap_get(skip, rs->descriptor_digest))
  577. continue;
  578. if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
  579. continue;
  580. /* XXXX Also skip if we're a noncache and wouldn't use this router.
  581. * XXXX NM Microdesc
  582. */
  583. smartlist_add(result, rs->descriptor_digest);
  584. } SMARTLIST_FOREACH_END(rs);
  585. return result;
  586. }
  587. /** Launch download requests for microdescriptors as appropriate.
  588. *
  589. * Specifically, we should launch download requests if we are configured to
  590. * download mirodescriptors, and there are some microdescriptors listed the
  591. * current microdesc consensus that we don't have, and either we never asked
  592. * for them, or we failed to download them but we're willing to retry.
  593. */
  594. void
  595. update_microdesc_downloads(time_t now)
  596. {
  597. const or_options_t *options = get_options();
  598. networkstatus_t *consensus;
  599. smartlist_t *missing;
  600. digestmap_t *pending;
  601. if (should_delay_dir_fetches(options))
  602. return;
  603. if (directory_too_idle_to_fetch_descriptors(options, now))
  604. return;
  605. consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  606. if (!consensus)
  607. return;
  608. if (!we_fetch_microdescriptors(options))
  609. return;
  610. pending = digestmap_new();
  611. list_pending_microdesc_downloads(pending);
  612. missing = microdesc_list_missing_digest256(consensus,
  613. get_microdesc_cache(),
  614. 1,
  615. pending);
  616. digestmap_free(pending, NULL);
  617. launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
  618. missing, NULL, now);
  619. smartlist_free(missing);
  620. }
  621. /** For every microdescriptor listed in the current microdecriptor consensus,
  622. * update its last_listed field to be at least as recent as the publication
  623. * time of the current microdescriptor consensus.
  624. */
  625. void
  626. update_microdescs_from_networkstatus(time_t now)
  627. {
  628. microdesc_cache_t *cache = get_microdesc_cache();
  629. microdesc_t *md;
  630. networkstatus_t *ns =
  631. networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  632. if (! ns)
  633. return;
  634. tor_assert(ns->flavor == FLAV_MICRODESC);
  635. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  636. md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
  637. if (md && ns->valid_after > md->last_listed)
  638. md->last_listed = ns->valid_after;
  639. } SMARTLIST_FOREACH_END(rs);
  640. }
  641. /** Return true iff we should prefer to use microdescriptors rather than
  642. * routerdescs for building circuits. */
  643. int
  644. we_use_microdescriptors_for_circuits(const or_options_t *options)
  645. {
  646. int ret = options->UseMicrodescriptors;
  647. if (ret == -1) {
  648. /* UseMicrodescriptors is "auto"; we need to decide: */
  649. /* If we are configured to use bridges and one of our bridges doesn't
  650. * know what a microdescriptor is, the answer is no. */
  651. if (options->UseBridges && any_bridges_dont_support_microdescriptors())
  652. return 0;
  653. /* Otherwise, we decide that we'll use microdescriptors iff we are
  654. * not a server, and we're not autofetching everything. */
  655. /* XXX023 what does not being a server have to do with it? also there's
  656. * a partitioning issue here where bridges differ from clients. */
  657. ret = !server_mode(options) && !options->FetchUselessDescriptors;
  658. }
  659. return ret;
  660. }
  661. /** Return true iff we should try to download microdescriptors at all. */
  662. int
  663. we_fetch_microdescriptors(const or_options_t *options)
  664. {
  665. if (directory_caches_dir_info(options))
  666. return 1;
  667. if (options->FetchUselessDescriptors)
  668. return 1;
  669. return we_use_microdescriptors_for_circuits(options);
  670. }
  671. /** Return true iff we should try to download router descriptors at all. */
  672. int
  673. we_fetch_router_descriptors(const or_options_t *options)
  674. {
  675. if (directory_caches_dir_info(options))
  676. return 1;
  677. if (options->FetchUselessDescriptors)
  678. return 1;
  679. return ! we_use_microdescriptors_for_circuits(options);
  680. }
  681. /** Return the consensus flavor we actually want to use to build circuits. */
  682. int
  683. usable_consensus_flavor(void)
  684. {
  685. if (we_use_microdescriptors_for_circuits(get_options())) {
  686. return FLAV_MICRODESC;
  687. } else {
  688. return FLAV_NS;
  689. }
  690. }