microdesc.c 36 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091
  1. /* Copyright (c) 2009-2019, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file microdesc.c
  5. *
  6. * \brief Implements microdescriptors -- an abbreviated description of
  7. * less-frequently-changing router information.
  8. */
  9. #include "core/or/or.h"
  10. #include "lib/fdio/fdio.h"
  11. #include "app/config/config.h"
  12. #include "core/or/circuitbuild.h"
  13. #include "core/or/policies.h"
  14. #include "feature/client/entrynodes.h"
  15. #include "feature/dircache/dirserv.h"
  16. #include "feature/dirclient/dlstatus.h"
  17. #include "feature/dircommon/directory.h"
  18. #include "feature/dirparse/microdesc_parse.h"
  19. #include "feature/nodelist/dirlist.h"
  20. #include "feature/nodelist/microdesc.h"
  21. #include "feature/nodelist/networkstatus.h"
  22. #include "feature/nodelist/nodefamily.h"
  23. #include "feature/nodelist/nodelist.h"
  24. #include "feature/nodelist/routerlist.h"
  25. #include "feature/relay/router.h"
  26. #include "feature/nodelist/microdesc_st.h"
  27. #include "feature/nodelist/networkstatus_st.h"
  28. #include "feature/nodelist/node_st.h"
  29. #include "feature/nodelist/routerstatus_st.h"
  30. #ifdef HAVE_FCNTL_H
  31. #include <fcntl.h>
  32. #endif
  33. #ifdef HAVE_SYS_STAT_H
  34. #include <sys/stat.h>
  35. #endif
  36. /** A data structure to hold a bunch of cached microdescriptors. There are
  37. * two active files in the cache: a "cache file" that we mmap, and a "journal
  38. * file" that we append to. Periodically, we rebuild the cache file to hold
  39. * only the microdescriptors that we want to keep */
  40. struct microdesc_cache_t {
  41. /** Map from sha256-digest to microdesc_t for every microdesc_t in the
  42. * cache. */
  43. HT_HEAD(microdesc_map, microdesc_t) map;
  44. /** Name of the cache file. */
  45. char *cache_fname;
  46. /** Name of the journal file. */
  47. char *journal_fname;
  48. /** Mmap'd contents of the cache file, or NULL if there is none. */
  49. tor_mmap_t *cache_content;
  50. /** Number of bytes used in the journal file. */
  51. size_t journal_len;
  52. /** Number of bytes in descriptors removed as too old. */
  53. size_t bytes_dropped;
  54. /** Total bytes of microdescriptor bodies we have added to this cache */
  55. uint64_t total_len_seen;
  56. /** Total number of microdescriptors we have added to this cache */
  57. unsigned n_seen;
  58. /** True iff we have loaded this cache from disk ever. */
  59. int is_loaded;
  60. };
  61. static microdesc_cache_t *get_microdesc_cache_noload(void);
  62. static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
  63. const char *activity);
  64. /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
  65. static inline unsigned int
  66. microdesc_hash_(microdesc_t *md)
  67. {
  68. return (unsigned) siphash24g(md->digest, sizeof(md->digest));
  69. }
  70. /** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
  71. * purposes. */
  72. static inline int
  73. microdesc_eq_(microdesc_t *a, microdesc_t *b)
  74. {
  75. return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
  76. }
  77. HT_PROTOTYPE(microdesc_map, microdesc_t, node,
  78. microdesc_hash_, microdesc_eq_)
  79. HT_GENERATE2(microdesc_map, microdesc_t, node,
  80. microdesc_hash_, microdesc_eq_, 0.6,
  81. tor_reallocarray_, tor_free_)
  82. /************************* md fetch fail cache *****************************/
  83. /* If we end up with too many outdated dirservers, something probably went
  84. * wrong so clean up the list. */
  85. #define TOO_MANY_OUTDATED_DIRSERVERS 30
  86. /** List of dirservers with outdated microdesc information. The smartlist is
  87. * filled with the hex digests of outdated dirservers. */
  88. static smartlist_t *outdated_dirserver_list = NULL;
  89. /** Note that we failed to fetch a microdescriptor from the relay with
  90. * <b>relay_digest</b> (of size DIGEST_LEN). */
  91. void
  92. microdesc_note_outdated_dirserver(const char *relay_digest)
  93. {
  94. char relay_hexdigest[HEX_DIGEST_LEN+1];
  95. /* If we have a reasonably live consensus, then most of our dirservers should
  96. * still be caching all the microdescriptors in it. Reasonably live
  97. * consensuses are up to a day old (or a day in the future). But
  98. * microdescriptors expire 7 days after the last consensus that referenced
  99. * them. */
  100. if (!networkstatus_get_reasonably_live_consensus(approx_time(),
  101. FLAV_MICRODESC)) {
  102. return;
  103. }
  104. if (!outdated_dirserver_list) {
  105. outdated_dirserver_list = smartlist_new();
  106. }
  107. tor_assert(outdated_dirserver_list);
  108. /* If the list grows too big, clean it up */
  109. if (BUG(smartlist_len(outdated_dirserver_list) >
  110. TOO_MANY_OUTDATED_DIRSERVERS)) {
  111. microdesc_reset_outdated_dirservers_list();
  112. }
  113. /* Turn the binary relay digest to a hex since smartlists have better support
  114. * for strings than digests. */
  115. base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
  116. relay_digest, DIGEST_LEN);
  117. /* Make sure we don't add a dirauth as an outdated dirserver */
  118. if (router_get_trusteddirserver_by_digest(relay_digest)) {
  119. log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
  120. return;
  121. }
  122. /* Don't double-add outdated dirservers */
  123. if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
  124. return;
  125. }
  126. /* Add it to the list of outdated dirservers */
  127. smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
  128. log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
  129. }
  130. /** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
  131. * outdated dirserver */
  132. int
  133. microdesc_relay_is_outdated_dirserver(const char *relay_digest)
  134. {
  135. char relay_hexdigest[HEX_DIGEST_LEN+1];
  136. if (!outdated_dirserver_list) {
  137. return 0;
  138. }
  139. /* Convert identity digest to hex digest */
  140. base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
  141. relay_digest, DIGEST_LEN);
  142. /* Last time we tried to fetch microdescs, was this directory mirror missing
  143. * any mds we asked for? */
  144. if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
  145. return 1;
  146. }
  147. return 0;
  148. }
  149. /** Reset the list of outdated dirservers. */
  150. void
  151. microdesc_reset_outdated_dirservers_list(void)
  152. {
  153. if (!outdated_dirserver_list) {
  154. return;
  155. }
  156. SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
  157. smartlist_clear(outdated_dirserver_list);
  158. }
  159. /****************************************************************************/
  160. /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  161. * On success, return the total number of bytes written, and set
  162. * *<b>annotation_len_out</b> to the number of bytes written as
  163. * annotations. */
  164. static ssize_t
  165. dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
  166. {
  167. ssize_t r = 0;
  168. ssize_t written;
  169. if (md->body == NULL) {
  170. *annotation_len_out = 0;
  171. return 0;
  172. }
  173. /* XXXX drops unknown annotations. */
  174. if (md->last_listed) {
  175. char buf[ISO_TIME_LEN+1];
  176. char annotation[ISO_TIME_LEN+32];
  177. format_iso_time(buf, md->last_listed);
  178. tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
  179. if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
  180. log_warn(LD_DIR,
  181. "Couldn't write microdescriptor annotation: %s",
  182. strerror(errno));
  183. return -1;
  184. }
  185. r += strlen(annotation);
  186. *annotation_len_out = r;
  187. } else {
  188. *annotation_len_out = 0;
  189. }
  190. md->off = tor_fd_getpos(fd);
  191. warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
  192. "dumping a microdescriptor");
  193. written = write_all_to_fd(fd, md->body, md->bodylen);
  194. if (written != (ssize_t)md->bodylen) {
  195. written = written < 0 ? 0 : written;
  196. log_warn(LD_DIR,
  197. "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
  198. (long)written, (unsigned long)md->bodylen,
  199. strerror(errno));
  200. return -1;
  201. }
  202. r += md->bodylen;
  203. return r;
  204. }
  205. /** Holds a pointer to the current microdesc_cache_t object, or NULL if no
  206. * such object has been allocated. */
  207. static microdesc_cache_t *the_microdesc_cache = NULL;
  208. /** Return a pointer to the microdescriptor cache, loading it if necessary. */
  209. microdesc_cache_t *
  210. get_microdesc_cache(void)
  211. {
  212. microdesc_cache_t *cache = get_microdesc_cache_noload();
  213. if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
  214. microdesc_cache_reload(cache);
  215. }
  216. return cache;
  217. }
  218. /** Return a pointer to the microdescriptor cache, creating (but not loading)
  219. * it if necessary. */
  220. static microdesc_cache_t *
  221. get_microdesc_cache_noload(void)
  222. {
  223. if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
  224. microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
  225. HT_INIT(microdesc_map, &cache->map);
  226. cache->cache_fname = get_cachedir_fname("cached-microdescs");
  227. cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
  228. the_microdesc_cache = cache;
  229. }
  230. return the_microdesc_cache;
  231. }
  232. /* There are three sources of microdescriptors:
  233. 1) Generated by us while acting as a directory authority.
  234. 2) Loaded from the cache on disk.
  235. 3) Downloaded.
  236. */
  237. /** Decode the microdescriptors from the string starting at <b>s</b> and
  238. * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
  239. * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
  240. * leave their bodies as pointers to the mmap'd cache. If where is
  241. * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
  242. * set the last_listed field of every microdesc to listed_at. If
  243. * requested_digests is non-null, then it contains a list of digests we mean
  244. * to allow, so we should reject any non-requested microdesc with a different
  245. * digest, and alter the list to contain only the digests of those microdescs
  246. * we didn't find.
  247. * Return a newly allocated list of the added microdescriptors, or NULL */
  248. smartlist_t *
  249. microdescs_add_to_cache(microdesc_cache_t *cache,
  250. const char *s, const char *eos, saved_location_t where,
  251. int no_save, time_t listed_at,
  252. smartlist_t *requested_digests256)
  253. {
  254. void * const DIGEST_REQUESTED = (void*)1;
  255. void * const DIGEST_RECEIVED = (void*)2;
  256. void * const DIGEST_INVALID = (void*)3;
  257. smartlist_t *descriptors, *added;
  258. const int allow_annotations = (where != SAVED_NOWHERE);
  259. smartlist_t *invalid_digests = smartlist_new();
  260. descriptors = microdescs_parse_from_string(s, eos,
  261. allow_annotations,
  262. where, invalid_digests);
  263. if (listed_at != (time_t)-1) {
  264. SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
  265. md->last_listed = listed_at);
  266. }
  267. if (requested_digests256) {
  268. digest256map_t *requested;
  269. requested = digest256map_new();
  270. /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
  271. SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
  272. digest256map_set(requested, cp, DIGEST_REQUESTED));
  273. /* Set requested[d] to DIGEST_INVALID for every md we requested which we
  274. * will never be able to parse. Remove the ones we didn't request from
  275. * invalid_digests.
  276. */
  277. SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
  278. if (digest256map_get(requested, cp)) {
  279. digest256map_set(requested, cp, DIGEST_INVALID);
  280. } else {
  281. tor_free(cp);
  282. SMARTLIST_DEL_CURRENT(invalid_digests, cp);
  283. }
  284. } SMARTLIST_FOREACH_END(cp);
  285. /* Update requested[d] to 2 for the mds we asked for and got. Delete the
  286. * ones we never requested from the 'descriptors' smartlist.
  287. */
  288. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  289. if (digest256map_get(requested, (const uint8_t*)md->digest)) {
  290. digest256map_set(requested, (const uint8_t*)md->digest,
  291. DIGEST_RECEIVED);
  292. } else {
  293. log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
  294. microdesc_free(md);
  295. SMARTLIST_DEL_CURRENT(descriptors, md);
  296. }
  297. } SMARTLIST_FOREACH_END(md);
  298. /* Remove the ones we got or the invalid ones from requested_digests256.
  299. */
  300. SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
  301. void *status = digest256map_get(requested, cp);
  302. if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
  303. tor_free(cp);
  304. SMARTLIST_DEL_CURRENT(requested_digests256, cp);
  305. }
  306. } SMARTLIST_FOREACH_END(cp);
  307. digest256map_free(requested, NULL);
  308. }
  309. /* For every requested microdescriptor that was unparseable, mark it
  310. * as not to be retried. */
  311. if (smartlist_len(invalid_digests)) {
  312. networkstatus_t *ns =
  313. networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
  314. if (ns) {
  315. SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
  316. routerstatus_t *rs =
  317. router_get_mutable_consensus_status_by_descriptor_digest(ns, d);
  318. if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
  319. download_status_mark_impossible(&rs->dl_status);
  320. }
  321. } SMARTLIST_FOREACH_END(d);
  322. }
  323. }
  324. SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
  325. smartlist_free(invalid_digests);
  326. added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
  327. smartlist_free(descriptors);
  328. return added;
  329. }
  330. /** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
  331. * a string to decode. Frees any members of <b>descriptors</b> that it does
  332. * not add. */
  333. smartlist_t *
  334. microdescs_add_list_to_cache(microdesc_cache_t *cache,
  335. smartlist_t *descriptors, saved_location_t where,
  336. int no_save)
  337. {
  338. smartlist_t *added;
  339. open_file_t *open_file = NULL;
  340. int fd = -1;
  341. // int n_added = 0;
  342. ssize_t size = 0;
  343. if (where == SAVED_NOWHERE && !no_save) {
  344. fd = start_writing_to_file(cache->journal_fname,
  345. OPEN_FLAGS_APPEND|O_BINARY,
  346. 0600, &open_file);
  347. if (fd < 0) {
  348. log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
  349. cache->journal_fname, strerror(errno));
  350. }
  351. }
  352. added = smartlist_new();
  353. SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
  354. microdesc_t *md2;
  355. md2 = HT_FIND(microdesc_map, &cache->map, md);
  356. if (md2) {
  357. /* We already had this one. */
  358. if (md2->last_listed < md->last_listed)
  359. md2->last_listed = md->last_listed;
  360. microdesc_free(md);
  361. if (where != SAVED_NOWHERE)
  362. cache->bytes_dropped += size;
  363. continue;
  364. }
  365. /* Okay, it's a new one. */
  366. if (fd >= 0) {
  367. size_t annotation_len;
  368. size = dump_microdescriptor(fd, md, &annotation_len);
  369. if (size < 0) {
  370. /* we already warned in dump_microdescriptor */
  371. abort_writing_to_file(open_file);
  372. fd = -1;
  373. } else {
  374. md->saved_location = SAVED_IN_JOURNAL;
  375. cache->journal_len += size;
  376. }
  377. } else {
  378. md->saved_location = where;
  379. }
  380. md->no_save = no_save;
  381. HT_INSERT(microdesc_map, &cache->map, md);
  382. md->held_in_map = 1;
  383. smartlist_add(added, md);
  384. ++cache->n_seen;
  385. cache->total_len_seen += md->bodylen;
  386. } SMARTLIST_FOREACH_END(md);
  387. if (fd >= 0) {
  388. if (finish_writing_to_file(open_file) < 0) {
  389. log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
  390. strerror(errno));
  391. smartlist_clear(added);
  392. return added;
  393. }
  394. }
  395. {
  396. networkstatus_t *ns = networkstatus_get_latest_consensus();
  397. if (ns && ns->flavor == FLAV_MICRODESC)
  398. SMARTLIST_FOREACH(added, microdesc_t *, md, nodelist_add_microdesc(md));
  399. }
  400. if (smartlist_len(added))
  401. router_dir_info_changed();
  402. return added;
  403. }
  404. /** Remove every microdescriptor in <b>cache</b>. */
  405. void
  406. microdesc_cache_clear(microdesc_cache_t *cache)
  407. {
  408. microdesc_t **entry, **next;
  409. for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
  410. microdesc_t *md = *entry;
  411. next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
  412. md->held_in_map = 0;
  413. microdesc_free(md);
  414. }
  415. HT_CLEAR(microdesc_map, &cache->map);
  416. if (cache->cache_content) {
  417. int res = tor_munmap_file(cache->cache_content);
  418. if (res != 0) {
  419. log_warn(LD_FS,
  420. "tor_munmap_file() failed clearing microdesc cache; "
  421. "we are probably about to leak memory.");
  422. /* TODO something smarter? */
  423. }
  424. cache->cache_content = NULL;
  425. }
  426. cache->total_len_seen = 0;
  427. cache->n_seen = 0;
  428. cache->bytes_dropped = 0;
  429. }
  430. static void
  431. warn_if_nul_found(const char *inp, size_t len, int64_t offset,
  432. const char *activity)
  433. {
  434. const char *nul_found = memchr(inp, 0, len);
  435. if (BUG(nul_found)) {
  436. log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
  437. "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
  438. activity, offset, (nul_found - inp), len);
  439. const char *start_excerpt_at, *eos = inp + len;
  440. if ((nul_found - inp) >= 16)
  441. start_excerpt_at = nul_found - 16;
  442. else
  443. start_excerpt_at = inp;
  444. size_t excerpt_len = MIN(32, eos - start_excerpt_at);
  445. char tmp[65];
  446. base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
  447. log_warn(LD_BUG, " surrounding string: %s", tmp);
  448. }
  449. }
  450. /** Reload the contents of <b>cache</b> from disk. If it is empty, load it
  451. * for the first time. Return 0 on success, -1 on failure. */
  452. int
  453. microdesc_cache_reload(microdesc_cache_t *cache)
  454. {
  455. struct stat st;
  456. char *journal_content;
  457. smartlist_t *added;
  458. tor_mmap_t *mm;
  459. int total = 0;
  460. microdesc_cache_clear(cache);
  461. cache->is_loaded = 1;
  462. mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
  463. if (mm) {
  464. warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
  465. added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
  466. SAVED_IN_CACHE, 0, -1, NULL);
  467. if (added) {
  468. total += smartlist_len(added);
  469. smartlist_free(added);
  470. }
  471. }
  472. journal_content = read_file_to_str(cache->journal_fname,
  473. RFTS_IGNORE_MISSING, &st);
  474. if (journal_content) {
  475. cache->journal_len = (size_t) st.st_size;
  476. warn_if_nul_found(journal_content, cache->journal_len, 0,
  477. "reading microdesc journal");
  478. added = microdescs_add_to_cache(cache, journal_content,
  479. journal_content+st.st_size,
  480. SAVED_IN_JOURNAL, 0, -1, NULL);
  481. if (added) {
  482. total += smartlist_len(added);
  483. smartlist_free(added);
  484. }
  485. tor_free(journal_content);
  486. }
  487. log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
  488. total);
  489. microdesc_cache_rebuild(cache, 0 /* don't force */);
  490. return 0;
  491. }
  492. /** By default, we remove any microdescriptors that have gone at least this
  493. * long without appearing in a current consensus. */
  494. #define TOLERATE_MICRODESC_AGE (7*24*60*60)
  495. /** Remove all microdescriptors from <b>cache</b> that haven't been listed for
  496. * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
  497. * positive, specifically remove microdescriptors that have been unlisted
  498. * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
  499. * if we have no current live microdescriptor consensus.
  500. */
  501. void
  502. microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
  503. {
  504. microdesc_t **mdp, *victim;
  505. int dropped=0, kept=0;
  506. size_t bytes_dropped = 0;
  507. time_t now = time(NULL);
  508. /* If we don't know a reasonably live consensus, don't believe last_listed
  509. * values: we might be starting up after being down for a while. */
  510. if (! force &&
  511. ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
  512. return;
  513. if (cutoff <= 0)
  514. cutoff = now - TOLERATE_MICRODESC_AGE;
  515. for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
  516. const int is_old = (*mdp)->last_listed < cutoff;
  517. const unsigned held_by_nodes = (*mdp)->held_by_nodes;
  518. if (is_old && !held_by_nodes) {
  519. ++dropped;
  520. victim = *mdp;
  521. mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
  522. victim->held_in_map = 0;
  523. bytes_dropped += victim->bodylen;
  524. microdesc_free(victim);
  525. } else {
  526. if (is_old) {
  527. /* It's old, but it has held_by_nodes set. That's not okay. */
  528. /* Let's try to diagnose and fix #7164 . */
  529. smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp);
  530. const networkstatus_t *ns = networkstatus_get_latest_consensus();
  531. long networkstatus_age = -1;
  532. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  533. if (ns) {
  534. networkstatus_age = now - ns->valid_after;
  535. }
  536. log_warn(LD_BUG, "Microdescriptor seemed very old "
  537. "(last listed %d hours ago vs %d hour cutoff), but is still "
  538. "marked as being held by %d node(s). I found %d node(s) "
  539. "holding it. Current networkstatus is %ld hours old. "
  540. "Hashtable badness is %d.",
  541. (int)((now - (*mdp)->last_listed) / 3600),
  542. (int)((now - cutoff) / 3600),
  543. held_by_nodes,
  544. smartlist_len(nodes),
  545. networkstatus_age / 3600,
  546. ht_badness);
  547. SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
  548. const char *rs_match = "No RS";
  549. const char *rs_present = "";
  550. if (node->rs) {
  551. if (tor_memeq(node->rs->descriptor_digest,
  552. (*mdp)->digest, DIGEST256_LEN)) {
  553. rs_match = "Microdesc digest in RS matches";
  554. } else {
  555. rs_match = "Microdesc digest in RS does match";
  556. }
  557. if (ns) {
  558. /* This should be impossible, but let's see! */
  559. rs_present = " RS not present in networkstatus.";
  560. SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, {
  561. if (rs == node->rs) {
  562. rs_present = " RS okay in networkstatus.";
  563. }
  564. });
  565. }
  566. }
  567. log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
  568. node_sl_idx,
  569. hex_str(node->identity, DIGEST_LEN),
  570. node->md, node->rs, node->ri, rs_match, rs_present);
  571. } SMARTLIST_FOREACH_END(node);
  572. smartlist_free(nodes);
  573. (*mdp)->last_listed = now;
  574. }
  575. ++kept;
  576. mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
  577. }
  578. }
  579. if (dropped) {
  580. log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
  581. dropped,dropped+kept);
  582. cache->bytes_dropped += bytes_dropped;
  583. }
  584. }
  585. static int
  586. should_rebuild_md_cache(microdesc_cache_t *cache)
  587. {
  588. const size_t old_len =
  589. cache->cache_content ? cache->cache_content->size : 0;
  590. const size_t journal_len = cache->journal_len;
  591. const size_t dropped = cache->bytes_dropped;
  592. if (journal_len < 16384)
  593. return 0; /* Don't bother, not enough has happened yet. */
  594. if (dropped > (journal_len + old_len) / 3)
  595. return 1; /* We could save 1/3 or more of the currently used space. */
  596. if (journal_len > old_len / 2)
  597. return 1; /* We should append to the regular file */
  598. return 0;
  599. }
  600. /**
  601. * Mark <b>md</b> as having no body, and release any storage previously held
  602. * by its body.
  603. */
  604. static void
  605. microdesc_wipe_body(microdesc_t *md)
  606. {
  607. if (!md)
  608. return;
  609. if (md->saved_location != SAVED_IN_CACHE)
  610. tor_free(md->body);
  611. md->off = 0;
  612. md->saved_location = SAVED_NOWHERE;
  613. md->body = NULL;
  614. md->bodylen = 0;
  615. md->no_save = 1;
  616. }
  617. /** Regenerate the main cache file for <b>cache</b>, clear the journal file,
  618. * and update every microdesc_t in the cache with pointers to its new
  619. * location. If <b>force</b> is true, do this unconditionally. If
  620. * <b>force</b> is false, do it only if we expect to save space on disk. */
  621. int
  622. microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
  623. {
  624. open_file_t *open_file;
  625. int fd = -1, res;
  626. microdesc_t **mdp;
  627. smartlist_t *wrote;
  628. ssize_t size;
  629. off_t off = 0, off_real;
  630. int orig_size, new_size;
  631. if (cache == NULL) {
  632. cache = the_microdesc_cache;
  633. if (cache == NULL)
  634. return 0;
  635. }
  636. /* Remove dead descriptors */
  637. microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
  638. if (!force && !should_rebuild_md_cache(cache))
  639. return 0;
  640. log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
  641. orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
  642. orig_size += (int)cache->journal_len;
  643. fd = start_writing_to_file(cache->cache_fname,
  644. OPEN_FLAGS_REPLACE|O_BINARY,
  645. 0600, &open_file);
  646. if (fd < 0)
  647. return -1;
  648. wrote = smartlist_new();
  649. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  650. microdesc_t *md = *mdp;
  651. size_t annotation_len;
  652. if (md->no_save || !md->body)
  653. continue;
  654. size = dump_microdescriptor(fd, md, &annotation_len);
  655. if (size < 0) {
  656. microdesc_wipe_body(md);
  657. /* rewind, in case it was a partial write. */
  658. tor_fd_setpos(fd, off);
  659. continue;
  660. }
  661. tor_assert(((size_t)size) == annotation_len + md->bodylen);
  662. md->off = off + annotation_len;
  663. off += size;
  664. off_real = tor_fd_getpos(fd);
  665. if (off_real != off) {
  666. log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
  667. "By my count, I'm at %"PRId64
  668. ", but I should be at %"PRId64,
  669. (int64_t)(off), (int64_t)(off_real));
  670. if (off_real >= 0)
  671. off = off_real;
  672. }
  673. if (md->saved_location != SAVED_IN_CACHE) {
  674. tor_free(md->body);
  675. md->saved_location = SAVED_IN_CACHE;
  676. }
  677. smartlist_add(wrote, md);
  678. }
  679. /* We must do this unmap _before_ we call finish_writing_to_file(), or
  680. * windows will not actually replace the file. */
  681. if (cache->cache_content) {
  682. res = tor_munmap_file(cache->cache_content);
  683. if (res != 0) {
  684. log_warn(LD_FS,
  685. "Failed to unmap old microdescriptor cache while rebuilding");
  686. }
  687. cache->cache_content = NULL;
  688. }
  689. if (finish_writing_to_file(open_file) < 0) {
  690. log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
  691. strerror(errno));
  692. /* Okay. Let's prevent from making things worse elsewhere. */
  693. cache->cache_content = NULL;
  694. HT_FOREACH(mdp, microdesc_map, &cache->map) {
  695. microdesc_t *md = *mdp;
  696. if (md->saved_location == SAVED_IN_CACHE) {
  697. microdesc_wipe_body(md);
  698. }
  699. }
  700. smartlist_free(wrote);
  701. return -1;
  702. }
  703. cache->cache_content = tor_mmap_file(cache->cache_fname);
  704. if (!cache->cache_content && smartlist_len(wrote)) {
  705. log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
  706. cache->cache_fname);
  707. smartlist_free(wrote);
  708. return -1;
  709. }
  710. SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) {
  711. tor_assert(md->saved_location == SAVED_IN_CACHE);
  712. md->body = (char*)cache->cache_content->data + md->off;
  713. if (PREDICT_UNLIKELY(
  714. md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
  715. /* XXXX once bug 2022 is solved, we can kill this block and turn it
  716. * into just the tor_assert(fast_memeq) */
  717. off_t avail = cache->cache_content->size - md->off;
  718. char *bad_str;
  719. tor_assert(avail >= 0);
  720. bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
  721. log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
  722. " At offset %d, I expected to find a microdescriptor starting "
  723. " with \"onion-key\". Instead I got %s.",
  724. (int)md->off, escaped(bad_str));
  725. tor_free(bad_str);
  726. tor_assert(fast_memeq(md->body, "onion-key", 9));
  727. }
  728. } SMARTLIST_FOREACH_END(md);
  729. smartlist_free(wrote);
  730. write_str_to_file(cache->journal_fname, "", 1);
  731. cache->journal_len = 0;
  732. cache->bytes_dropped = 0;
  733. new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
  734. log_info(LD_DIR, "Done rebuilding microdesc cache. "
  735. "Saved %d bytes; %d still used.",
  736. orig_size-new_size, new_size);
  737. return 0;
  738. }
  739. /** Make sure that the reference count of every microdescriptor in cache is
  740. * accurate. */
  741. void
  742. microdesc_check_counts(void)
  743. {
  744. microdesc_t **mdp;
  745. if (!the_microdesc_cache)
  746. return;
  747. HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
  748. microdesc_t *md = *mdp;
  749. unsigned int found=0;
  750. const smartlist_t *nodes = nodelist_get_list();
  751. SMARTLIST_FOREACH(nodes, node_t *, node, {
  752. if (node->md == md) {
  753. ++found;
  754. }
  755. });
  756. tor_assert(found == md->held_by_nodes);
  757. }
  758. }
  759. /** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
  760. * previously been removed from the cache if it had ever been inserted. */
  761. void
  762. microdesc_free_(microdesc_t *md, const char *fname, int lineno)
  763. {
  764. if (!md)
  765. return;
  766. /* Make sure that the microdesc was really removed from the appropriate data
  767. structures. */
  768. if (md->held_in_map) {
  769. microdesc_cache_t *cache = get_microdesc_cache_noload();
  770. microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
  771. if (md2 == md) {
  772. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  773. "in microdesc_map", fname, lineno);
  774. HT_REMOVE(microdesc_map, &cache->map, md);
  775. } else {
  776. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
  777. "set, but microdesc was not in the map.", fname, lineno);
  778. }
  779. tor_fragile_assert();
  780. }
  781. if (md->held_by_nodes) {
  782. microdesc_cache_t *cache = get_microdesc_cache_noload();
  783. int found=0;
  784. const smartlist_t *nodes = nodelist_get_list();
  785. const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
  786. SMARTLIST_FOREACH(nodes, node_t *, node, {
  787. if (node->md == md) {
  788. ++found;
  789. node->md = NULL;
  790. }
  791. });
  792. if (found) {
  793. log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
  794. "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
  795. fname, lineno, found, md->held_by_nodes, ht_badness);
  796. } else {
  797. log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
  798. "set to %u, but md was not referenced by any nodes. "
  799. "ht_badness == %d",
  800. fname, lineno, md->held_by_nodes, ht_badness);
  801. }
  802. tor_fragile_assert();
  803. }
  804. //tor_assert(md->held_in_map == 0);
  805. //tor_assert(md->held_by_nodes == 0);
  806. if (md->onion_pkey)
  807. tor_free(md->onion_pkey);
  808. tor_free(md->onion_curve25519_pkey);
  809. tor_free(md->ed25519_identity_pkey);
  810. if (md->body && md->saved_location != SAVED_IN_CACHE)
  811. tor_free(md->body);
  812. nodefamily_free(md->family);
  813. short_policy_free(md->exit_policy);
  814. short_policy_free(md->ipv6_exit_policy);
  815. tor_free(md);
  816. }
  817. /** Free all storage held in the microdesc.c module. */
  818. void
  819. microdesc_free_all(void)
  820. {
  821. if (the_microdesc_cache) {
  822. microdesc_cache_clear(the_microdesc_cache);
  823. tor_free(the_microdesc_cache->cache_fname);
  824. tor_free(the_microdesc_cache->journal_fname);
  825. tor_free(the_microdesc_cache);
  826. }
  827. if (outdated_dirserver_list) {
  828. SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
  829. smartlist_free(outdated_dirserver_list);
  830. }
  831. }
  832. /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
  833. * <b>d</b>, return it. Otherwise return NULL. */
  834. microdesc_t *
  835. microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
  836. {
  837. microdesc_t *md, search;
  838. if (!cache)
  839. cache = get_microdesc_cache();
  840. memcpy(search.digest, d, DIGEST256_LEN);
  841. md = HT_FIND(microdesc_map, &cache->map, &search);
  842. return md;
  843. }
  844. /** Return a smartlist of all the sha256 digest of the microdescriptors that
  845. * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  846. * to internals of <b>ns</b>; you should not free the members of the resulting
  847. * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
  848. smartlist_t *
  849. microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
  850. int downloadable_only, digest256map_t *skip)
  851. {
  852. smartlist_t *result = smartlist_new();
  853. time_t now = time(NULL);
  854. tor_assert(ns->flavor == FLAV_MICRODESC);
  855. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  856. if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
  857. continue;
  858. if (downloadable_only &&
  859. !download_status_is_ready(&rs->dl_status, now))
  860. continue;
  861. if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
  862. continue;
  863. if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
  864. continue;
  865. /* XXXX Also skip if we're a noncache and wouldn't use this router.
  866. * XXXX NM Microdesc
  867. */
  868. smartlist_add(result, rs->descriptor_digest);
  869. } SMARTLIST_FOREACH_END(rs);
  870. return result;
  871. }
  872. /** Launch download requests for microdescriptors as appropriate.
  873. *
  874. * Specifically, we should launch download requests if we are configured to
  875. * download mirodescriptors, and there are some microdescriptors listed in the
  876. * current microdesc consensus that we don't have, and either we never asked
  877. * for them, or we failed to download them but we're willing to retry.
  878. */
  879. void
  880. update_microdesc_downloads(time_t now)
  881. {
  882. const or_options_t *options = get_options();
  883. networkstatus_t *consensus;
  884. smartlist_t *missing;
  885. digest256map_t *pending;
  886. if (should_delay_dir_fetches(options, NULL))
  887. return;
  888. if (directory_too_idle_to_fetch_descriptors(options, now))
  889. return;
  890. /* Give up if we don't have a reasonably live consensus. */
  891. consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  892. if (!consensus)
  893. return;
  894. if (!we_fetch_microdescriptors(options))
  895. return;
  896. pending = digest256map_new();
  897. list_pending_microdesc_downloads(pending);
  898. missing = microdesc_list_missing_digest256(consensus,
  899. get_microdesc_cache(),
  900. 1,
  901. pending);
  902. digest256map_free(pending, NULL);
  903. launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC,
  904. missing, NULL, now);
  905. smartlist_free(missing);
  906. }
  907. /** For every microdescriptor listed in the current microdescriptor consensus,
  908. * update its last_listed field to be at least as recent as the publication
  909. * time of the current microdescriptor consensus.
  910. */
  911. void
  912. update_microdescs_from_networkstatus(time_t now)
  913. {
  914. microdesc_cache_t *cache = get_microdesc_cache();
  915. microdesc_t *md;
  916. networkstatus_t *ns =
  917. networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
  918. if (! ns)
  919. return;
  920. tor_assert(ns->flavor == FLAV_MICRODESC);
  921. SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
  922. md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
  923. if (md && ns->valid_after > md->last_listed)
  924. md->last_listed = ns->valid_after;
  925. } SMARTLIST_FOREACH_END(rs);
  926. }
  927. /** Return true iff we should prefer to use microdescriptors rather than
  928. * routerdescs for building circuits. */
  929. int
  930. we_use_microdescriptors_for_circuits(const or_options_t *options)
  931. {
  932. if (options->UseMicrodescriptors == 0)
  933. return 0; /* the user explicitly picked no */
  934. return 1; /* yes and auto both mean yes */
  935. }
  936. /** Return true iff we should try to download microdescriptors at all. */
  937. int
  938. we_fetch_microdescriptors(const or_options_t *options)
  939. {
  940. if (directory_caches_dir_info(options))
  941. return 1;
  942. if (options->FetchUselessDescriptors)
  943. return 1;
  944. return we_use_microdescriptors_for_circuits(options);
  945. }
  946. /** Return true iff we should try to download router descriptors at all. */
  947. int
  948. we_fetch_router_descriptors(const or_options_t *options)
  949. {
  950. if (directory_caches_dir_info(options))
  951. return 1;
  952. if (options->FetchUselessDescriptors)
  953. return 1;
  954. return ! we_use_microdescriptors_for_circuits(options);
  955. }
  956. /** Return the consensus flavor we actually want to use to build circuits. */
  957. MOCK_IMPL(int,
  958. usable_consensus_flavor,(void))
  959. {
  960. if (we_use_microdescriptors_for_circuits(get_options())) {
  961. return FLAV_MICRODESC;
  962. } else {
  963. return FLAV_NS;
  964. }
  965. }