conscache.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541
  1. /* Copyright (c) 2017, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. #include "or.h"
  4. #include "config.h"
  5. #include "conscache.h"
  6. #include "storagedir.h"
  7. #define CCE_MAGIC 0x17162253
  8. /**
  9. * A consensus_cache_entry_t is a reference-counted handle to an
  10. * item in a consensus_cache_t. It can be mmapped into RAM, or not,
  11. * depending whether it's currently in use.
  12. */
  13. struct consensus_cache_entry_t {
  14. uint32_t magic; /**< Must be set to CCE_MAGIC */
  15. HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
  16. int32_t refcnt; /**< Reference count. */
  17. unsigned can_remove : 1; /**< If true, we want to delete this file. */
  18. /** If true, we intend to unmap this file as soon as we're done with it. */
  19. unsigned release_aggressively : 1;
  20. /** Filename for this object within the storage_dir_t */
  21. char *fname;
  22. /** Labels associated with this object. Immutable once the object
  23. * is created. */
  24. config_line_t *labels;
  25. /** Pointer to the cache that includes this entry (if any). */
  26. consensus_cache_t *in_cache;
  27. /** Since what time has this object been mapped into RAM, but with the cache
  28. * being the only having a reference to it? */
  29. time_t unused_since;
  30. /** mmaped contents of the underlying file. May be NULL */
  31. tor_mmap_t *map;
  32. /** Length of the body within <b>map</b>. */
  33. size_t bodylen;
  34. /** Pointer to the body within <b>map</b>. */
  35. const uint8_t *body;
  36. };
  37. /**
  38. * A consensus_cache_t holds a directory full of labeled items.
  39. */
  40. struct consensus_cache_t {
  41. /** Underling storage_dir_t to handle persistence */
  42. storage_dir_t *dir;
  43. /** List of all the entries in the directory. */
  44. smartlist_t *entries;
  45. };
  46. static void consensus_cache_clear(consensus_cache_t *cache);
  47. static void consensus_cache_rescan(consensus_cache_t *);
  48. static void consensus_cache_entry_map(consensus_cache_t *,
  49. consensus_cache_entry_t *);
  50. static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
  51. /**
  52. * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
  53. * data directory, to hold up to <b>max_entries</b> of data.
  54. */
  55. consensus_cache_t *
  56. consensus_cache_open(const char *subdir, int max_entries)
  57. {
  58. consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
  59. char *directory = get_datadir_fname(subdir);
  60. cache->dir = storage_dir_new(directory, max_entries);
  61. tor_free(directory);
  62. if (!cache->dir) {
  63. tor_free(cache);
  64. return NULL;
  65. }
  66. consensus_cache_rescan(cache);
  67. return cache;
  68. }
  69. /**
  70. * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  71. * operations that <b>cache</b> will need.
  72. */
  73. int
  74. consensus_cache_register_with_sandbox(consensus_cache_t *cache,
  75. struct sandbox_cfg_elem **cfg)
  76. {
  77. return storage_dir_register_with_sandbox(cache->dir, cfg);
  78. }
  79. /**
  80. * Helper: clear all entries from <b>cache</b> (but do not delete
  81. * any that aren't marked for removal
  82. */
  83. static void
  84. consensus_cache_clear(consensus_cache_t *cache)
  85. {
  86. consensus_cache_delete_pending(cache, 0);
  87. SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
  88. ent->in_cache = NULL;
  89. consensus_cache_entry_decref(ent);
  90. } SMARTLIST_FOREACH_END(ent);
  91. smartlist_free(cache->entries);
  92. cache->entries = NULL;
  93. }
  94. /**
  95. * Drop all storage held by <b>cache</b>.
  96. */
  97. void
  98. consensus_cache_free(consensus_cache_t *cache)
  99. {
  100. if (! cache)
  101. return;
  102. if (cache->entries) {
  103. consensus_cache_clear(cache);
  104. }
  105. storage_dir_free(cache->dir);
  106. tor_free(cache);
  107. }
  108. /**
  109. * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
  110. * labeling that data with <b>labels</b>. On failure, return NULL. On
  111. * success, return a newly created consensus_cache_entry_t.
  112. *
  113. * The returned value will be owned by the cache, and you will have a
  114. * reference to it. Call consensus_cache_entry_decref() when you are
  115. * done with it.
  116. *
  117. * The provided <b>labels</b> MUST have distinct keys: if they don't,
  118. * this API does not specify which values (if any) for the duplicate keys
  119. * will be considered.
  120. */
  121. consensus_cache_entry_t *
  122. consensus_cache_add(consensus_cache_t *cache,
  123. const config_line_t *labels,
  124. const uint8_t *data,
  125. size_t datalen)
  126. {
  127. char *fname = NULL;
  128. int r = storage_dir_save_labeled_to_file(cache->dir,
  129. labels, data, datalen, &fname);
  130. if (r < 0 || fname == NULL) {
  131. return NULL;
  132. }
  133. consensus_cache_entry_t *ent =
  134. tor_malloc_zero(sizeof(consensus_cache_entry_t));
  135. ent->magic = CCE_MAGIC;
  136. ent->fname = fname;
  137. ent->labels = config_lines_dup(labels);
  138. ent->in_cache = cache;
  139. ent->unused_since = TIME_MAX;
  140. smartlist_add(cache->entries, ent);
  141. /* Start the reference count at 2: the caller owns one copy, and the
  142. * cache owns another.
  143. */
  144. ent->refcnt = 2;
  145. return ent;
  146. }
  147. /**
  148. * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
  149. * Return NULL if no such entry exists.
  150. *
  151. * Does not adjust reference counts.
  152. */
  153. consensus_cache_entry_t *
  154. consensus_cache_find_first(consensus_cache_t *cache,
  155. const char *key,
  156. const char *value)
  157. {
  158. smartlist_t *tmp = smartlist_new();
  159. consensus_cache_find_all(tmp, cache, key, value);
  160. consensus_cache_entry_t *ent = NULL;
  161. if (smartlist_len(tmp))
  162. ent = smartlist_get(tmp, 0);
  163. smartlist_free(tmp);
  164. return ent;
  165. }
  166. /**
  167. * Given a <b>cache</b>, add every entry to <b>out<b> for which
  168. * <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry.
  169. *
  170. * Do not add any entry that has been marked for removal.
  171. *
  172. * Does not adjust reference counts.
  173. */
  174. void
  175. consensus_cache_find_all(smartlist_t *out,
  176. consensus_cache_t *cache,
  177. const char *key,
  178. const char *value)
  179. {
  180. SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
  181. if (ent->can_remove == 1) {
  182. /* We want to delete this; pretend it isn't there. */
  183. continue;
  184. }
  185. if (! key) {
  186. smartlist_add(out, ent);
  187. continue;
  188. }
  189. const char *found_val = consensus_cache_entry_get_value(ent, key);
  190. if (found_val && !strcmp(value, found_val)) {
  191. smartlist_add(out, ent);
  192. }
  193. } SMARTLIST_FOREACH_END(ent);
  194. }
  195. /**
  196. * Given a list of consensus_cache_entry_t, remove all those entries
  197. * that do not have <b>key</b>=<b>value</b> in their labels.
  198. *
  199. * Does not adjust reference counts.
  200. */
  201. void
  202. consensus_cache_filter_list(smartlist_t *lst,
  203. const char *key,
  204. const char *value)
  205. {
  206. if (BUG(lst == NULL))
  207. return; // LCOV_EXCL_LINE
  208. if (key == NULL)
  209. return;
  210. SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
  211. const char *found_val = consensus_cache_entry_get_value(ent, key);
  212. if (! found_val || strcmp(value, found_val)) {
  213. SMARTLIST_DEL_CURRENT(lst, ent);
  214. }
  215. } SMARTLIST_FOREACH_END(ent);
  216. }
  217. /**
  218. * If <b>ent</b> has a label with the given <b>key</b>, return its
  219. * value. Otherwise return NULL.
  220. *
  221. * The return value is only guaranteed to be valid for as long as you
  222. * hold a reference to <b>ent</b>.
  223. */
  224. const char *
  225. consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
  226. const char *key)
  227. {
  228. const config_line_t *match = config_line_find(ent->labels, key);
  229. if (match)
  230. return match->value;
  231. else
  232. return NULL;
  233. }
  234. /**
  235. * Return a pointer to the labels in <b>ent</b>.
  236. *
  237. * This pointer is only guaranteed to be valid for as long as you
  238. * hold a reference to <b>ent</b>.
  239. */
  240. const config_line_t *
  241. consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
  242. {
  243. return ent->labels;
  244. }
  245. /**
  246. * Increase the reference count of <b>ent</b>.
  247. */
  248. void
  249. consensus_cache_entry_incref(consensus_cache_entry_t *ent)
  250. {
  251. if (BUG(ent->magic != CCE_MAGIC))
  252. return; // LCOV_EXCL_LINE
  253. ++ent->refcnt;
  254. ent->unused_since = TIME_MAX;
  255. }
  256. /**
  257. * Release a reference held to <b>ent</b>.
  258. *
  259. * If it was the last reference, ent will be freed. Therefore, you must not
  260. * use <b>ent</b> after calling this function.
  261. */
  262. void
  263. consensus_cache_entry_decref(consensus_cache_entry_t *ent)
  264. {
  265. if (! ent)
  266. return;
  267. if (BUG(ent->refcnt <= 0))
  268. return; // LCOV_EXCL_LINE
  269. if (BUG(ent->magic != CCE_MAGIC))
  270. return; // LCOV_EXCL_LINE
  271. --ent->refcnt;
  272. if (ent->refcnt == 1 && ent->in_cache) {
  273. /* Only the cache has a reference: we don't need to keep the file
  274. * mapped */
  275. if (ent->map) {
  276. if (ent->release_aggressively) {
  277. consensus_cache_entry_unmap(ent);
  278. } else {
  279. ent->unused_since = approx_time();
  280. }
  281. }
  282. return;
  283. }
  284. if (ent->refcnt > 0)
  285. return;
  286. /* Refcount is zero; we can free it. */
  287. if (ent->map) {
  288. consensus_cache_entry_unmap(ent);
  289. }
  290. tor_free(ent->fname);
  291. config_free_lines(ent->labels);
  292. consensus_cache_entry_handles_clear(ent);
  293. memwipe(ent, 0, sizeof(consensus_cache_entry_t));
  294. tor_free(ent);
  295. }
  296. /**
  297. * Mark <b>ent</b> for deletion from the cache. Deletion will not occur
  298. * until the cache is the only place that holds a reference to <b>ent</b>.
  299. */
  300. void
  301. consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
  302. {
  303. ent->can_remove = 1;
  304. }
  305. /**
  306. * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
  307. * any longer than we're actually using it.
  308. */
  309. void
  310. consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
  311. {
  312. ent->release_aggressively = 1;
  313. }
  314. /**
  315. * Try to read the body of <b>ent</b> into memory if it isn't already
  316. * loaded. On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
  317. * to its size, and return 0. On failure return -1.
  318. *
  319. * The resulting body pointer will only be valid for as long as you
  320. * hold a reference to <b>ent</b>.
  321. */
  322. int
  323. consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
  324. const uint8_t **body_out,
  325. size_t *sz_out)
  326. {
  327. if (BUG(ent->magic != CCE_MAGIC))
  328. return -1; // LCOV_EXCL_LINE
  329. if (! ent->map) {
  330. if (! ent->in_cache)
  331. return -1;
  332. consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
  333. (consensus_cache_entry_t *)ent);
  334. if (! ent->map) {
  335. return -1;
  336. }
  337. }
  338. *body_out = ent->body;
  339. *sz_out = ent->bodylen;
  340. return 0;
  341. }
  342. /**
  343. * Unmap every mmap'd element of <b>cache</b> that has been unused
  344. * since <b>cutoff</b>.
  345. */
  346. void
  347. consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
  348. {
  349. SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
  350. tor_assert_nonfatal(ent->in_cache == cache);
  351. if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
  352. /* Somebody is using this entry right now */
  353. continue;
  354. }
  355. if (ent->unused_since > cutoff) {
  356. /* Has been unused only for a little while */
  357. continue;
  358. }
  359. if (ent->map == NULL) {
  360. /* Not actually mapped. */
  361. continue;
  362. }
  363. consensus_cache_entry_unmap(ent);
  364. } SMARTLIST_FOREACH_END(ent);
  365. }
  366. /**
  367. * Return the number of currently unused filenames available in this cache.
  368. */
  369. int
  370. consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
  371. {
  372. tor_assert(cache);
  373. int max = storage_dir_get_max_files(cache->dir);
  374. int used = smartlist_len(storage_dir_list(cache->dir));
  375. tor_assert_nonfatal(max >= used);
  376. return max - used;
  377. }
  378. /**
  379. * Delete every element of <b>cache</b> has been marked with
  380. * consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
  381. * retain those entries which are not in use except by the cache.
  382. */
  383. void
  384. consensus_cache_delete_pending(consensus_cache_t *cache, int force)
  385. {
  386. SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
  387. tor_assert_nonfatal(ent->in_cache == cache);
  388. if (! force) {
  389. if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
  390. /* Somebody is using this entry right now */
  391. continue;
  392. }
  393. }
  394. if (ent->can_remove == 0) {
  395. /* Don't want to delete this. */
  396. continue;
  397. }
  398. if (BUG(ent->refcnt <= 0)) {
  399. continue; // LCOV_EXCL_LINE
  400. }
  401. SMARTLIST_DEL_CURRENT(cache->entries, ent);
  402. ent->in_cache = NULL;
  403. char *fname = tor_strdup(ent->fname); /* save a copy */
  404. consensus_cache_entry_decref(ent);
  405. storage_dir_remove_file(cache->dir, fname);
  406. tor_free(fname);
  407. } SMARTLIST_FOREACH_END(ent);
  408. }
  409. /**
  410. * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
  411. */
  412. static void
  413. consensus_cache_rescan(consensus_cache_t *cache)
  414. {
  415. if (cache->entries) {
  416. consensus_cache_clear(cache);
  417. }
  418. cache->entries = smartlist_new();
  419. const smartlist_t *fnames = storage_dir_list(cache->dir);
  420. SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
  421. tor_mmap_t *map = NULL;
  422. config_line_t *labels = NULL;
  423. const uint8_t *body;
  424. size_t bodylen;
  425. map = storage_dir_map_labeled(cache->dir, fname,
  426. &labels, &body, &bodylen);
  427. if (! map) {
  428. /* Can't load this; continue */
  429. log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
  430. escaped(fname), strerror(errno));
  431. continue;
  432. }
  433. consensus_cache_entry_t *ent =
  434. tor_malloc_zero(sizeof(consensus_cache_entry_t));
  435. ent->magic = CCE_MAGIC;
  436. ent->fname = tor_strdup(fname);
  437. ent->labels = labels;
  438. ent->refcnt = 1;
  439. ent->in_cache = cache;
  440. ent->unused_since = TIME_MAX;
  441. smartlist_add(cache->entries, ent);
  442. tor_munmap_file(map); /* don't actually need to keep this around */
  443. } SMARTLIST_FOREACH_END(fname);
  444. }
  445. /**
  446. * Make sure that <b>ent</b> is mapped into RAM.
  447. */
  448. static void
  449. consensus_cache_entry_map(consensus_cache_t *cache,
  450. consensus_cache_entry_t *ent)
  451. {
  452. if (ent->map)
  453. return;
  454. ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
  455. NULL, &ent->body, &ent->bodylen);
  456. ent->unused_since = TIME_MAX;
  457. }
  458. /**
  459. * Unmap <b>ent</b> from RAM.
  460. *
  461. * Do not call this if something other than the cache is holding a reference
  462. * to <b>ent</b>
  463. */
  464. static void
  465. consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
  466. {
  467. ent->unused_since = TIME_MAX;
  468. if (!ent->map)
  469. return;
  470. tor_munmap_file(ent->map);
  471. ent->map = NULL;
  472. ent->body = NULL;
  473. ent->bodylen = 0;
  474. ent->unused_since = TIME_MAX;
  475. }
  476. HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
  477. #ifdef TOR_UNIT_TESTS
  478. /**
  479. * Testing only: Return true iff <b>ent</b> is mapped into memory.
  480. *
  481. * (In normal operation, this information is not exposed.)
  482. */
  483. int
  484. consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
  485. {
  486. if (ent->map) {
  487. tor_assert(ent->body);
  488. return 1;
  489. } else {
  490. tor_assert(!ent->body);
  491. return 0;
  492. }
  493. }
  494. #endif