storagedir.c 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606
  1. /* Copyright (c) 2017-2018, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file storagedir.c
  5. *
  6. * \brief An abstraction for a directory full of similar files.
  7. *
  8. * Storagedirs are used by our consensus cache code, and may someday also get
  9. * used for unparseable objects. A large part of the need for this type is to
  10. * work around the limitations in our sandbox code, where all filenames need
  11. * to be registered in advance.
  12. **/
  13. #include "lib/fs/storagedir.h"
  14. #include "lib/container/smartlist.h"
  15. #include "lib/encoding/confline.h"
  16. #include "lib/fs/dir.h"
  17. #include "lib/fs/files.h"
  18. #include "lib/fs/mmap.h"
  19. #include "lib/log/escape.h"
  20. #include "lib/log/torlog.h"
  21. #include "lib/log/util_bug.h"
  22. #include "lib/malloc/util_malloc.h"
  23. #include "lib/memarea/memarea.h"
  24. #include "lib/sandbox/sandbox.h"
  25. #include "lib/string/printf.h"
  26. #include "lib/string/util_string.h"
  27. #ifdef HAVE_SYS_TYPES_H
  28. #include <sys/types.h>
  29. #endif
  30. #ifdef HAVE_SYS_STAT_H
  31. #include <sys/stat.h>
  32. #endif
  33. #ifdef HAVE_UNISTD_H
  34. #include <unistd.h>
  35. #endif
  36. #include <stdlib.h>
  37. #include <errno.h>
  38. #include <string.h>
  39. #define FNAME_MIN_NUM 1000
  40. /** A storage_dir_t represents a directory full of similar cached
  41. * files. Filenames are decimal integers. Files can be cleaned as needed
  42. * to limit total disk usage. */
  43. struct storage_dir_t {
  44. /** Directory holding the files for this storagedir. */
  45. char *directory;
  46. /** Either NULL, or a directory listing of the directory (as a smartlist
  47. * of strings */
  48. smartlist_t *contents;
  49. /** The largest number of non-temporary files we'll place in the
  50. * directory. */
  51. int max_files;
  52. /** If true, then 'usage' has been computed. */
  53. int usage_known;
  54. /** The total number of bytes used in this directory */
  55. uint64_t usage;
  56. };
  57. /** Create or open a new storage directory at <b>dirname</b>, with
  58. * capacity for up to <b>max_files</b> files.
  59. */
  60. storage_dir_t *
  61. storage_dir_new(const char *dirname, int max_files)
  62. {
  63. if (check_private_dir(dirname, CPD_CREATE, NULL) < 0)
  64. return NULL;
  65. storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t));
  66. d->directory = tor_strdup(dirname);
  67. d->max_files = max_files;
  68. return d;
  69. }
  70. /**
  71. * Drop all in-RAM storage for <b>d</b>. Does not delete any files.
  72. */
  73. void
  74. storage_dir_free_(storage_dir_t *d)
  75. {
  76. if (d == NULL)
  77. return;
  78. tor_free(d->directory);
  79. if (d->contents) {
  80. SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
  81. smartlist_free(d->contents);
  82. }
  83. tor_free(d);
  84. }
  85. /**
  86. * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  87. * operations that <b>d</b> will need.
  88. *
  89. * The presence of this function is why we need an upper limit on the
  90. * number of files in a storage_dir_t: we need to approve file operations
  91. * one by one.
  92. */
  93. int
  94. storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
  95. {
  96. int problems = 0;
  97. int idx;
  98. for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) {
  99. char *path = NULL, *tmppath = NULL;
  100. tor_asprintf(&path, "%s/%d", d->directory, idx);
  101. tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx);
  102. problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path));
  103. problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath));
  104. problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path));
  105. problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath));
  106. problems += sandbox_cfg_allow_rename(cfg,
  107. tor_strdup(tmppath), tor_strdup(path));
  108. tor_free(path);
  109. tor_free(tmppath);
  110. }
  111. return problems ? -1 : 0;
  112. }
  113. /**
  114. * Remove all files in <b>d</b> whose names end with ".tmp".
  115. *
  116. * Requires that the contents field of <b>d</b> is set.
  117. */
  118. static void
  119. storage_dir_clean_tmpfiles(storage_dir_t *d)
  120. {
  121. if (!d->contents)
  122. return;
  123. SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) {
  124. if (strcmpend(fname, ".tmp"))
  125. continue;
  126. char *path = NULL;
  127. tor_asprintf(&path, "%s/%s", d->directory, fname);
  128. if (unlink(sandbox_intern_string(path))) {
  129. log_warn(LD_FS, "Unable to unlink %s while cleaning "
  130. "temporary files: %s", escaped(path), strerror(errno));
  131. tor_free(path);
  132. continue;
  133. }
  134. tor_free(path);
  135. SMARTLIST_DEL_CURRENT(d->contents, fname);
  136. tor_free(fname);
  137. } SMARTLIST_FOREACH_END(fname);
  138. d->usage_known = 0;
  139. }
  140. /**
  141. * Re-scan the directory <b>d</b> to learn its contents.
  142. */
  143. static int
  144. storage_dir_rescan(storage_dir_t *d)
  145. {
  146. if (d->contents) {
  147. SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
  148. smartlist_free(d->contents);
  149. }
  150. d->usage = 0;
  151. d->usage_known = 0;
  152. if (NULL == (d->contents = tor_listdir(d->directory))) {
  153. return -1;
  154. }
  155. storage_dir_clean_tmpfiles(d);
  156. return 0;
  157. }
  158. /**
  159. * Return a smartlist containing the filenames within <b>d</b>.
  160. */
  161. const smartlist_t *
  162. storage_dir_list(storage_dir_t *d)
  163. {
  164. if (! d->contents)
  165. storage_dir_rescan(d);
  166. return d->contents;
  167. }
  168. /**
  169. * Return the total number of bytes used for storage in <b>d</b>.
  170. */
  171. uint64_t
  172. storage_dir_get_usage(storage_dir_t *d)
  173. {
  174. if (d->usage_known)
  175. return d->usage;
  176. uint64_t total = 0;
  177. SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) {
  178. char *path = NULL;
  179. struct stat st;
  180. tor_asprintf(&path, "%s/%s", d->directory, cp);
  181. if (stat(sandbox_intern_string(path), &st) == 0) {
  182. total += st.st_size;
  183. }
  184. tor_free(path);
  185. } SMARTLIST_FOREACH_END(cp);
  186. d->usage = total;
  187. d->usage_known = 1;
  188. return d->usage;
  189. }
  190. /** Mmap a specified file within <b>d</b>.
  191. *
  192. * On failure, return NULL and set errno as for tor_mmap_file(). */
  193. tor_mmap_t *
  194. storage_dir_map(storage_dir_t *d, const char *fname)
  195. {
  196. char *path = NULL;
  197. tor_asprintf(&path, "%s/%s", d->directory, fname);
  198. tor_mmap_t *result = tor_mmap_file(path);
  199. int errval = errno;
  200. tor_free(path);
  201. if (result == NULL)
  202. errno = errval;
  203. return result;
  204. }
  205. /** Read a file within <b>d</b> into a newly allocated buffer. Set
  206. * *<b>sz_out</b> to its size. */
  207. uint8_t *
  208. storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
  209. {
  210. const int flags = bin ? RFTS_BIN : 0;
  211. char *path = NULL;
  212. tor_asprintf(&path, "%s/%s", d->directory, fname);
  213. struct stat st;
  214. char *contents = read_file_to_str(path, flags, &st);
  215. if (contents && sz_out) {
  216. // it fits in RAM, so we know its size is less than SIZE_MAX
  217. #if UINT64_MAX > SIZE_MAX
  218. tor_assert((uint64_t)st.st_size <= SIZE_MAX);
  219. #endif
  220. *sz_out = (size_t) st.st_size;
  221. }
  222. tor_free(path);
  223. return (uint8_t *) contents;
  224. }
  225. /** Helper: Find an unused filename within the directory */
  226. static char *
  227. find_unused_fname(storage_dir_t *d)
  228. {
  229. if (!d->contents) {
  230. if (storage_dir_rescan(d) < 0)
  231. return NULL;
  232. }
  233. char buf[16];
  234. int i;
  235. /* Yuck; this is quadratic. Fortunately, that shouldn't matter much,
  236. * since disk writes are more expensive by a lot. */
  237. for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) {
  238. tor_snprintf(buf, sizeof(buf), "%d", i);
  239. if (!smartlist_contains_string(d->contents, buf)) {
  240. return tor_strdup(buf);
  241. }
  242. }
  243. return NULL;
  244. }
  245. /** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
  246. * sized_chunk_t rather than a single byte array. */
  247. static int
  248. storage_dir_save_chunks_to_file(storage_dir_t *d,
  249. const smartlist_t *chunks,
  250. int binary,
  251. char **fname_out)
  252. {
  253. uint64_t total_length = 0;
  254. char *fname = find_unused_fname(d);
  255. if (!fname)
  256. return -1;
  257. SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
  258. total_length += ch->len);
  259. char *path = NULL;
  260. tor_asprintf(&path, "%s/%s", d->directory, fname);
  261. int r = write_chunks_to_file(path, chunks, binary, 0);
  262. if (r == 0) {
  263. if (d->usage_known)
  264. d->usage += total_length;
  265. if (fname_out) {
  266. *fname_out = tor_strdup(fname);
  267. }
  268. if (d->contents)
  269. smartlist_add(d->contents, tor_strdup(fname));
  270. }
  271. tor_free(fname);
  272. tor_free(path);
  273. return r;
  274. }
  275. /** Try to write the <b>length</b> bytes at <b>data</b> into a new file
  276. * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a
  277. * newly allocated string containing the filename. On failure, return
  278. * -1. */
  279. int
  280. storage_dir_save_bytes_to_file(storage_dir_t *d,
  281. const uint8_t *data,
  282. size_t length,
  283. int binary,
  284. char **fname_out)
  285. {
  286. smartlist_t *chunks = smartlist_new();
  287. sized_chunk_t chunk = { (const char *)data, length };
  288. smartlist_add(chunks, &chunk);
  289. int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
  290. smartlist_free(chunks);
  291. return r;
  292. }
  293. /**
  294. * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
  295. * <b>str</b>.
  296. */
  297. int
  298. storage_dir_save_string_to_file(storage_dir_t *d,
  299. const char *str,
  300. int binary,
  301. char **fname_out)
  302. {
  303. return storage_dir_save_bytes_to_file(d,
  304. (const uint8_t*)str, strlen(str), binary, fname_out);
  305. }
  306. /**
  307. * As storage_dir_save_bytes_to_file, but associates the data with the
  308. * key-value pairs in <b>labels</b>. Files stored in this format can be
  309. * recovered with storage_dir_map_labeled() or storage_dir_read_labeled().
  310. */
  311. int
  312. storage_dir_save_labeled_to_file(storage_dir_t *d,
  313. const config_line_t *labels,
  314. const uint8_t *data,
  315. size_t length,
  316. char **fname_out)
  317. {
  318. /*
  319. * The storage format is to prefix the data with the key-value pairs in
  320. * <b>labels</b>, and a single NUL separator. But code outside this module
  321. * MUST NOT rely on that format.
  322. */
  323. smartlist_t *chunks = smartlist_new();
  324. memarea_t *area = memarea_new();
  325. const config_line_t *line;
  326. for (line = labels; line; line = line->next) {
  327. sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
  328. sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
  329. const size_t allocated = sz->len + 1;
  330. char *bytes = memarea_alloc(area, allocated);
  331. tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
  332. sz->bytes = bytes;
  333. smartlist_add(chunks, sz);
  334. }
  335. sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
  336. nul->len = 1;
  337. nul->bytes = "\0";
  338. smartlist_add(chunks, nul);
  339. sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
  340. datachunk->bytes = (const char *)data;
  341. datachunk->len = length;
  342. smartlist_add(chunks, datachunk);
  343. int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
  344. smartlist_free(chunks);
  345. memarea_drop_all(area);
  346. return r;
  347. }
  348. /**
  349. * Map a file that was created with storage_dir_save_labeled_to_file(). On
  350. * failure, return NULL. On success, write a set of newly allocated labels
  351. * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and
  352. * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t
  353. * object whose contents should not be used -- it needs to be kept around,
  354. * though, for as long as <b>data_out</b> is going to be valid.
  355. *
  356. * On failure, set errno as for tor_mmap_file() if the file was missing or
  357. * empty, and set errno to EINVAL if the file was not in the labeled
  358. * format expected.
  359. */
  360. tor_mmap_t *
  361. storage_dir_map_labeled(storage_dir_t *dir,
  362. const char *fname,
  363. config_line_t **labels_out,
  364. const uint8_t **data_out,
  365. size_t *sz_out)
  366. {
  367. tor_mmap_t *m = storage_dir_map(dir, fname);
  368. int errval;
  369. if (! m) {
  370. errval = errno;
  371. goto err;
  372. }
  373. const char *nulp = memchr(m->data, '\0', m->size);
  374. if (! nulp) {
  375. errval = EINVAL;
  376. goto err;
  377. }
  378. if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) {
  379. errval = EINVAL;
  380. goto err;
  381. }
  382. size_t offset = nulp - m->data + 1;
  383. tor_assert(offset <= m->size);
  384. *data_out = (const uint8_t *)(m->data + offset);
  385. *sz_out = m->size - offset;
  386. return m;
  387. err:
  388. tor_munmap_file(m);
  389. errno = errval;
  390. return NULL;
  391. }
  392. /** As storage_dir_map_labeled, but return a new byte array containing the
  393. * data. */
  394. uint8_t *
  395. storage_dir_read_labeled(storage_dir_t *dir,
  396. const char *fname,
  397. config_line_t **labels_out,
  398. size_t *sz_out)
  399. {
  400. const uint8_t *data = NULL;
  401. tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
  402. &data, sz_out);
  403. if (m == NULL)
  404. return NULL;
  405. uint8_t *result = tor_memdup(data, *sz_out);
  406. tor_munmap_file(m);
  407. return result;
  408. }
  409. /* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>.
  410. * This function is a no-op if <b>d->usage_known</b> is 0. */
  411. static void
  412. storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size)
  413. {
  414. if (d->usage_known) {
  415. if (! BUG(d->usage < removed_file_size)) {
  416. /* This bug can also be triggered if an external process resized a file
  417. * between the call to storage_dir_get_usage() that last checked
  418. * actual usage (rather than relaying on cached usage), and the call to
  419. * this function. */
  420. d->usage -= removed_file_size;
  421. } else {
  422. /* If we underflowed the cached directory size, re-check the sizes of all
  423. * the files in the directory. This makes storage_dir_shrink() quadratic,
  424. * but only if a process is continually changing file sizes in the
  425. * storage directory (in which case, we have bigger issues).
  426. *
  427. * We can't just reset usage_known, because storage_dir_shrink() relies
  428. * on knowing the usage. */
  429. storage_dir_rescan(d);
  430. (void)storage_dir_get_usage(d);
  431. }
  432. }
  433. }
  434. /**
  435. * Remove the file called <b>fname</b> from <b>d</b>.
  436. */
  437. void
  438. storage_dir_remove_file(storage_dir_t *d,
  439. const char *fname)
  440. {
  441. char *path = NULL;
  442. tor_asprintf(&path, "%s/%s", d->directory, fname);
  443. const char *ipath = sandbox_intern_string(path);
  444. uint64_t size = 0;
  445. if (d->usage_known) {
  446. struct stat st;
  447. if (stat(ipath, &st) == 0) {
  448. size = st.st_size;
  449. }
  450. }
  451. if (unlink(ipath) == 0) {
  452. storage_dir_reduce_usage(d, size);
  453. } else {
  454. log_warn(LD_FS, "Unable to unlink %s while removing file: %s",
  455. escaped(path), strerror(errno));
  456. tor_free(path);
  457. return;
  458. }
  459. if (d->contents) {
  460. smartlist_string_remove(d->contents, fname);
  461. }
  462. tor_free(path);
  463. }
  464. /** Helper type: used to sort the members of storage directory by mtime. */
  465. typedef struct shrinking_dir_entry_t {
  466. time_t mtime;
  467. uint64_t size;
  468. char *path;
  469. } shrinking_dir_entry_t;
  470. /** Helper: use with qsort to sort shrinking_dir_entry_t structs. */
  471. static int
  472. shrinking_dir_entry_compare(const void *a_, const void *b_)
  473. {
  474. const shrinking_dir_entry_t *a = a_;
  475. const shrinking_dir_entry_t *b = b_;
  476. if (a->mtime < b->mtime)
  477. return -1;
  478. else if (a->mtime > b->mtime)
  479. return 1;
  480. else
  481. return 0;
  482. }
  483. /**
  484. * Try to free space by removing the oldest files in <b>d</b>. Delete
  485. * until no more than <b>target_size</b> bytes are left, and at least
  486. * <b>min_to_remove</b> files have been removed... or until there is
  487. * nothing left to remove.
  488. *
  489. * Return 0 on success; -1 on failure.
  490. */
  491. int
  492. storage_dir_shrink(storage_dir_t *d,
  493. uint64_t target_size,
  494. int min_to_remove)
  495. {
  496. if (d->usage_known && d->usage <= target_size && !min_to_remove) {
  497. /* Already small enough. */
  498. return 0;
  499. }
  500. if (storage_dir_rescan(d) < 0)
  501. return -1;
  502. const uint64_t orig_usage = storage_dir_get_usage(d);
  503. if (orig_usage <= target_size && !min_to_remove) {
  504. /* Okay, small enough after rescan! */
  505. return 0;
  506. }
  507. const int n = smartlist_len(d->contents);
  508. shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t));
  509. SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) {
  510. shrinking_dir_entry_t *ent = &ents[fname_sl_idx];
  511. struct stat st;
  512. tor_asprintf(&ent->path, "%s/%s", d->directory, fname);
  513. if (stat(sandbox_intern_string(ent->path), &st) == 0) {
  514. ent->mtime = st.st_mtime;
  515. ent->size = st.st_size;
  516. }
  517. } SMARTLIST_FOREACH_END(fname);
  518. qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare);
  519. int idx = 0;
  520. while ((d->usage > target_size || min_to_remove > 0) && idx < n) {
  521. if (unlink(sandbox_intern_string(ents[idx].path)) == 0) {
  522. storage_dir_reduce_usage(d, ents[idx].size);
  523. --min_to_remove;
  524. }
  525. ++idx;
  526. }
  527. for (idx = 0; idx < n; ++idx) {
  528. tor_free(ents[idx].path);
  529. }
  530. tor_free(ents);
  531. storage_dir_rescan(d);
  532. return 0;
  533. }
  534. /** Remove all files in <b>d</b>. */
  535. int
  536. storage_dir_remove_all(storage_dir_t *d)
  537. {
  538. return storage_dir_shrink(d, 0, d->max_files);
  539. }
  540. /**
  541. * Return the largest number of non-temporary files we're willing to
  542. * store in <b>d</b>.
  543. */
  544. int
  545. storage_dir_get_max_files(storage_dir_t *d)
  546. {
  547. return d->max_files;
  548. }