consdiffmgr.c 63 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911
  1. /* Copyright (c) 2017, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file consdiffmsr.c
  5. *
  6. * \brief consensus diff manager functions
  7. *
  8. * This module is run by directory authorities and caches in order
  9. * to remember a number of past consensus documents, and to generate
  10. * and serve the diffs from those documents to the latest consensus.
  11. */
  12. #define CONSDIFFMGR_PRIVATE
  13. #include "or.h"
  14. #include "config.h"
  15. #include "conscache.h"
  16. #include "consdiff.h"
  17. #include "consdiffmgr.h"
  18. #include "cpuworker.h"
  19. #include "networkstatus.h"
  20. #include "routerparse.h"
  21. #include "workqueue.h"
  22. /**
  23. * Labels to apply to items in the conscache object.
  24. *
  25. * @{
  26. */
  27. /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
  28. #define LABEL_DOCTYPE "document-type"
  29. /* The valid-after time for a consensus (or for the target consensus of a
  30. * diff), encoded as ISO UTC. */
  31. #define LABEL_VALID_AFTER "consensus-valid-after"
  32. /* The fresh-until time for a consensus (or for the target consensus of a
  33. * diff), encoded as ISO UTC. */
  34. #define LABEL_FRESH_UNTIL "consensus-fresh-until"
  35. /* The valid-until time for a consensus (or for the target consensus of a
  36. * diff), encoded as ISO UTC. */
  37. #define LABEL_VALID_UNTIL "consensus-valid-until"
  38. /* Comma-separated list of hex-encoded identity digests for the voting
  39. * authorities. */
  40. #define LABEL_SIGNATORIES "consensus-signatories"
  41. /* A hex encoded SHA3 digest of the object, as compressed (if any) */
  42. #define LABEL_SHA3_DIGEST "sha3-digest"
  43. /* A hex encoded SHA3 digest of the object before compression. */
  44. #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
  45. /* A hex encoded SHA3 digest-as-signed of a consensus */
  46. #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
  47. /* The flavor of the consensus or consensuses diff */
  48. #define LABEL_FLAVOR "consensus-flavor"
  49. /* Diff only: the SHA3 digest-as-signed of the source consensus. */
  50. #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
  51. /* Diff only: the SHA3 digest-in-full of the target consensus. */
  52. #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
  53. /* Diff only: the valid-after date of the source consensus. */
  54. #define LABEL_FROM_VALID_AFTER "from-valid-after"
  55. /* What kind of compression was used? */
  56. #define LABEL_COMPRESSION_TYPE "compression"
  57. /** @} */
  58. #define DOCTYPE_CONSENSUS "consensus"
  59. #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
  60. /**
  61. * Underlying directory that stores consensuses and consensus diffs. Don't
  62. * use this directly: use cdm_cache_get() instead.
  63. */
  64. static consensus_cache_t *cons_diff_cache = NULL;
  65. /**
  66. * If true, we have learned at least one new consensus since the
  67. * consensus cache was last up-to-date.
  68. */
  69. static int cdm_cache_dirty = 0;
  70. /**
  71. * If true, we have scanned the cache to update our hashtable of diffs.
  72. */
  73. static int cdm_cache_loaded = 0;
  74. /**
  75. * Possible status values for cdm_diff_t.cdm_diff_status
  76. **/
  77. typedef enum cdm_diff_status_t {
  78. CDM_DIFF_PRESENT=1,
  79. CDM_DIFF_IN_PROGRESS=2,
  80. CDM_DIFF_ERROR=3,
  81. } cdm_diff_status_t;
  82. /** Which methods do we use for precompressing diffs? */
  83. static const compress_method_t compress_diffs_with[] = {
  84. NO_METHOD,
  85. GZIP_METHOD,
  86. #ifdef HAVE_LZMA
  87. LZMA_METHOD,
  88. #endif
  89. #ifdef HAVE_ZSTD
  90. ZSTD_METHOD,
  91. #endif
  92. };
  93. /** How many different methods will we try to use for diff compression? */
  94. STATIC unsigned
  95. n_diff_compression_methods(void)
  96. {
  97. return ARRAY_LENGTH(compress_diffs_with);
  98. }
  99. /** Which methods do we use for precompressing consensuses? */
  100. static const compress_method_t compress_consensus_with[] = {
  101. ZLIB_METHOD,
  102. #ifdef HAVE_LZMA
  103. LZMA_METHOD,
  104. #endif
  105. #ifdef HAVE_ZSTD
  106. ZSTD_METHOD,
  107. #endif
  108. };
  109. /** How many different methods will we try to use for diff compression? */
  110. STATIC unsigned
  111. n_consensus_compression_methods(void)
  112. {
  113. return ARRAY_LENGTH(compress_consensus_with);
  114. }
  115. /** For which compression method do we retain old consensuses? There's no
  116. * need to keep all of them, since we won't be serving them. We'll
  117. * go with ZLIB_METHOD because it's pretty fast and everyone has it.
  118. */
  119. #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
  120. /** Handles pointing to the latest consensus entries as compressed and
  121. * stored. */
  122. static consensus_cache_entry_handle_t *
  123. latest_consensus[N_CONSENSUS_FLAVORS]
  124. [ARRAY_LENGTH(compress_consensus_with)];
  125. /** Hashtable node used to remember the current status of the diff
  126. * from a given sha3 digest to the current consensus. */
  127. typedef struct cdm_diff_t {
  128. HT_ENTRY(cdm_diff_t) node;
  129. /** Consensus flavor for this diff (part of ht key) */
  130. consensus_flavor_t flavor;
  131. /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
  132. * ht key) */
  133. uint8_t from_sha3[DIGEST256_LEN];
  134. /** Method by which the diff is compressed. (part of the ht key */
  135. compress_method_t compress_method;
  136. /** One of the CDM_DIFF_* values, depending on whether this diff
  137. * is available, in progress, or impossible to compute. */
  138. cdm_diff_status_t cdm_diff_status;
  139. /** SHA3-256 digest of the consensus that this diff is _to. */
  140. uint8_t target_sha3[DIGEST256_LEN];
  141. /** Handle to the cache entry for this diff, if any. We use a handle here
  142. * to avoid thinking too hard about cache entry lifetime issues. */
  143. consensus_cache_entry_handle_t *entry;
  144. } cdm_diff_t;
  145. /** Hashtable mapping flavor and source consensus digest to status. */
  146. static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
  147. /**
  148. * Configuration for this module
  149. */
  150. static consdiff_cfg_t consdiff_cfg = {
  151. // XXXX I'd like to make this number bigger, but it interferes with the
  152. // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
  153. // XXXX rules.
  154. /* .cache_max_num = */ 128
  155. };
  156. static int consdiffmgr_ensure_space_for_files(int n);
  157. static int consensus_queue_compression_work(const char *consensus,
  158. const networkstatus_t *as_parsed);
  159. static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
  160. consensus_cache_entry_t *diff_to);
  161. static void consdiffmgr_set_cache_flags(void);
  162. /* =====
  163. * Hashtable setup
  164. * ===== */
  165. /** Helper: hash the key of a cdm_diff_t. */
  166. static unsigned
  167. cdm_diff_hash(const cdm_diff_t *diff)
  168. {
  169. uint8_t tmp[DIGEST256_LEN + 2];
  170. memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
  171. tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
  172. tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
  173. return (unsigned) siphash24g(tmp, sizeof(tmp));
  174. }
  175. /** Helper: compare two cdm_diff_t objects for key equality */
  176. static int
  177. cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
  178. {
  179. return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
  180. diff1->flavor == diff2->flavor &&
  181. diff1->compress_method == diff2->compress_method;
  182. }
  183. HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
  184. HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
  185. 0.6, tor_reallocarray, tor_free_)
  186. #define cdm_diff_free(diff) \
  187. FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
  188. /** Release all storage held in <b>diff</b>. */
  189. static void
  190. cdm_diff_free_(cdm_diff_t *diff)
  191. {
  192. if (!diff)
  193. return;
  194. consensus_cache_entry_handle_free(diff->entry);
  195. tor_free(diff);
  196. }
  197. /** Create and return a new cdm_diff_t with the given values. Does not
  198. * add it to the hashtable. */
  199. static cdm_diff_t *
  200. cdm_diff_new(consensus_flavor_t flav,
  201. const uint8_t *from_sha3,
  202. const uint8_t *target_sha3,
  203. compress_method_t method)
  204. {
  205. cdm_diff_t *ent;
  206. ent = tor_malloc_zero(sizeof(cdm_diff_t));
  207. ent->flavor = flav;
  208. memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
  209. memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
  210. ent->compress_method = method;
  211. return ent;
  212. }
  213. /**
  214. * Examine the diff hashtable to see whether we know anything about computing
  215. * a diff of type <b>flav</b> between consensuses with the two provided
  216. * SHA3-256 digests. If a computation is in progress, or if the computation
  217. * has already been tried and failed, return 1. Otherwise, note the
  218. * computation as "in progress" so that we don't reattempt it later, and
  219. * return 0.
  220. */
  221. static int
  222. cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
  223. const uint8_t *from_sha3,
  224. const uint8_t *target_sha3)
  225. {
  226. struct cdm_diff_t search, *ent;
  227. unsigned u;
  228. int result = 0;
  229. for (u = 0; u < n_diff_compression_methods(); ++u) {
  230. compress_method_t method = compress_diffs_with[u];
  231. memset(&search, 0, sizeof(cdm_diff_t));
  232. search.flavor = flav;
  233. search.compress_method = method;
  234. memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
  235. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  236. if (ent) {
  237. tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
  238. result = 1;
  239. continue;
  240. }
  241. ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
  242. ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
  243. HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
  244. }
  245. return result;
  246. }
  247. /**
  248. * Update the status of the diff of type <b>flav</b> between consensuses with
  249. * the two provided SHA3-256 digests, so that its status becomes
  250. * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
  251. * is NULL, then the old handle (if any) is freed, and replaced with NULL.
  252. */
  253. static void
  254. cdm_diff_ht_set_status(consensus_flavor_t flav,
  255. const uint8_t *from_sha3,
  256. const uint8_t *to_sha3,
  257. compress_method_t method,
  258. int status,
  259. consensus_cache_entry_handle_t *handle)
  260. {
  261. if (handle == NULL) {
  262. tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
  263. }
  264. struct cdm_diff_t search, *ent;
  265. memset(&search, 0, sizeof(cdm_diff_t));
  266. search.flavor = flav;
  267. search.compress_method = method,
  268. memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
  269. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  270. if (!ent) {
  271. ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
  272. ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
  273. HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
  274. } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
  275. // This can happen under certain really pathological conditions
  276. // if we decide we don't care about a diff before it is actually
  277. // done computing.
  278. return;
  279. }
  280. tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
  281. ent->cdm_diff_status = status;
  282. consensus_cache_entry_handle_free(ent->entry);
  283. ent->entry = handle;
  284. }
  285. /**
  286. * Helper: Remove from the hash table every present (actually computed) diff
  287. * of type <b>flav</b> whose target digest does not match
  288. * <b>unless_target_sha3_matches</b>.
  289. *
  290. * This function is used for the hash table to throw away references to diffs
  291. * that do not lead to the most given consensus of a given flavor.
  292. */
  293. static void
  294. cdm_diff_ht_purge(consensus_flavor_t flav,
  295. const uint8_t *unless_target_sha3_matches)
  296. {
  297. cdm_diff_t **diff, **next;
  298. for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
  299. cdm_diff_t *this = *diff;
  300. if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
  301. flav == (*diff)->flavor) {
  302. if (BUG((*diff)->entry == NULL) ||
  303. consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
  304. /* the underlying entry has gone away; drop this. */
  305. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  306. cdm_diff_free(this);
  307. continue;
  308. }
  309. if (unless_target_sha3_matches &&
  310. fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
  311. DIGEST256_LEN)) {
  312. /* target hash doesn't match; drop this. */
  313. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  314. cdm_diff_free(this);
  315. continue;
  316. }
  317. }
  318. next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
  319. }
  320. }
  321. /**
  322. * Helper: initialize <b>cons_diff_cache</b>.
  323. */
  324. static void
  325. cdm_cache_init(void)
  326. {
  327. unsigned n_entries = consdiff_cfg.cache_max_num * 2;
  328. tor_assert(cons_diff_cache == NULL);
  329. cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
  330. if (cons_diff_cache == NULL) {
  331. // LCOV_EXCL_START
  332. log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
  333. tor_assert_unreached();
  334. // LCOV_EXCL_STOP
  335. } else {
  336. consdiffmgr_set_cache_flags();
  337. }
  338. cdm_cache_dirty = 1;
  339. cdm_cache_loaded = 0;
  340. }
  341. /**
  342. * Helper: return the consensus_cache_t * that backs this manager,
  343. * initializing it if needed.
  344. */
  345. STATIC consensus_cache_t *
  346. cdm_cache_get(void)
  347. {
  348. if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
  349. cdm_cache_init();
  350. }
  351. return cons_diff_cache;
  352. }
  353. /**
  354. * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
  355. * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
  356. * with <b>label</b> as its label.
  357. */
  358. static void
  359. cdm_labels_prepend_sha3(config_line_t **labels,
  360. const char *label,
  361. const uint8_t *body,
  362. size_t bodylen)
  363. {
  364. uint8_t sha3_digest[DIGEST256_LEN];
  365. char hexdigest[HEX_DIGEST256_LEN+1];
  366. crypto_digest256((char *)sha3_digest,
  367. (const char *)body, bodylen, DIGEST_SHA3_256);
  368. base16_encode(hexdigest, sizeof(hexdigest),
  369. (const char *)sha3_digest, sizeof(sha3_digest));
  370. config_line_prepend(labels, label, hexdigest);
  371. }
  372. /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
  373. * given label, set <b>digest_out</b> to that value (decoded), and return 0.
  374. *
  375. * Return -1 if there is no such label, and -2 if it is badly formatted. */
  376. STATIC int
  377. cdm_entry_get_sha3_value(uint8_t *digest_out,
  378. consensus_cache_entry_t *ent,
  379. const char *label)
  380. {
  381. if (ent == NULL)
  382. return -1;
  383. const char *hex = consensus_cache_entry_get_value(ent, label);
  384. if (hex == NULL)
  385. return -1;
  386. int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
  387. if (n != DIGEST256_LEN)
  388. return -2;
  389. else
  390. return 0;
  391. }
  392. /**
  393. * Helper: look for a consensus with the given <b>flavor</b> and
  394. * <b>valid_after</b> time in the cache. Return that consensus if it's
  395. * present, or NULL if it's missing.
  396. */
  397. STATIC consensus_cache_entry_t *
  398. cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
  399. {
  400. char formatted_time[ISO_TIME_LEN+1];
  401. format_iso_time_nospace(formatted_time, valid_after);
  402. const char *flavname = networkstatus_get_flavor_name(flavor);
  403. /* We'll filter by valid-after time first, since that should
  404. * match the fewest documents. */
  405. /* We could add an extra hashtable here, but since we only do this scan
  406. * when adding a new consensus, it probably doesn't matter much. */
  407. smartlist_t *matches = smartlist_new();
  408. consensus_cache_find_all(matches, cdm_cache_get(),
  409. LABEL_VALID_AFTER, formatted_time);
  410. consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
  411. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  412. consensus_cache_entry_t *result = NULL;
  413. if (smartlist_len(matches)) {
  414. result = smartlist_get(matches, 0);
  415. }
  416. smartlist_free(matches);
  417. return result;
  418. }
  419. /** Return the maximum age (in seconds) of consensuses that we should consider
  420. * storing. The available space in the directory may impose additional limits
  421. * on how much we store. */
  422. static int32_t
  423. get_max_age_to_cache(void)
  424. {
  425. const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
  426. const int32_t MIN_MAX_AGE_TO_CACHE = 0;
  427. const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
  428. const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
  429. const or_options_t *options = get_options();
  430. if (options->MaxConsensusAgeForDiffs) {
  431. const int v = options->MaxConsensusAgeForDiffs;
  432. if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
  433. return MAX_MAX_AGE_TO_CACHE;
  434. else
  435. return v;
  436. }
  437. /* The parameter is in hours, so we multiply */
  438. return 3600 * networkstatus_get_param(NULL,
  439. MAX_AGE_TO_CACHE_NAME,
  440. DEFAULT_MAX_AGE_TO_CACHE,
  441. MIN_MAX_AGE_TO_CACHE,
  442. MAX_MAX_AGE_TO_CACHE);
  443. }
  444. /**
  445. * Given a string containing a networkstatus consensus, and the results of
  446. * having parsed that consensus, add that consensus to the cache if it is not
  447. * already present and not too old. Create new consensus diffs from or to
  448. * that consensus as appropriate.
  449. *
  450. * Return 0 on success and -1 on failure.
  451. */
  452. int
  453. consdiffmgr_add_consensus(const char *consensus,
  454. const networkstatus_t *as_parsed)
  455. {
  456. if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
  457. return -1; // LCOV_EXCL_LINE
  458. if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
  459. return -1; // LCOV_EXCL_LINE
  460. const consensus_flavor_t flavor = as_parsed->flavor;
  461. const time_t valid_after = as_parsed->valid_after;
  462. if (valid_after < approx_time() - get_max_age_to_cache()) {
  463. log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
  464. "too old.");
  465. return -1;
  466. }
  467. /* Do we already have this one? */
  468. consensus_cache_entry_t *entry =
  469. cdm_cache_lookup_consensus(flavor, valid_after);
  470. if (entry) {
  471. log_info(LD_DIRSERV, "We already have a copy of that consensus");
  472. return -1;
  473. }
  474. /* We don't have it. Add it to the cache. */
  475. return consensus_queue_compression_work(consensus, as_parsed);
  476. }
  477. /**
  478. * Helper: used to sort two smartlists of consensus_cache_entry_t by their
  479. * LABEL_VALID_AFTER labels.
  480. */
  481. static int
  482. compare_by_valid_after_(const void **a, const void **b)
  483. {
  484. const consensus_cache_entry_t *e1 = *a;
  485. const consensus_cache_entry_t *e2 = *b;
  486. /* We're in luck here: sorting UTC iso-encoded values lexically will work
  487. * fine (until 9999). */
  488. return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
  489. consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
  490. }
  491. /**
  492. * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
  493. * entry.
  494. */
  495. static consensus_cache_entry_t *
  496. sort_and_find_most_recent(smartlist_t *lst)
  497. {
  498. smartlist_sort(lst, compare_by_valid_after_);
  499. if (smartlist_len(lst)) {
  500. return smartlist_get(lst, smartlist_len(lst) - 1);
  501. } else {
  502. return NULL;
  503. }
  504. }
  505. /** Return i such that compress_consensus_with[i] == method. Return
  506. * -1 if no such i exists. */
  507. static int
  508. consensus_compression_method_pos(compress_method_t method)
  509. {
  510. unsigned i;
  511. for (i = 0; i < n_consensus_compression_methods(); ++i) {
  512. if (compress_consensus_with[i] == method) {
  513. return i;
  514. }
  515. }
  516. return -1;
  517. }
  518. /**
  519. * If we know a consensus with the flavor <b>flavor</b> compressed with
  520. * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
  521. * for consdiffmgr_find_diff_from().
  522. */
  523. consdiff_status_t
  524. consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
  525. consensus_flavor_t flavor,
  526. compress_method_t method)
  527. {
  528. tor_assert(entry_out);
  529. tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
  530. int pos = consensus_compression_method_pos(method);
  531. if (pos < 0) {
  532. // We don't compress consensuses with this method.
  533. return CONSDIFF_NOT_FOUND;
  534. }
  535. consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
  536. if (!handle)
  537. return CONSDIFF_NOT_FOUND;
  538. *entry_out = consensus_cache_entry_handle_get(handle);
  539. if (*entry_out)
  540. return CONSDIFF_AVAILABLE;
  541. else
  542. return CONSDIFF_NOT_FOUND;
  543. }
  544. /**
  545. * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
  546. * from the source consensus with the specified digest (which must be SHA3).
  547. *
  548. * If the diff is present, store it into *<b>entry_out</b> and return
  549. * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
  550. * CONSDIFF_IN_PROGRESS.
  551. */
  552. consdiff_status_t
  553. consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
  554. consensus_flavor_t flavor,
  555. int digest_type,
  556. const uint8_t *digest,
  557. size_t digestlen,
  558. compress_method_t method)
  559. {
  560. if (BUG(digest_type != DIGEST_SHA3_256) ||
  561. BUG(digestlen != DIGEST256_LEN)) {
  562. return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
  563. }
  564. // Try to look up the entry in the hashtable.
  565. cdm_diff_t search, *ent;
  566. memset(&search, 0, sizeof(search));
  567. search.flavor = flavor;
  568. search.compress_method = method;
  569. memcpy(search.from_sha3, digest, DIGEST256_LEN);
  570. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  571. if (ent == NULL ||
  572. ent->cdm_diff_status == CDM_DIFF_ERROR) {
  573. return CONSDIFF_NOT_FOUND;
  574. } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
  575. return CONSDIFF_IN_PROGRESS;
  576. } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
  577. return CONSDIFF_IN_PROGRESS;
  578. }
  579. if (BUG(ent->entry == NULL)) {
  580. return CONSDIFF_NOT_FOUND;
  581. }
  582. *entry_out = consensus_cache_entry_handle_get(ent->entry);
  583. return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
  584. #if 0
  585. // XXXX Remove this. I'm keeping it around for now in case we need to
  586. // XXXX debug issues in the hashtable.
  587. char hex[HEX_DIGEST256_LEN+1];
  588. base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
  589. const char *flavname = networkstatus_get_flavor_name(flavor);
  590. smartlist_t *matches = smartlist_new();
  591. consensus_cache_find_all(matches, cdm_cache_get(),
  592. LABEL_FROM_SHA3_DIGEST, hex);
  593. consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
  594. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  595. *entry_out = sort_and_find_most_recent(matches);
  596. consdiff_status_t result =
  597. (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
  598. smartlist_free(matches);
  599. return result;
  600. #endif /* 0 */
  601. }
  602. /**
  603. * Perform periodic cleanup tasks on the consensus diff cache. Return
  604. * the number of objects marked for deletion.
  605. */
  606. int
  607. consdiffmgr_cleanup(void)
  608. {
  609. smartlist_t *objects = smartlist_new();
  610. smartlist_t *consensuses = smartlist_new();
  611. smartlist_t *diffs = smartlist_new();
  612. int n_to_delete = 0;
  613. log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
  614. // 1. Delete any consensus or diff or anything whose valid_after is too old.
  615. const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
  616. consensus_cache_find_all(objects, cdm_cache_get(),
  617. NULL, NULL);
  618. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  619. const char *lv_valid_after =
  620. consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
  621. if (! lv_valid_after) {
  622. log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
  623. LABEL_VALID_AFTER);
  624. continue;
  625. }
  626. time_t valid_after = 0;
  627. if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
  628. log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
  629. "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
  630. continue;
  631. }
  632. if (valid_after < valid_after_cutoff) {
  633. log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
  634. "too old", LABEL_VALID_AFTER, lv_valid_after);
  635. consensus_cache_entry_mark_for_removal(ent);
  636. ++n_to_delete;
  637. }
  638. } SMARTLIST_FOREACH_END(ent);
  639. // 2. Delete all diffs that lead to a consensus whose valid-after is not the
  640. // latest.
  641. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  642. const char *flavname = networkstatus_get_flavor_name(flav);
  643. /* Determine the most recent consensus of this flavor */
  644. consensus_cache_find_all(consensuses, cdm_cache_get(),
  645. LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  646. consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
  647. consensus_cache_entry_t *most_recent =
  648. sort_and_find_most_recent(consensuses);
  649. if (most_recent == NULL)
  650. continue;
  651. const char *most_recent_sha3 =
  652. consensus_cache_entry_get_value(most_recent,
  653. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  654. if (BUG(most_recent_sha3 == NULL))
  655. continue; // LCOV_EXCL_LINE
  656. /* consider all such-flavored diffs, and look to see if they match. */
  657. consensus_cache_find_all(diffs, cdm_cache_get(),
  658. LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  659. consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
  660. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  661. const char *this_diff_target_sha3 =
  662. consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
  663. if (!this_diff_target_sha3)
  664. continue;
  665. if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
  666. consensus_cache_entry_mark_for_removal(diff);
  667. ++n_to_delete;
  668. }
  669. } SMARTLIST_FOREACH_END(diff);
  670. smartlist_clear(consensuses);
  671. smartlist_clear(diffs);
  672. }
  673. // 3. Delete all consensuses except the most recent that are compressed with
  674. // an un-preferred method.
  675. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  676. const char *flavname = networkstatus_get_flavor_name(flav);
  677. /* Determine the most recent consensus of this flavor */
  678. consensus_cache_find_all(consensuses, cdm_cache_get(),
  679. LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  680. consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
  681. consensus_cache_entry_t *most_recent =
  682. sort_and_find_most_recent(consensuses);
  683. if (most_recent == NULL)
  684. continue;
  685. const char *most_recent_sha3_uncompressed =
  686. consensus_cache_entry_get_value(most_recent,
  687. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  688. const char *retain_methodname = compression_method_get_name(
  689. RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
  690. if (BUG(most_recent_sha3_uncompressed == NULL))
  691. continue;
  692. SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
  693. const char *lv_sha3_uncompressed =
  694. consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
  695. if (BUG(! lv_sha3_uncompressed))
  696. continue;
  697. if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
  698. continue; // This _is_ the most recent.
  699. const char *lv_methodname =
  700. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  701. if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
  702. consensus_cache_entry_mark_for_removal(ent);
  703. ++n_to_delete;
  704. }
  705. } SMARTLIST_FOREACH_END(ent);
  706. }
  707. smartlist_free(objects);
  708. smartlist_free(consensuses);
  709. smartlist_free(diffs);
  710. // Actually remove files, if they're not used.
  711. consensus_cache_delete_pending(cdm_cache_get(), 0);
  712. return n_to_delete;
  713. }
  714. /**
  715. * Initialize the consensus diff manager and its cache, and configure
  716. * its parameters based on the latest torrc and networkstatus parameters.
  717. */
  718. void
  719. consdiffmgr_configure(const consdiff_cfg_t *cfg)
  720. {
  721. if (cfg)
  722. memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
  723. (void) cdm_cache_get();
  724. }
  725. /**
  726. * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  727. * operations that the consensus diff manager will need.
  728. */
  729. int
  730. consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
  731. {
  732. return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
  733. }
  734. /**
  735. * Scan the consensus diff manager's cache for any grossly malformed entries,
  736. * and mark them as deletable. Return 0 if no problems were found; 1
  737. * if problems were found and fixed.
  738. */
  739. int
  740. consdiffmgr_validate(void)
  741. {
  742. /* Right now, we only check for entries that have bad sha3 values */
  743. int problems = 0;
  744. smartlist_t *objects = smartlist_new();
  745. consensus_cache_find_all(objects, cdm_cache_get(),
  746. NULL, NULL);
  747. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
  748. uint8_t sha3_expected[DIGEST256_LEN];
  749. uint8_t sha3_received[DIGEST256_LEN];
  750. int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
  751. if (r == -1) {
  752. /* digest isn't there; that's allowed */
  753. continue;
  754. } else if (r == -2) {
  755. /* digest is malformed; that's not allowed */
  756. problems = 1;
  757. consensus_cache_entry_mark_for_removal(obj);
  758. continue;
  759. }
  760. const uint8_t *body;
  761. size_t bodylen;
  762. consensus_cache_entry_incref(obj);
  763. r = consensus_cache_entry_get_body(obj, &body, &bodylen);
  764. if (r == 0) {
  765. crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
  766. DIGEST_SHA3_256);
  767. }
  768. consensus_cache_entry_decref(obj);
  769. if (r < 0)
  770. continue;
  771. // Deconfuse coverity about the possibility of sha3_received being
  772. // uninitialized
  773. tor_assert(r <= 0);
  774. if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
  775. problems = 1;
  776. consensus_cache_entry_mark_for_removal(obj);
  777. continue;
  778. }
  779. } SMARTLIST_FOREACH_END(obj);
  780. smartlist_free(objects);
  781. return problems;
  782. }
  783. /**
  784. * Helper: build new diffs of <b>flavor</b> as needed
  785. */
  786. static void
  787. consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
  788. {
  789. smartlist_t *matches = NULL;
  790. smartlist_t *diffs = NULL;
  791. smartlist_t *compute_diffs_from = NULL;
  792. strmap_t *have_diff_from = NULL;
  793. // look for the most recent consensus, and for all previous in-range
  794. // consensuses. Do they all have diffs to it?
  795. const char *flavname = networkstatus_get_flavor_name(flavor);
  796. // 1. find the most recent consensus, and the ones that we might want
  797. // to diff to it.
  798. const char *methodname = compression_method_get_name(
  799. RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
  800. matches = smartlist_new();
  801. consensus_cache_find_all(matches, cdm_cache_get(),
  802. LABEL_FLAVOR, flavname);
  803. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  804. consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
  805. consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
  806. if (!most_recent) {
  807. log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
  808. "not making diffs", flavname);
  809. goto done;
  810. }
  811. tor_assert(smartlist_len(matches));
  812. smartlist_del(matches, smartlist_len(matches) - 1);
  813. const char *most_recent_valid_after =
  814. consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
  815. if (BUG(most_recent_valid_after == NULL))
  816. goto done; //LCOV_EXCL_LINE
  817. uint8_t most_recent_sha3[DIGEST256_LEN];
  818. if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
  819. LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
  820. goto done; //LCOV_EXCL_LINE
  821. // 2. Find all the relevant diffs _to_ this consensus. These are ones
  822. // that we don't need to compute.
  823. diffs = smartlist_new();
  824. consensus_cache_find_all(diffs, cdm_cache_get(),
  825. LABEL_VALID_AFTER, most_recent_valid_after);
  826. consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  827. consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
  828. have_diff_from = strmap_new();
  829. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  830. const char *va = consensus_cache_entry_get_value(diff,
  831. LABEL_FROM_VALID_AFTER);
  832. if (BUG(va == NULL))
  833. continue; // LCOV_EXCL_LINE
  834. strmap_set(have_diff_from, va, diff);
  835. } SMARTLIST_FOREACH_END(diff);
  836. // 3. See which consensuses in 'matches' don't have diffs yet.
  837. smartlist_reverse(matches); // from newest to oldest.
  838. compute_diffs_from = smartlist_new();
  839. SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
  840. const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
  841. if (BUG(va == NULL))
  842. continue; // LCOV_EXCL_LINE
  843. if (strmap_get(have_diff_from, va) != NULL)
  844. continue; /* we already have this one. */
  845. smartlist_add(compute_diffs_from, ent);
  846. /* Since we are not going to serve this as the most recent consensus
  847. * any more, we should stop keeping it mmap'd when it's not in use.
  848. */
  849. consensus_cache_entry_mark_for_aggressive_release(ent);
  850. } SMARTLIST_FOREACH_END(ent);
  851. log_info(LD_DIRSERV,
  852. "The most recent %s consensus is valid-after %s. We have diffs to "
  853. "this consensus for %d/%d older %s consensuses. Generating diffs "
  854. "for the other %d.",
  855. flavname,
  856. most_recent_valid_after,
  857. smartlist_len(matches) - smartlist_len(compute_diffs_from),
  858. smartlist_len(matches),
  859. flavname,
  860. smartlist_len(compute_diffs_from));
  861. // 4. Update the hashtable; remove entries in this flavor to other
  862. // target consensuses.
  863. cdm_diff_ht_purge(flavor, most_recent_sha3);
  864. // 5. Actually launch the requests.
  865. SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
  866. if (BUG(c == most_recent))
  867. continue; // LCOV_EXCL_LINE
  868. uint8_t this_sha3[DIGEST256_LEN];
  869. if (cdm_entry_get_sha3_value(this_sha3, c,
  870. LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
  871. // Not actually a bug, since we might be running with a directory
  872. // with stale files from before the #22143 fixes.
  873. continue;
  874. }
  875. if (cdm_diff_ht_check_and_note_pending(flavor,
  876. this_sha3, most_recent_sha3)) {
  877. // This is already pending, or we encountered an error.
  878. continue;
  879. }
  880. consensus_diff_queue_diff_work(c, most_recent);
  881. } SMARTLIST_FOREACH_END(c);
  882. done:
  883. smartlist_free(matches);
  884. smartlist_free(diffs);
  885. smartlist_free(compute_diffs_from);
  886. strmap_free(have_diff_from, NULL);
  887. }
  888. /**
  889. * Scan the cache for the latest consensuses and add their handles to
  890. * latest_consensus
  891. */
  892. static void
  893. consdiffmgr_consensus_load(void)
  894. {
  895. smartlist_t *matches = smartlist_new();
  896. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  897. const char *flavname = networkstatus_get_flavor_name(flav);
  898. smartlist_clear(matches);
  899. consensus_cache_find_all(matches, cdm_cache_get(),
  900. LABEL_FLAVOR, flavname);
  901. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  902. consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
  903. if (! most_recent)
  904. continue; // no consensuses.
  905. const char *most_recent_sha3 =
  906. consensus_cache_entry_get_value(most_recent,
  907. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  908. if (BUG(most_recent_sha3 == NULL))
  909. continue; // LCOV_EXCL_LINE
  910. consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
  911. most_recent_sha3);
  912. // Everything that remains matches the most recent consensus of this
  913. // flavor.
  914. SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
  915. const char *lv_compression =
  916. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  917. compress_method_t method =
  918. compression_method_get_by_name(lv_compression);
  919. int pos = consensus_compression_method_pos(method);
  920. if (pos < 0)
  921. continue;
  922. consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
  923. latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
  924. } SMARTLIST_FOREACH_END(ent);
  925. }
  926. smartlist_free(matches);
  927. }
  928. /**
  929. * Scan the cache for diffs, and add them to the hashtable.
  930. */
  931. static void
  932. consdiffmgr_diffs_load(void)
  933. {
  934. smartlist_t *diffs = smartlist_new();
  935. consensus_cache_find_all(diffs, cdm_cache_get(),
  936. LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  937. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  938. const char *lv_flavor =
  939. consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
  940. if (!lv_flavor)
  941. continue;
  942. int flavor = networkstatus_parse_flavor_name(lv_flavor);
  943. if (flavor < 0)
  944. continue;
  945. const char *lv_compression =
  946. consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
  947. compress_method_t method = NO_METHOD;
  948. if (lv_compression) {
  949. method = compression_method_get_by_name(lv_compression);
  950. if (method == UNKNOWN_METHOD) {
  951. continue;
  952. }
  953. }
  954. uint8_t from_sha3[DIGEST256_LEN];
  955. uint8_t to_sha3[DIGEST256_LEN];
  956. if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
  957. continue;
  958. if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
  959. continue;
  960. cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
  961. method,
  962. CDM_DIFF_PRESENT,
  963. consensus_cache_entry_handle_new(diff));
  964. } SMARTLIST_FOREACH_END(diff);
  965. smartlist_free(diffs);
  966. }
  967. /**
  968. * Build new diffs as needed.
  969. */
  970. void
  971. consdiffmgr_rescan(void)
  972. {
  973. if (cdm_cache_dirty == 0)
  974. return;
  975. // Clean up here to make room for new diffs, and to ensure that older
  976. // consensuses do not have any entries.
  977. consdiffmgr_cleanup();
  978. if (cdm_cache_loaded == 0) {
  979. consdiffmgr_diffs_load();
  980. consdiffmgr_consensus_load();
  981. cdm_cache_loaded = 1;
  982. }
  983. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  984. consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
  985. }
  986. cdm_cache_dirty = 0;
  987. }
  988. /**
  989. * Helper: compare two files by their from-valid-after and valid-after labels,
  990. * trying to sort in ascending order by from-valid-after (when present) and
  991. * valid-after (when not). Place everything that has neither label first in
  992. * the list.
  993. */
  994. static int
  995. compare_by_staleness_(const void **a, const void **b)
  996. {
  997. const consensus_cache_entry_t *e1 = *a;
  998. const consensus_cache_entry_t *e2 = *b;
  999. const char *va1, *fva1, *va2, *fva2;
  1000. va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
  1001. va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
  1002. fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
  1003. fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
  1004. if (fva1)
  1005. va1 = fva1;
  1006. if (fva2)
  1007. va2 = fva2;
  1008. /* See note about iso-encoded values in compare_by_valid_after_. Also note
  1009. * that missing dates will get placed first. */
  1010. return strcmp_opt(va1, va2);
  1011. }
  1012. /** If there are not enough unused filenames to store <b>n</b> files, then
  1013. * delete old consensuses until there are. (We have to keep track of the
  1014. * number of filenames because of the way that the seccomp2 cache works.)
  1015. *
  1016. * Return 0 on success, -1 on failure.
  1017. **/
  1018. static int
  1019. consdiffmgr_ensure_space_for_files(int n)
  1020. {
  1021. consensus_cache_t *cache = cdm_cache_get();
  1022. if (consensus_cache_get_n_filenames_available(cache) >= n) {
  1023. // there are already enough unused filenames.
  1024. return 0;
  1025. }
  1026. // Try a cheap deletion of stuff that's waiting to get deleted.
  1027. consensus_cache_delete_pending(cache, 0);
  1028. if (consensus_cache_get_n_filenames_available(cache) >= n) {
  1029. // okay, _that_ made enough filenames available.
  1030. return 0;
  1031. }
  1032. // Let's get more assertive: clean out unused stuff, and force-remove
  1033. // the files that we can.
  1034. consdiffmgr_cleanup();
  1035. consensus_cache_delete_pending(cache, 1);
  1036. const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
  1037. if (n_to_remove <= 0) {
  1038. // okay, finally!
  1039. return 0;
  1040. }
  1041. // At this point, we're going to have to throw out objects that will be
  1042. // missed. Too bad!
  1043. smartlist_t *objects = smartlist_new();
  1044. consensus_cache_find_all(objects, cache, NULL, NULL);
  1045. smartlist_sort(objects, compare_by_staleness_);
  1046. int n_marked = 0;
  1047. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  1048. consensus_cache_entry_mark_for_removal(ent);
  1049. if (++n_marked >= n_to_remove)
  1050. break;
  1051. } SMARTLIST_FOREACH_END(ent);
  1052. smartlist_free(objects);
  1053. consensus_cache_delete_pending(cache, 1);
  1054. if (consensus_cache_may_overallocate(cache)) {
  1055. /* If we're allowed to throw extra files into the cache, let's do so
  1056. * rather getting upset.
  1057. */
  1058. return 0;
  1059. }
  1060. if (BUG(n_marked < n_to_remove))
  1061. return -1;
  1062. else
  1063. return 0;
  1064. }
  1065. /**
  1066. * Set consensus cache flags on the objects in this consdiffmgr.
  1067. */
  1068. static void
  1069. consdiffmgr_set_cache_flags(void)
  1070. {
  1071. /* Right now, we just mark the consensus objects for aggressive release,
  1072. * so that they get mmapped for as little time as possible. */
  1073. smartlist_t *objects = smartlist_new();
  1074. consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
  1075. DOCTYPE_CONSENSUS);
  1076. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  1077. consensus_cache_entry_mark_for_aggressive_release(ent);
  1078. } SMARTLIST_FOREACH_END(ent);
  1079. smartlist_free(objects);
  1080. }
  1081. /**
  1082. * Called before shutdown: drop all storage held by the consdiffmgr.c module.
  1083. */
  1084. void
  1085. consdiffmgr_free_all(void)
  1086. {
  1087. cdm_diff_t **diff, **next;
  1088. for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
  1089. cdm_diff_t *this = *diff;
  1090. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  1091. cdm_diff_free(this);
  1092. }
  1093. int i;
  1094. unsigned j;
  1095. for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
  1096. for (j = 0; j < n_consensus_compression_methods(); ++j) {
  1097. consensus_cache_entry_handle_free(latest_consensus[i][j]);
  1098. }
  1099. }
  1100. memset(latest_consensus, 0, sizeof(latest_consensus));
  1101. consensus_cache_free(cons_diff_cache);
  1102. cons_diff_cache = NULL;
  1103. }
  1104. /* =====
  1105. Thread workers
  1106. =====*/
  1107. typedef struct compressed_result_t {
  1108. config_line_t *labels;
  1109. /**
  1110. * Output: Body of the diff, as compressed.
  1111. */
  1112. uint8_t *body;
  1113. /**
  1114. * Output: length of body_out
  1115. */
  1116. size_t bodylen;
  1117. } compressed_result_t;
  1118. /**
  1119. * Compress the bytestring <b>input</b> of length <b>len</b> using the
  1120. * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
  1121. *
  1122. * For each successful compression, set the fields in the <b>results_out</b>
  1123. * array in the position corresponding to the compression method. Use
  1124. * <b>labels_in</b> as a basis for the labels of the result.
  1125. *
  1126. * Return 0 if all compression succeeded; -1 if any failed.
  1127. */
  1128. static int
  1129. compress_multiple(compressed_result_t *results_out, int n_methods,
  1130. const compress_method_t *methods,
  1131. const uint8_t *input, size_t len,
  1132. const config_line_t *labels_in)
  1133. {
  1134. int rv = 0;
  1135. int i;
  1136. for (i = 0; i < n_methods; ++i) {
  1137. compress_method_t method = methods[i];
  1138. const char *methodname = compression_method_get_name(method);
  1139. char *result;
  1140. size_t sz;
  1141. if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
  1142. results_out[i].body = (uint8_t*)result;
  1143. results_out[i].bodylen = sz;
  1144. results_out[i].labels = config_lines_dup(labels_in);
  1145. cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
  1146. results_out[i].body,
  1147. results_out[i].bodylen);
  1148. config_line_prepend(&results_out[i].labels,
  1149. LABEL_COMPRESSION_TYPE,
  1150. methodname);
  1151. } else {
  1152. rv = -1;
  1153. }
  1154. }
  1155. return rv;
  1156. }
  1157. /**
  1158. * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
  1159. * as produced by compress_multiple, store them all into the
  1160. * consdiffmgr, and store handles to them in the <b>handles_out</b>
  1161. * array.
  1162. *
  1163. * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
  1164. * was stored.
  1165. */
  1166. static cdm_diff_status_t
  1167. store_multiple(consensus_cache_entry_handle_t **handles_out,
  1168. int n,
  1169. const compress_method_t *methods,
  1170. const compressed_result_t *results,
  1171. const char *description)
  1172. {
  1173. cdm_diff_status_t status = CDM_DIFF_ERROR;
  1174. consdiffmgr_ensure_space_for_files(n);
  1175. int i;
  1176. for (i = 0; i < n; ++i) {
  1177. compress_method_t method = methods[i];
  1178. uint8_t *body_out = results[i].body;
  1179. size_t bodylen_out = results[i].bodylen;
  1180. config_line_t *labels = results[i].labels;
  1181. const char *methodname = compression_method_get_name(method);
  1182. if (body_out && bodylen_out && labels) {
  1183. /* Success! Store the results */
  1184. log_info(LD_DIRSERV, "Adding %s, compressed with %s",
  1185. description, methodname);
  1186. consensus_cache_entry_t *ent =
  1187. consensus_cache_add(cdm_cache_get(),
  1188. labels,
  1189. body_out,
  1190. bodylen_out);
  1191. if (ent == NULL) {
  1192. static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
  1193. log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
  1194. "Unable to store object %s compressed with %s.",
  1195. description, methodname);
  1196. continue;
  1197. }
  1198. status = CDM_DIFF_PRESENT;
  1199. handles_out[i] = consensus_cache_entry_handle_new(ent);
  1200. consensus_cache_entry_decref(ent);
  1201. }
  1202. }
  1203. return status;
  1204. }
  1205. /**
  1206. * An object passed to a worker thread that will try to produce a consensus
  1207. * diff.
  1208. */
  1209. typedef struct consensus_diff_worker_job_t {
  1210. /**
  1211. * Input: The consensus to compute the diff from. Holds a reference to the
  1212. * cache entry, which must not be released until the job is passed back to
  1213. * the main thread. The body must be mapped into memory in the main thread.
  1214. */
  1215. consensus_cache_entry_t *diff_from;
  1216. /**
  1217. * Input: The consensus to compute the diff to. Holds a reference to the
  1218. * cache entry, which must not be released until the job is passed back to
  1219. * the main thread. The body must be mapped into memory in the main thread.
  1220. */
  1221. consensus_cache_entry_t *diff_to;
  1222. /** Output: labels and bodies */
  1223. compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
  1224. } consensus_diff_worker_job_t;
  1225. /** Given a consensus_cache_entry_t, check whether it has a label claiming
  1226. * that it was compressed. If so, uncompress its contents into <b>out</b> and
  1227. * set <b>outlen</b> to hold their size. If not, just copy the body into
  1228. * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success,
  1229. * -1 on failure.
  1230. *
  1231. * In all cases, the output is nul-terminated. */
  1232. STATIC int
  1233. uncompress_or_copy(char **out, size_t *outlen,
  1234. consensus_cache_entry_t *ent)
  1235. {
  1236. const uint8_t *body;
  1237. size_t bodylen;
  1238. if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
  1239. return -1;
  1240. const char *lv_compression =
  1241. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  1242. compress_method_t method = NO_METHOD;
  1243. if (lv_compression)
  1244. method = compression_method_get_by_name(lv_compression);
  1245. return tor_uncompress(out, outlen, (const char *)body, bodylen,
  1246. method, 1, LOG_WARN);
  1247. }
  1248. /**
  1249. * Worker function. This function runs inside a worker thread and receives
  1250. * a consensus_diff_worker_job_t as its input.
  1251. */
  1252. static workqueue_reply_t
  1253. consensus_diff_worker_threadfn(void *state_, void *work_)
  1254. {
  1255. (void)state_;
  1256. consensus_diff_worker_job_t *job = work_;
  1257. const uint8_t *diff_from, *diff_to;
  1258. size_t len_from, len_to;
  1259. int r;
  1260. /* We need to have the body already mapped into RAM here.
  1261. */
  1262. r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
  1263. if (BUG(r < 0))
  1264. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1265. r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
  1266. if (BUG(r < 0))
  1267. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1268. const char *lv_to_valid_after =
  1269. consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
  1270. const char *lv_to_fresh_until =
  1271. consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
  1272. const char *lv_to_valid_until =
  1273. consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
  1274. const char *lv_to_signatories =
  1275. consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
  1276. const char *lv_from_valid_after =
  1277. consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
  1278. const char *lv_from_digest =
  1279. consensus_cache_entry_get_value(job->diff_from,
  1280. LABEL_SHA3_DIGEST_AS_SIGNED);
  1281. const char *lv_from_flavor =
  1282. consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
  1283. const char *lv_to_flavor =
  1284. consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
  1285. const char *lv_to_digest =
  1286. consensus_cache_entry_get_value(job->diff_to,
  1287. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  1288. if (! lv_from_digest) {
  1289. /* This isn't a bug right now, since it can happen if you're migrating
  1290. * from an older version of master to a newer one. The older ones didn't
  1291. * annotate their stored consensus objects with sha3-digest-as-signed.
  1292. */
  1293. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1294. }
  1295. /* All these values are mandatory on the input */
  1296. if (BUG(!lv_to_valid_after) ||
  1297. BUG(!lv_from_valid_after) ||
  1298. BUG(!lv_from_flavor) ||
  1299. BUG(!lv_to_flavor)) {
  1300. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1301. }
  1302. /* The flavors need to match */
  1303. if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
  1304. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1305. }
  1306. char *consensus_diff;
  1307. {
  1308. char *diff_from_nt = NULL, *diff_to_nt = NULL;
  1309. size_t diff_from_nt_len, diff_to_nt_len;
  1310. if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
  1311. job->diff_from) < 0) {
  1312. return WQ_RPL_REPLY;
  1313. }
  1314. if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
  1315. job->diff_to) < 0) {
  1316. tor_free(diff_from_nt);
  1317. return WQ_RPL_REPLY;
  1318. }
  1319. tor_assert(diff_from_nt);
  1320. tor_assert(diff_to_nt);
  1321. // XXXX ugh; this is going to calculate the SHA3 of both its
  1322. // XXXX inputs again, even though we already have that. Maybe it's time
  1323. // XXXX to change the API here?
  1324. consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
  1325. tor_free(diff_from_nt);
  1326. tor_free(diff_to_nt);
  1327. }
  1328. if (!consensus_diff) {
  1329. /* Couldn't generate consensus; we'll leave the reply blank. */
  1330. return WQ_RPL_REPLY;
  1331. }
  1332. /* Compress the results and send the reply */
  1333. tor_assert(compress_diffs_with[0] == NO_METHOD);
  1334. size_t difflen = strlen(consensus_diff);
  1335. job->out[0].body = (uint8_t *) consensus_diff;
  1336. job->out[0].bodylen = difflen;
  1337. config_line_t *common_labels = NULL;
  1338. if (lv_to_valid_until)
  1339. config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
  1340. if (lv_to_fresh_until)
  1341. config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
  1342. if (lv_to_signatories)
  1343. config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
  1344. cdm_labels_prepend_sha3(&common_labels,
  1345. LABEL_SHA3_DIGEST_UNCOMPRESSED,
  1346. job->out[0].body,
  1347. job->out[0].bodylen);
  1348. config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
  1349. lv_from_valid_after);
  1350. config_line_prepend(&common_labels, LABEL_VALID_AFTER,
  1351. lv_to_valid_after);
  1352. config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
  1353. config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
  1354. lv_from_digest);
  1355. config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
  1356. lv_to_digest);
  1357. config_line_prepend(&common_labels, LABEL_DOCTYPE,
  1358. DOCTYPE_CONSENSUS_DIFF);
  1359. job->out[0].labels = config_lines_dup(common_labels);
  1360. cdm_labels_prepend_sha3(&job->out[0].labels,
  1361. LABEL_SHA3_DIGEST,
  1362. job->out[0].body,
  1363. job->out[0].bodylen);
  1364. compress_multiple(job->out+1,
  1365. n_diff_compression_methods()-1,
  1366. compress_diffs_with+1,
  1367. (const uint8_t*)consensus_diff, difflen, common_labels);
  1368. config_free_lines(common_labels);
  1369. return WQ_RPL_REPLY;
  1370. }
  1371. #define consensus_diff_worker_job_free(job) \
  1372. FREE_AND_NULL(consensus_diff_worker_job_t, \
  1373. consensus_diff_worker_job_free_, (job))
  1374. /**
  1375. * Helper: release all storage held in <b>job</b>.
  1376. */
  1377. static void
  1378. consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
  1379. {
  1380. if (!job)
  1381. return;
  1382. unsigned u;
  1383. for (u = 0; u < n_diff_compression_methods(); ++u) {
  1384. config_free_lines(job->out[u].labels);
  1385. tor_free(job->out[u].body);
  1386. }
  1387. consensus_cache_entry_decref(job->diff_from);
  1388. consensus_cache_entry_decref(job->diff_to);
  1389. tor_free(job);
  1390. }
  1391. /**
  1392. * Worker function: This function runs in the main thread, and receives
  1393. * a consensus_diff_worker_job_t that the worker thread has already
  1394. * processed.
  1395. */
  1396. static void
  1397. consensus_diff_worker_replyfn(void *work_)
  1398. {
  1399. tor_assert(in_main_thread());
  1400. tor_assert(work_);
  1401. consensus_diff_worker_job_t *job = work_;
  1402. const char *lv_from_digest =
  1403. consensus_cache_entry_get_value(job->diff_from,
  1404. LABEL_SHA3_DIGEST_AS_SIGNED);
  1405. const char *lv_to_digest =
  1406. consensus_cache_entry_get_value(job->diff_to,
  1407. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  1408. const char *lv_flavor =
  1409. consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
  1410. if (BUG(lv_from_digest == NULL))
  1411. lv_from_digest = "???"; // LCOV_EXCL_LINE
  1412. if (BUG(lv_to_digest == NULL))
  1413. lv_to_digest = "???"; // LCOV_EXCL_LINE
  1414. uint8_t from_sha3[DIGEST256_LEN];
  1415. uint8_t to_sha3[DIGEST256_LEN];
  1416. int flav = -1;
  1417. int cache = 1;
  1418. if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
  1419. LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
  1420. cache = 0;
  1421. if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
  1422. LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
  1423. cache = 0;
  1424. if (BUG(lv_flavor == NULL)) {
  1425. cache = 0;
  1426. } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
  1427. cache = 0;
  1428. }
  1429. consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
  1430. memset(handles, 0, sizeof(handles));
  1431. char description[128];
  1432. tor_snprintf(description, sizeof(description),
  1433. "consensus diff from %s to %s",
  1434. lv_from_digest, lv_to_digest);
  1435. int status = store_multiple(handles,
  1436. n_diff_compression_methods(),
  1437. compress_diffs_with,
  1438. job->out,
  1439. description);
  1440. if (status != CDM_DIFF_PRESENT) {
  1441. /* Failure! Nothing to do but complain */
  1442. log_warn(LD_DIRSERV,
  1443. "Worker was unable to compute consensus diff "
  1444. "from %s to %s", lv_from_digest, lv_to_digest);
  1445. /* Cache this error so we don't try to compute this one again. */
  1446. status = CDM_DIFF_ERROR;
  1447. }
  1448. unsigned u;
  1449. for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
  1450. compress_method_t method = compress_diffs_with[u];
  1451. if (cache) {
  1452. consensus_cache_entry_handle_t *h = handles[u];
  1453. int this_status = status;
  1454. if (h == NULL) {
  1455. this_status = CDM_DIFF_ERROR;
  1456. }
  1457. tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
  1458. cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
  1459. } else {
  1460. consensus_cache_entry_handle_free(handles[u]);
  1461. }
  1462. }
  1463. consensus_diff_worker_job_free(job);
  1464. }
  1465. /**
  1466. * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
  1467. * in a worker thread.
  1468. */
  1469. static int
  1470. consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
  1471. consensus_cache_entry_t *diff_to)
  1472. {
  1473. tor_assert(in_main_thread());
  1474. consensus_cache_entry_incref(diff_from);
  1475. consensus_cache_entry_incref(diff_to);
  1476. consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
  1477. job->diff_from = diff_from;
  1478. job->diff_to = diff_to;
  1479. /* Make sure body is mapped. */
  1480. const uint8_t *body;
  1481. size_t bodylen;
  1482. int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
  1483. int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
  1484. if (r1 < 0 || r2 < 0)
  1485. goto err;
  1486. workqueue_entry_t *work;
  1487. work = cpuworker_queue_work(WQ_PRI_LOW,
  1488. consensus_diff_worker_threadfn,
  1489. consensus_diff_worker_replyfn,
  1490. job);
  1491. if (!work)
  1492. goto err;
  1493. return 0;
  1494. err:
  1495. consensus_diff_worker_job_free(job); // includes decrefs.
  1496. return -1;
  1497. }
  1498. /**
  1499. * Holds requests and replies for consensus_compress_workers.
  1500. */
  1501. typedef struct consensus_compress_worker_job_t {
  1502. char *consensus;
  1503. size_t consensus_len;
  1504. consensus_flavor_t flavor;
  1505. config_line_t *labels_in;
  1506. compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
  1507. } consensus_compress_worker_job_t;
  1508. #define consensus_compress_worker_job_free(job) \
  1509. FREE_AND_NULL(consensus_compress_worker_job_t, \
  1510. consensus_compress_worker_job_free_, (job))
  1511. /**
  1512. * Free all resources held in <b>job</b>
  1513. */
  1514. static void
  1515. consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
  1516. {
  1517. if (!job)
  1518. return;
  1519. tor_free(job->consensus);
  1520. config_free_lines(job->labels_in);
  1521. unsigned u;
  1522. for (u = 0; u < n_consensus_compression_methods(); ++u) {
  1523. config_free_lines(job->out[u].labels);
  1524. tor_free(job->out[u].body);
  1525. }
  1526. tor_free(job);
  1527. }
  1528. /**
  1529. * Worker function. This function runs inside a worker thread and receives
  1530. * a consensus_compress_worker_job_t as its input.
  1531. */
  1532. static workqueue_reply_t
  1533. consensus_compress_worker_threadfn(void *state_, void *work_)
  1534. {
  1535. (void)state_;
  1536. consensus_compress_worker_job_t *job = work_;
  1537. consensus_flavor_t flavor = job->flavor;
  1538. const char *consensus = job->consensus;
  1539. size_t bodylen = job->consensus_len;
  1540. config_line_t *labels = config_lines_dup(job->labels_in);
  1541. const char *flavname = networkstatus_get_flavor_name(flavor);
  1542. cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
  1543. (const uint8_t *)consensus, bodylen);
  1544. {
  1545. const char *start, *end;
  1546. if (router_get_networkstatus_v3_signed_boundaries(consensus,
  1547. &start, &end) < 0) {
  1548. start = consensus;
  1549. end = consensus+bodylen;
  1550. }
  1551. cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
  1552. (const uint8_t *)start,
  1553. end - start);
  1554. }
  1555. config_line_prepend(&labels, LABEL_FLAVOR, flavname);
  1556. config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  1557. compress_multiple(job->out,
  1558. n_consensus_compression_methods(),
  1559. compress_consensus_with,
  1560. (const uint8_t*)consensus, bodylen, labels);
  1561. config_free_lines(labels);
  1562. return WQ_RPL_REPLY;
  1563. }
  1564. /**
  1565. * Worker function: This function runs in the main thread, and receives
  1566. * a consensus_diff_compress_job_t that the worker thread has already
  1567. * processed.
  1568. */
  1569. static void
  1570. consensus_compress_worker_replyfn(void *work_)
  1571. {
  1572. consensus_compress_worker_job_t *job = work_;
  1573. consensus_cache_entry_handle_t *handles[
  1574. ARRAY_LENGTH(compress_consensus_with)];
  1575. memset(handles, 0, sizeof(handles));
  1576. store_multiple(handles,
  1577. n_consensus_compression_methods(),
  1578. compress_consensus_with,
  1579. job->out,
  1580. "consensus");
  1581. cdm_cache_dirty = 1;
  1582. unsigned u;
  1583. consensus_flavor_t f = job->flavor;
  1584. tor_assert((int)f < N_CONSENSUS_FLAVORS);
  1585. for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
  1586. if (handles[u] == NULL)
  1587. continue;
  1588. consensus_cache_entry_handle_free(latest_consensus[f][u]);
  1589. latest_consensus[f][u] = handles[u];
  1590. }
  1591. consensus_compress_worker_job_free(job);
  1592. }
  1593. /**
  1594. * If true, we compress in worker threads.
  1595. */
  1596. static int background_compression = 0;
  1597. /**
  1598. * Queue a job to compress <b>consensus</b> and store its compressed
  1599. * text in the cache.
  1600. */
  1601. static int
  1602. consensus_queue_compression_work(const char *consensus,
  1603. const networkstatus_t *as_parsed)
  1604. {
  1605. tor_assert(consensus);
  1606. tor_assert(as_parsed);
  1607. consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
  1608. job->consensus = tor_strdup(consensus);
  1609. job->consensus_len = strlen(consensus);
  1610. job->flavor = as_parsed->flavor;
  1611. char va_str[ISO_TIME_LEN+1];
  1612. char vu_str[ISO_TIME_LEN+1];
  1613. char fu_str[ISO_TIME_LEN+1];
  1614. format_iso_time_nospace(va_str, as_parsed->valid_after);
  1615. format_iso_time_nospace(fu_str, as_parsed->fresh_until);
  1616. format_iso_time_nospace(vu_str, as_parsed->valid_until);
  1617. config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
  1618. config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
  1619. config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
  1620. if (as_parsed->voters) {
  1621. smartlist_t *hexvoters = smartlist_new();
  1622. SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
  1623. networkstatus_voter_info_t *, vi) {
  1624. if (smartlist_len(vi->sigs) == 0)
  1625. continue; // didn't sign.
  1626. char d[HEX_DIGEST_LEN+1];
  1627. base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
  1628. smartlist_add_strdup(hexvoters, d);
  1629. } SMARTLIST_FOREACH_END(vi);
  1630. char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
  1631. config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
  1632. tor_free(signers);
  1633. SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
  1634. smartlist_free(hexvoters);
  1635. }
  1636. if (background_compression) {
  1637. workqueue_entry_t *work;
  1638. work = cpuworker_queue_work(WQ_PRI_LOW,
  1639. consensus_compress_worker_threadfn,
  1640. consensus_compress_worker_replyfn,
  1641. job);
  1642. if (!work) {
  1643. consensus_compress_worker_job_free(job);
  1644. return -1;
  1645. }
  1646. return 0;
  1647. } else {
  1648. consensus_compress_worker_threadfn(NULL, job);
  1649. consensus_compress_worker_replyfn(job);
  1650. return 0;
  1651. }
  1652. }
  1653. /**
  1654. * Tell the consdiffmgr backend to compress consensuses in worker threads.
  1655. */
  1656. void
  1657. consdiffmgr_enable_background_compression(void)
  1658. {
  1659. // This isn't the default behavior because it would break unit tests.
  1660. background_compression = 1;
  1661. }
  1662. /** Read the set of voters from the cached object <b>ent</b> into
  1663. * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
  1664. * -1 if no signatories were recorded. */
  1665. int
  1666. consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
  1667. smartlist_t *out)
  1668. {
  1669. tor_assert(ent);
  1670. tor_assert(out);
  1671. const char *s;
  1672. s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
  1673. if (s == NULL)
  1674. return -1;
  1675. smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
  1676. return 0;
  1677. }
  1678. /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
  1679. * and return 0, or return -1 if no such time was recorded. */
  1680. int
  1681. consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
  1682. time_t *out)
  1683. {
  1684. tor_assert(ent);
  1685. tor_assert(out);
  1686. const char *s;
  1687. s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
  1688. if (s == NULL || parse_iso_time_nospace(s, out) < 0)
  1689. return -1;
  1690. else
  1691. return 0;
  1692. }
  1693. /** Read the valid until timestamp from the cached object <b>ent</b> into
  1694. * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
  1695. int
  1696. consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
  1697. time_t *out)
  1698. {
  1699. tor_assert(ent);
  1700. tor_assert(out);
  1701. const char *s;
  1702. s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
  1703. if (s == NULL || parse_iso_time_nospace(s, out) < 0)
  1704. return -1;
  1705. else
  1706. return 0;
  1707. }
  1708. /** Read the valid after timestamp from the cached object <b>ent</b> into
  1709. * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
  1710. int
  1711. consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
  1712. time_t *out)
  1713. {
  1714. tor_assert(ent);
  1715. tor_assert(out);
  1716. const char *s;
  1717. s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
  1718. if (s == NULL || parse_iso_time_nospace(s, out) < 0)
  1719. return -1;
  1720. else
  1721. return 0;
  1722. }