consdiffmgr.c 58 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790
  1. /* Copyright (c) 2017, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /**
  4. * \file consdiffmsr.c
  5. *
  6. * \brief consensus diff manager functions
  7. *
  8. * This module is run by directory authorities and caches in order
  9. * to remember a number of past consensus documents, and to generate
  10. * and serve the diffs from those documents to the latest consensus.
  11. */
  12. #define CONSDIFFMGR_PRIVATE
  13. #include "or.h"
  14. #include "conscache.h"
  15. #include "consdiff.h"
  16. #include "consdiffmgr.h"
  17. #include "cpuworker.h"
  18. #include "networkstatus.h"
  19. #include "routerparse.h"
  20. #include "workqueue.h"
  21. /**
  22. * Labels to apply to items in the conscache object.
  23. *
  24. * @{
  25. */
  26. /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
  27. #define LABEL_DOCTYPE "document-type"
  28. /* The valid-after time for a consensus (or for the target consensus of a
  29. * diff), encoded as ISO UTC. */
  30. #define LABEL_VALID_AFTER "consensus-valid-after"
  31. /* The fresh-until time for a consensus (or for the target consensus of a
  32. * diff), encoded as ISO UTC. */
  33. #define LABEL_FRESH_UNTIL "consensus-fresh-until"
  34. /* The valid-until time for a consensus (or for the target consensus of a
  35. * diff), encoded as ISO UTC. */
  36. #define LABEL_VALID_UNTIL "consensus-valid-until"
  37. /* Comma-separated list of hex-encoded identity digests for the voting
  38. * authorities. */
  39. #define LABEL_SIGNATORIES "consensus-signatories"
  40. /* A hex encoded SHA3 digest of the object, as compressed (if any) */
  41. #define LABEL_SHA3_DIGEST "sha3-digest"
  42. /* A hex encoded SHA3 digest of the object before compression. */
  43. #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
  44. /* A hex encoded SHA3 digest-as-signed of a consensus */
  45. #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
  46. /* The flavor of the consensus or consensuses diff */
  47. #define LABEL_FLAVOR "consensus-flavor"
  48. /* Diff only: the SHA3 digest-as-signed of the source consensus. */
  49. #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
  50. /* Diff only: the SHA3 digest-in-full of the target consensus. */
  51. #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
  52. /* Diff only: the valid-after date of the source consensus. */
  53. #define LABEL_FROM_VALID_AFTER "from-valid-after"
  54. /* What kind of compression was used? */
  55. #define LABEL_COMPRESSION_TYPE "compression"
  56. /** @} */
  57. #define DOCTYPE_CONSENSUS "consensus"
  58. #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
  59. /**
  60. * Underlying directory that stores consensuses and consensus diffs. Don't
  61. * use this directly: use cdm_cache_get() instead.
  62. */
  63. static consensus_cache_t *cons_diff_cache = NULL;
  64. /**
  65. * If true, we have learned at least one new consensus since the
  66. * consensus cache was last up-to-date.
  67. */
  68. static int cdm_cache_dirty = 0;
  69. /**
  70. * If true, we have scanned the cache to update our hashtable of diffs.
  71. */
  72. static int cdm_cache_loaded = 0;
  73. /**
  74. * Possible status values for cdm_diff_t.cdm_diff_status
  75. **/
  76. typedef enum cdm_diff_status_t {
  77. CDM_DIFF_PRESENT=1,
  78. CDM_DIFF_IN_PROGRESS=2,
  79. CDM_DIFF_ERROR=3,
  80. } cdm_diff_status_t;
  81. /** Which methods do we use for precompressing diffs? */
  82. static const compress_method_t compress_diffs_with[] = {
  83. NO_METHOD,
  84. GZIP_METHOD,
  85. #ifdef HAVE_LZMA
  86. LZMA_METHOD,
  87. #endif
  88. #ifdef HAVE_ZSTD
  89. ZSTD_METHOD,
  90. #endif
  91. };
  92. /** How many different methods will we try to use for diff compression? */
  93. STATIC unsigned
  94. n_diff_compression_methods(void)
  95. {
  96. return ARRAY_LENGTH(compress_diffs_with);
  97. }
  98. /** Which methods do we use for precompressing consensuses? */
  99. static const compress_method_t compress_consensus_with[] = {
  100. ZLIB_METHOD,
  101. #ifdef HAVE_LZMA
  102. LZMA_METHOD,
  103. #endif
  104. #ifdef HAVE_ZSTD
  105. ZSTD_METHOD,
  106. #endif
  107. };
  108. /** How many different methods will we try to use for diff compression? */
  109. STATIC unsigned
  110. n_consensus_compression_methods(void)
  111. {
  112. return ARRAY_LENGTH(compress_consensus_with);
  113. }
  114. /** For which compression method do we retain old consensuses? There's no
  115. * need to keep all of them, since we won't be serving them. We'll
  116. * go with ZLIB_METHOD because it's pretty fast and everyone has it.
  117. */
  118. #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
  119. /** Handles pointing to the latest consensus entries as compressed and
  120. * stored. */
  121. static consensus_cache_entry_handle_t *
  122. latest_consensus[N_CONSENSUS_FLAVORS]
  123. [ARRAY_LENGTH(compress_consensus_with)];
  124. /** Hashtable node used to remember the current status of the diff
  125. * from a given sha3 digest to the current consensus. */
  126. typedef struct cdm_diff_t {
  127. HT_ENTRY(cdm_diff_t) node;
  128. /** Consensus flavor for this diff (part of ht key) */
  129. consensus_flavor_t flavor;
  130. /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
  131. * ht key) */
  132. uint8_t from_sha3[DIGEST256_LEN];
  133. /** Method by which the diff is compressed. (part of the ht key */
  134. compress_method_t compress_method;
  135. /** One of the CDM_DIFF_* values, depending on whether this diff
  136. * is available, in progress, or impossible to compute. */
  137. cdm_diff_status_t cdm_diff_status;
  138. /** SHA3-256 digest of the consensus that this diff is _to. */
  139. uint8_t target_sha3[DIGEST256_LEN];
  140. /** Handle to the cache entry for this diff, if any. We use a handle here
  141. * to avoid thinking too hard about cache entry lifetime issues. */
  142. consensus_cache_entry_handle_t *entry;
  143. } cdm_diff_t;
  144. /** Hashtable mapping flavor and source consensus digest to status. */
  145. static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
  146. /**
  147. * Configuration for this module
  148. */
  149. static consdiff_cfg_t consdiff_cfg = {
  150. /* .cache_max_age_hours = */ 24 * 90,
  151. // XXXX I'd like to make this number bigger, but it interferes with the
  152. // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
  153. // XXXX rules.
  154. /* .cache_max_num = */ 128
  155. };
  156. static int consdiffmgr_ensure_space_for_files(int n);
  157. static int consensus_queue_compression_work(const char *consensus,
  158. consensus_flavor_t flavor,
  159. time_t valid_after);
  160. static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
  161. consensus_cache_entry_t *diff_to);
  162. static void consdiffmgr_set_cache_flags(void);
  163. /* =====
  164. * Hashtable setup
  165. * ===== */
  166. /** Helper: hash the key of a cdm_diff_t. */
  167. static unsigned
  168. cdm_diff_hash(const cdm_diff_t *diff)
  169. {
  170. uint8_t tmp[DIGEST256_LEN + 2];
  171. memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
  172. tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
  173. tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
  174. return (unsigned) siphash24g(tmp, sizeof(tmp));
  175. }
  176. /** Helper: compare two cdm_diff_t objects for key equality */
  177. static int
  178. cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
  179. {
  180. return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
  181. diff1->flavor == diff2->flavor &&
  182. diff1->compress_method == diff2->compress_method;
  183. }
  184. HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
  185. HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
  186. 0.6, tor_reallocarray, tor_free_)
  187. /** Release all storage held in <b>diff</b>. */
  188. static void
  189. cdm_diff_free(cdm_diff_t *diff)
  190. {
  191. if (!diff)
  192. return;
  193. consensus_cache_entry_handle_free(diff->entry);
  194. tor_free(diff);
  195. }
  196. /** Create and return a new cdm_diff_t with the given values. Does not
  197. * add it to the hashtable. */
  198. static cdm_diff_t *
  199. cdm_diff_new(consensus_flavor_t flav,
  200. const uint8_t *from_sha3,
  201. const uint8_t *target_sha3,
  202. compress_method_t method)
  203. {
  204. cdm_diff_t *ent;
  205. ent = tor_malloc_zero(sizeof(cdm_diff_t));
  206. ent->flavor = flav;
  207. memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
  208. memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
  209. ent->compress_method = method;
  210. return ent;
  211. }
  212. /**
  213. * Examine the diff hashtable to see whether we know anything about computing
  214. * a diff of type <b>flav</b> between consensuses with the two provided
  215. * SHA3-256 digests. If a computation is in progress, or if the computation
  216. * has already been tried and failed, return 1. Otherwise, note the
  217. * computation as "in progress" so that we don't reattempt it later, and
  218. * return 0.
  219. */
  220. static int
  221. cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
  222. const uint8_t *from_sha3,
  223. const uint8_t *target_sha3)
  224. {
  225. struct cdm_diff_t search, *ent;
  226. unsigned u;
  227. int result = 0;
  228. for (u = 0; u < n_diff_compression_methods(); ++u) {
  229. compress_method_t method = compress_diffs_with[u];
  230. memset(&search, 0, sizeof(cdm_diff_t));
  231. search.flavor = flav;
  232. search.compress_method = method;
  233. memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
  234. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  235. if (ent) {
  236. tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
  237. result = 1;
  238. continue;
  239. }
  240. ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
  241. ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
  242. HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
  243. }
  244. return result;
  245. }
  246. /**
  247. * Update the status of the diff of type <b>flav</b> between consensuses with
  248. * the two provided SHA3-256 digests, so that its status becomes
  249. * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
  250. * is NULL, then the old handle (if any) is freed, and replaced with NULL.
  251. */
  252. static void
  253. cdm_diff_ht_set_status(consensus_flavor_t flav,
  254. const uint8_t *from_sha3,
  255. const uint8_t *to_sha3,
  256. compress_method_t method,
  257. int status,
  258. consensus_cache_entry_handle_t *handle)
  259. {
  260. struct cdm_diff_t search, *ent;
  261. memset(&search, 0, sizeof(cdm_diff_t));
  262. search.flavor = flav;
  263. search.compress_method = method,
  264. memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
  265. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  266. if (!ent) {
  267. ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
  268. ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
  269. HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
  270. } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
  271. // This can happen under certain really pathological conditions
  272. // if we decide we don't care about a diff before it is actually
  273. // done computing.
  274. return;
  275. }
  276. tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
  277. ent->cdm_diff_status = status;
  278. consensus_cache_entry_handle_free(ent->entry);
  279. ent->entry = handle;
  280. }
  281. /**
  282. * Helper: Remove from the hash table every present (actually computed) diff
  283. * of type <b>flav</b> whose target digest does not match
  284. * <b>unless_target_sha3_matches</b>.
  285. *
  286. * This function is used for the hash table to throw away references to diffs
  287. * that do not lead to the most given consensus of a given flavor.
  288. */
  289. static void
  290. cdm_diff_ht_purge(consensus_flavor_t flav,
  291. const uint8_t *unless_target_sha3_matches)
  292. {
  293. cdm_diff_t **diff, **next;
  294. for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
  295. cdm_diff_t *this = *diff;
  296. if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
  297. flav == (*diff)->flavor) {
  298. if (consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
  299. /* the underlying entry has gone away; drop this. */
  300. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  301. cdm_diff_free(this);
  302. continue;
  303. }
  304. if (unless_target_sha3_matches &&
  305. fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
  306. DIGEST256_LEN)) {
  307. /* target hash doesn't match; drop this. */
  308. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  309. cdm_diff_free(this);
  310. continue;
  311. }
  312. }
  313. next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
  314. }
  315. }
  316. /**
  317. * Helper: initialize <b>cons_diff_cache</b>.
  318. */
  319. static void
  320. cdm_cache_init(void)
  321. {
  322. unsigned n_entries = consdiff_cfg.cache_max_num * 2;
  323. tor_assert(cons_diff_cache == NULL);
  324. cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
  325. if (cons_diff_cache == NULL) {
  326. // LCOV_EXCL_START
  327. log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
  328. tor_assert_unreached();
  329. // LCOV_EXCL_STOP
  330. } else {
  331. consdiffmgr_set_cache_flags();
  332. }
  333. cdm_cache_dirty = 1;
  334. cdm_cache_loaded = 0;
  335. }
  336. /**
  337. * Helper: return the consensus_cache_t * that backs this manager,
  338. * initializing it if needed.
  339. */
  340. STATIC consensus_cache_t *
  341. cdm_cache_get(void)
  342. {
  343. if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
  344. cdm_cache_init();
  345. }
  346. return cons_diff_cache;
  347. }
  348. /**
  349. * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
  350. * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
  351. * with <b>label</b> as its label.
  352. */
  353. static void
  354. cdm_labels_prepend_sha3(config_line_t **labels,
  355. const char *label,
  356. const uint8_t *body,
  357. size_t bodylen)
  358. {
  359. uint8_t sha3_digest[DIGEST256_LEN];
  360. char hexdigest[HEX_DIGEST256_LEN+1];
  361. crypto_digest256((char *)sha3_digest,
  362. (const char *)body, bodylen, DIGEST_SHA3_256);
  363. base16_encode(hexdigest, sizeof(hexdigest),
  364. (const char *)sha3_digest, sizeof(sha3_digest));
  365. config_line_prepend(labels, label, hexdigest);
  366. }
  367. /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
  368. * given label, set <b>digest_out</b> to that value (decoded), and return 0.
  369. *
  370. * Return -1 if there is no such label, and -2 if it is badly formatted. */
  371. STATIC int
  372. cdm_entry_get_sha3_value(uint8_t *digest_out,
  373. consensus_cache_entry_t *ent,
  374. const char *label)
  375. {
  376. if (ent == NULL)
  377. return -1;
  378. const char *hex = consensus_cache_entry_get_value(ent, label);
  379. if (hex == NULL)
  380. return -1;
  381. int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
  382. if (n != DIGEST256_LEN)
  383. return -2;
  384. else
  385. return 0;
  386. }
  387. /**
  388. * Helper: look for a consensus with the given <b>flavor</b> and
  389. * <b>valid_after</b> time in the cache. Return that consensus if it's
  390. * present, or NULL if it's missing.
  391. */
  392. STATIC consensus_cache_entry_t *
  393. cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
  394. {
  395. char formatted_time[ISO_TIME_LEN+1];
  396. format_iso_time_nospace(formatted_time, valid_after);
  397. const char *flavname = networkstatus_get_flavor_name(flavor);
  398. /* We'll filter by valid-after time first, since that should
  399. * match the fewest documents. */
  400. /* We could add an extra hashtable here, but since we only do this scan
  401. * when adding a new consensus, it probably doesn't matter much. */
  402. smartlist_t *matches = smartlist_new();
  403. consensus_cache_find_all(matches, cdm_cache_get(),
  404. LABEL_VALID_AFTER, formatted_time);
  405. consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
  406. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  407. consensus_cache_entry_t *result = NULL;
  408. if (smartlist_len(matches)) {
  409. result = smartlist_get(matches, 0);
  410. }
  411. smartlist_free(matches);
  412. return result;
  413. }
  414. /**
  415. * Given a string containing a networkstatus consensus, and the results of
  416. * having parsed that consensus, add that consensus to the cache if it is not
  417. * already present and not too old. Create new consensus diffs from or to
  418. * that consensus as appropriate.
  419. *
  420. * Return 0 on success and -1 on failure.
  421. */
  422. int
  423. consdiffmgr_add_consensus(const char *consensus,
  424. const networkstatus_t *as_parsed)
  425. {
  426. if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
  427. return -1; // LCOV_EXCL_LINE
  428. if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
  429. return -1; // LCOV_EXCL_LINE
  430. const consensus_flavor_t flavor = as_parsed->flavor;
  431. const time_t valid_after = as_parsed->valid_after;
  432. if (valid_after < approx_time() - 3600 * consdiff_cfg.cache_max_age_hours) {
  433. log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
  434. "too old.");
  435. return -1;
  436. }
  437. /* Do we already have this one? */
  438. consensus_cache_entry_t *entry =
  439. cdm_cache_lookup_consensus(flavor, valid_after);
  440. if (entry) {
  441. log_info(LD_DIRSERV, "We already have a copy of that consensus");
  442. return -1;
  443. }
  444. /* We don't have it. Add it to the cache. */
  445. return consensus_queue_compression_work(consensus, flavor, valid_after);
  446. }
  447. /**
  448. * Helper: used to sort two smartlists of consensus_cache_entry_t by their
  449. * LABEL_VALID_AFTER labels.
  450. */
  451. static int
  452. compare_by_valid_after_(const void **a, const void **b)
  453. {
  454. const consensus_cache_entry_t *e1 = *a;
  455. const consensus_cache_entry_t *e2 = *b;
  456. /* We're in luck here: sorting UTC iso-encoded values lexically will work
  457. * fine (until 9999). */
  458. return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
  459. consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
  460. }
  461. /**
  462. * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
  463. * entry.
  464. */
  465. static consensus_cache_entry_t *
  466. sort_and_find_most_recent(smartlist_t *lst)
  467. {
  468. smartlist_sort(lst, compare_by_valid_after_);
  469. if (smartlist_len(lst)) {
  470. return smartlist_get(lst, smartlist_len(lst) - 1);
  471. } else {
  472. return NULL;
  473. }
  474. }
  475. /** Return i such that compress_consensus_with[i] == method. Return
  476. * -1 if no such i exists. */
  477. static int
  478. consensus_compression_method_pos(compress_method_t method)
  479. {
  480. unsigned i;
  481. for (i = 0; i < n_consensus_compression_methods(); ++i) {
  482. if (compress_consensus_with[i] == method) {
  483. return i;
  484. }
  485. }
  486. return -1;
  487. }
  488. /**
  489. * If we know a consensus with the flavor <b>flavor</b> compressed with
  490. * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
  491. * for consdiffmgr_find_diff_from().
  492. */
  493. consdiff_status_t
  494. consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
  495. consensus_flavor_t flavor,
  496. compress_method_t method)
  497. {
  498. tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
  499. int pos = consensus_compression_method_pos(method);
  500. if (pos < 0) {
  501. // We don't compress consensuses with this method.
  502. return CONSDIFF_NOT_FOUND;
  503. }
  504. consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
  505. if (!handle)
  506. return CONSDIFF_NOT_FOUND;
  507. *entry_out = consensus_cache_entry_handle_get(handle);
  508. if (entry_out)
  509. return CONSDIFF_AVAILABLE;
  510. else
  511. return CONSDIFF_NOT_FOUND;
  512. }
  513. /**
  514. * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
  515. * from the source consensus with the specified digest (which must be SHA3).
  516. *
  517. * If the diff is present, store it into *<b>entry_out</b> and return
  518. * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
  519. * CONSDIFF_IN_PROGRESS.
  520. */
  521. consdiff_status_t
  522. consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
  523. consensus_flavor_t flavor,
  524. int digest_type,
  525. const uint8_t *digest,
  526. size_t digestlen,
  527. compress_method_t method)
  528. {
  529. if (BUG(digest_type != DIGEST_SHA3_256) ||
  530. BUG(digestlen != DIGEST256_LEN)) {
  531. return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
  532. }
  533. // Try to look up the entry in the hashtable.
  534. cdm_diff_t search, *ent;
  535. memset(&search, 0, sizeof(search));
  536. search.flavor = flavor;
  537. search.compress_method = method;
  538. memcpy(search.from_sha3, digest, DIGEST256_LEN);
  539. ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
  540. if (ent == NULL ||
  541. ent->cdm_diff_status == CDM_DIFF_ERROR) {
  542. return CONSDIFF_NOT_FOUND;
  543. } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
  544. return CONSDIFF_IN_PROGRESS;
  545. } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
  546. return CONSDIFF_IN_PROGRESS;
  547. }
  548. *entry_out = consensus_cache_entry_handle_get(ent->entry);
  549. return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
  550. #if 0
  551. // XXXX Remove this. I'm keeping it around for now in case we need to
  552. // XXXX debug issues in the hashtable.
  553. char hex[HEX_DIGEST256_LEN+1];
  554. base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
  555. const char *flavname = networkstatus_get_flavor_name(flavor);
  556. smartlist_t *matches = smartlist_new();
  557. consensus_cache_find_all(matches, cdm_cache_get(),
  558. LABEL_FROM_SHA3_DIGEST, hex);
  559. consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
  560. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  561. *entry_out = sort_and_find_most_recent(matches);
  562. consdiff_status_t result =
  563. (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
  564. smartlist_free(matches);
  565. return result;
  566. #endif
  567. }
  568. /**
  569. * Perform periodic cleanup tasks on the consensus diff cache. Return
  570. * the number of objects marked for deletion.
  571. */
  572. int
  573. consdiffmgr_cleanup(void)
  574. {
  575. smartlist_t *objects = smartlist_new();
  576. smartlist_t *consensuses = smartlist_new();
  577. smartlist_t *diffs = smartlist_new();
  578. int n_to_delete = 0;
  579. log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
  580. // 1. Delete any consensus or diff or anything whose valid_after is too old.
  581. const time_t valid_after_cutoff =
  582. approx_time() - 3600 * consdiff_cfg.cache_max_age_hours;
  583. consensus_cache_find_all(objects, cdm_cache_get(),
  584. NULL, NULL);
  585. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  586. const char *lv_valid_after =
  587. consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
  588. if (! lv_valid_after) {
  589. log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
  590. LABEL_VALID_AFTER);
  591. continue;
  592. }
  593. time_t valid_after = 0;
  594. if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
  595. log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
  596. "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
  597. continue;
  598. }
  599. if (valid_after < valid_after_cutoff) {
  600. log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
  601. "too old", LABEL_VALID_AFTER, lv_valid_after);
  602. consensus_cache_entry_mark_for_removal(ent);
  603. ++n_to_delete;
  604. }
  605. } SMARTLIST_FOREACH_END(ent);
  606. // 2. Delete all diffs that lead to a consensus whose valid-after is not the
  607. // latest.
  608. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  609. const char *flavname = networkstatus_get_flavor_name(flav);
  610. /* Determine the most recent consensus of this flavor */
  611. consensus_cache_find_all(consensuses, cdm_cache_get(),
  612. LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  613. consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
  614. consensus_cache_entry_t *most_recent =
  615. sort_and_find_most_recent(consensuses);
  616. if (most_recent == NULL)
  617. continue;
  618. const char *most_recent_sha3 =
  619. consensus_cache_entry_get_value(most_recent,
  620. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  621. if (BUG(most_recent_sha3 == NULL))
  622. continue; // LCOV_EXCL_LINE
  623. /* consider all such-flavored diffs, and look to see if they match. */
  624. consensus_cache_find_all(diffs, cdm_cache_get(),
  625. LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  626. consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
  627. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  628. const char *this_diff_target_sha3 =
  629. consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
  630. if (!this_diff_target_sha3)
  631. continue;
  632. if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
  633. consensus_cache_entry_mark_for_removal(diff);
  634. ++n_to_delete;
  635. }
  636. } SMARTLIST_FOREACH_END(diff);
  637. smartlist_clear(consensuses);
  638. smartlist_clear(diffs);
  639. }
  640. // 3. Delete all consensuses except the most recent that are compressed with
  641. // an un-preferred method.
  642. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  643. const char *flavname = networkstatus_get_flavor_name(flav);
  644. /* Determine the most recent consensus of this flavor */
  645. consensus_cache_find_all(consensuses, cdm_cache_get(),
  646. LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  647. consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
  648. consensus_cache_entry_t *most_recent =
  649. sort_and_find_most_recent(consensuses);
  650. if (most_recent == NULL)
  651. continue;
  652. const char *most_recent_sha3_uncompressed =
  653. consensus_cache_entry_get_value(most_recent,
  654. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  655. const char *retain_methodname = compression_method_get_name(
  656. RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
  657. if (BUG(most_recent_sha3_uncompressed == NULL))
  658. continue;
  659. SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
  660. const char *lv_sha3_uncompressed =
  661. consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
  662. if (BUG(! lv_sha3_uncompressed))
  663. continue;
  664. if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
  665. continue; // This _is_ the most recent.
  666. const char *lv_methodname =
  667. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  668. if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
  669. consensus_cache_entry_mark_for_removal(ent);
  670. ++n_to_delete;
  671. }
  672. } SMARTLIST_FOREACH_END(ent);
  673. }
  674. smartlist_free(objects);
  675. smartlist_free(consensuses);
  676. smartlist_free(diffs);
  677. // Actually remove files, if they're not used.
  678. consensus_cache_delete_pending(cdm_cache_get(), 0);
  679. return n_to_delete;
  680. }
  681. /**
  682. * Initialize the consensus diff manager and its cache, and configure
  683. * its parameters based on the latest torrc and networkstatus parameters.
  684. */
  685. void
  686. consdiffmgr_configure(const consdiff_cfg_t *cfg)
  687. {
  688. if (cfg)
  689. memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
  690. (void) cdm_cache_get();
  691. }
  692. /**
  693. * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  694. * operations that the consensus diff manager will need.
  695. */
  696. int
  697. consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
  698. {
  699. return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
  700. }
  701. /**
  702. * Scan the consensus diff manager's cache for any grossly malformed entries,
  703. * and mark them as deletable. Return 0 if no problems were found; 1
  704. * if problems were found and fixed.
  705. */
  706. int
  707. consdiffmgr_validate(void)
  708. {
  709. /* Right now, we only check for entries that have bad sha3 values */
  710. int problems = 0;
  711. smartlist_t *objects = smartlist_new();
  712. consensus_cache_find_all(objects, cdm_cache_get(),
  713. NULL, NULL);
  714. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
  715. uint8_t sha3_expected[DIGEST256_LEN];
  716. uint8_t sha3_received[DIGEST256_LEN];
  717. int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
  718. if (r == -1) {
  719. /* digest isn't there; that's allowed */
  720. continue;
  721. } else if (r == -2) {
  722. /* digest is malformed; that's not allowed */
  723. problems = 1;
  724. consensus_cache_entry_mark_for_removal(obj);
  725. continue;
  726. }
  727. const uint8_t *body;
  728. size_t bodylen;
  729. consensus_cache_entry_incref(obj);
  730. r = consensus_cache_entry_get_body(obj, &body, &bodylen);
  731. if (r == 0) {
  732. crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
  733. DIGEST_SHA3_256);
  734. }
  735. consensus_cache_entry_decref(obj);
  736. if (r < 0)
  737. continue;
  738. // Deconfuse coverity about the possibility of sha3_received being
  739. // uninitialized
  740. tor_assert(r <= 0);
  741. if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
  742. problems = 1;
  743. consensus_cache_entry_mark_for_removal(obj);
  744. continue;
  745. }
  746. } SMARTLIST_FOREACH_END(obj);
  747. smartlist_free(objects);
  748. return problems;
  749. }
  750. /**
  751. * Helper: build new diffs of <b>flavor</b> as needed
  752. */
  753. static void
  754. consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
  755. {
  756. smartlist_t *matches = NULL;
  757. smartlist_t *diffs = NULL;
  758. smartlist_t *compute_diffs_from = NULL;
  759. strmap_t *have_diff_from = NULL;
  760. // look for the most recent consensus, and for all previous in-range
  761. // consensuses. Do they all have diffs to it?
  762. const char *flavname = networkstatus_get_flavor_name(flavor);
  763. // 1. find the most recent consensus, and the ones that we might want
  764. // to diff to it.
  765. const char *methodname = compression_method_get_name(
  766. RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
  767. matches = smartlist_new();
  768. consensus_cache_find_all(matches, cdm_cache_get(),
  769. LABEL_FLAVOR, flavname);
  770. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  771. consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
  772. consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
  773. if (!most_recent) {
  774. log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
  775. "not making diffs", flavname);
  776. goto done;
  777. }
  778. tor_assert(smartlist_len(matches));
  779. smartlist_del(matches, smartlist_len(matches) - 1);
  780. const char *most_recent_valid_after =
  781. consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
  782. if (BUG(most_recent_valid_after == NULL))
  783. goto done; //LCOV_EXCL_LINE
  784. uint8_t most_recent_sha3[DIGEST256_LEN];
  785. if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
  786. LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
  787. goto done; //LCOV_EXCL_LINE
  788. // 2. Find all the relevant diffs _to_ this consensus. These are ones
  789. // that we don't need to compute.
  790. diffs = smartlist_new();
  791. consensus_cache_find_all(diffs, cdm_cache_get(),
  792. LABEL_VALID_AFTER, most_recent_valid_after);
  793. consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  794. consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
  795. have_diff_from = strmap_new();
  796. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  797. const char *va = consensus_cache_entry_get_value(diff,
  798. LABEL_FROM_VALID_AFTER);
  799. if (BUG(va == NULL))
  800. continue; // LCOV_EXCL_LINE
  801. strmap_set(have_diff_from, va, diff);
  802. } SMARTLIST_FOREACH_END(diff);
  803. // 3. See which consensuses in 'matches' don't have diffs yet.
  804. smartlist_reverse(matches); // from newest to oldest.
  805. compute_diffs_from = smartlist_new();
  806. SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
  807. const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
  808. if (BUG(va == NULL))
  809. continue; // LCOV_EXCL_LINE
  810. if (strmap_get(have_diff_from, va) != NULL)
  811. continue; /* we already have this one. */
  812. smartlist_add(compute_diffs_from, ent);
  813. /* Since we are not going to serve this as the most recent consensus
  814. * any more, we should stop keeping it mmap'd when it's not in use.
  815. */
  816. consensus_cache_entry_mark_for_aggressive_release(ent);
  817. } SMARTLIST_FOREACH_END(ent);
  818. log_info(LD_DIRSERV,
  819. "The most recent %s consensus is valid-after %s. We have diffs to "
  820. "this consensus for %d/%d older %s consensuses. Generating diffs "
  821. "for the other %d.",
  822. flavname,
  823. most_recent_valid_after,
  824. smartlist_len(matches) - smartlist_len(compute_diffs_from),
  825. smartlist_len(matches),
  826. flavname,
  827. smartlist_len(compute_diffs_from));
  828. // 4. Update the hashtable; remove entries in this flavor to other
  829. // target consensuses.
  830. cdm_diff_ht_purge(flavor, most_recent_sha3);
  831. // 5. Actually launch the requests.
  832. SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
  833. if (BUG(c == most_recent))
  834. continue; // LCOV_EXCL_LINE
  835. uint8_t this_sha3[DIGEST256_LEN];
  836. if (cdm_entry_get_sha3_value(this_sha3, c,
  837. LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
  838. // Not actually a bug, since we might be running with a directory
  839. // with stale files from before the #22143 fixes.
  840. continue;
  841. }
  842. if (cdm_diff_ht_check_and_note_pending(flavor,
  843. this_sha3, most_recent_sha3)) {
  844. // This is already pending, or we encountered an error.
  845. continue;
  846. }
  847. consensus_diff_queue_diff_work(c, most_recent);
  848. } SMARTLIST_FOREACH_END(c);
  849. done:
  850. smartlist_free(matches);
  851. smartlist_free(diffs);
  852. smartlist_free(compute_diffs_from);
  853. strmap_free(have_diff_from, NULL);
  854. }
  855. /**
  856. * Scan the cache for the latest consensuses and add their handles to
  857. * latest_consensus
  858. */
  859. static void
  860. consdiffmgr_consensus_load(void)
  861. {
  862. smartlist_t *matches = smartlist_new();
  863. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  864. const char *flavname = networkstatus_get_flavor_name(flav);
  865. smartlist_clear(matches);
  866. consensus_cache_find_all(matches, cdm_cache_get(),
  867. LABEL_FLAVOR, flavname);
  868. consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  869. consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
  870. if (! most_recent)
  871. continue; // no consensuses.
  872. const char *most_recent_sha3 =
  873. consensus_cache_entry_get_value(most_recent,
  874. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  875. if (BUG(most_recent_sha3 == NULL))
  876. continue; // LCOV_EXCL_LINE
  877. consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
  878. most_recent_sha3);
  879. // Everything that remains matches the most recent consensus of this
  880. // flavor.
  881. SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
  882. const char *lv_compression =
  883. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  884. compress_method_t method =
  885. compression_method_get_by_name(lv_compression);
  886. int pos = consensus_compression_method_pos(method);
  887. if (pos < 0)
  888. continue;
  889. consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
  890. latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
  891. } SMARTLIST_FOREACH_END(ent);
  892. }
  893. smartlist_free(matches);
  894. }
  895. /**
  896. * Scan the cache for diffs, and add them to the hashtable.
  897. */
  898. static void
  899. consdiffmgr_diffs_load(void)
  900. {
  901. smartlist_t *diffs = smartlist_new();
  902. consensus_cache_find_all(diffs, cdm_cache_get(),
  903. LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
  904. SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
  905. const char *lv_flavor =
  906. consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
  907. if (!lv_flavor)
  908. continue;
  909. int flavor = networkstatus_parse_flavor_name(lv_flavor);
  910. if (flavor < 0)
  911. continue;
  912. const char *lv_compression =
  913. consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
  914. compress_method_t method = NO_METHOD;
  915. if (lv_compression) {
  916. method = compression_method_get_by_name(lv_compression);
  917. if (method == UNKNOWN_METHOD) {
  918. continue;
  919. }
  920. }
  921. uint8_t from_sha3[DIGEST256_LEN];
  922. uint8_t to_sha3[DIGEST256_LEN];
  923. if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
  924. continue;
  925. if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
  926. continue;
  927. cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
  928. method,
  929. CDM_DIFF_PRESENT,
  930. consensus_cache_entry_handle_new(diff));
  931. } SMARTLIST_FOREACH_END(diff);
  932. smartlist_free(diffs);
  933. }
  934. /**
  935. * Build new diffs as needed.
  936. */
  937. void
  938. consdiffmgr_rescan(void)
  939. {
  940. if (cdm_cache_dirty == 0)
  941. return;
  942. // Clean up here to make room for new diffs, and to ensure that older
  943. // consensuses do not have any entries.
  944. consdiffmgr_cleanup();
  945. if (cdm_cache_loaded == 0) {
  946. consdiffmgr_diffs_load();
  947. consdiffmgr_consensus_load();
  948. cdm_cache_loaded = 1;
  949. }
  950. for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
  951. consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
  952. }
  953. cdm_cache_dirty = 0;
  954. }
  955. /**
  956. * Helper: compare two files by their from-valid-after and valid-after labels,
  957. * trying to sort in ascending order by from-valid-after (when present) and
  958. * valid-after (when not). Place everything that has neither label first in
  959. * the list.
  960. */
  961. static int
  962. compare_by_staleness_(const void **a, const void **b)
  963. {
  964. const consensus_cache_entry_t *e1 = *a;
  965. const consensus_cache_entry_t *e2 = *b;
  966. const char *va1, *fva1, *va2, *fva2;
  967. va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
  968. va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
  969. fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
  970. fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
  971. if (fva1)
  972. va1 = fva1;
  973. if (fva2)
  974. va2 = fva2;
  975. /* See note about iso-encoded values in compare_by_valid_after_. Also note
  976. * that missing dates will get placed first. */
  977. return strcmp_opt(va1, va2);
  978. }
  979. /** If there are not enough unused filenames to store <b>n</b> files, then
  980. * delete old consensuses until there are. (We have to keep track of the
  981. * number of filenames because of the way that the seccomp2 cache works.)
  982. *
  983. * Return 0 on success, -1 on failure.
  984. **/
  985. static int
  986. consdiffmgr_ensure_space_for_files(int n)
  987. {
  988. consensus_cache_t *cache = cdm_cache_get();
  989. if (consensus_cache_get_n_filenames_available(cache) >= n) {
  990. // there are already enough unused filenames.
  991. return 0;
  992. }
  993. // Try a cheap deletion of stuff that's waiting to get deleted.
  994. consensus_cache_delete_pending(cache, 0);
  995. if (consensus_cache_get_n_filenames_available(cache) >= n) {
  996. // okay, _that_ made enough filenames available.
  997. return 0;
  998. }
  999. // Let's get more assertive: clean out unused stuff, and force-remove
  1000. // the files.
  1001. consdiffmgr_cleanup();
  1002. consensus_cache_delete_pending(cache, 1);
  1003. const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
  1004. if (n_to_remove <= 0) {
  1005. // okay, finally!
  1006. return 0;
  1007. }
  1008. // At this point, we're going to have to throw out objects that will be
  1009. // missed. Too bad!
  1010. smartlist_t *objects = smartlist_new();
  1011. consensus_cache_find_all(objects, cache, NULL, NULL);
  1012. smartlist_sort(objects, compare_by_staleness_);
  1013. int n_marked = 0;
  1014. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  1015. consensus_cache_entry_mark_for_removal(ent);
  1016. if (++n_marked >= n_to_remove)
  1017. break;
  1018. } SMARTLIST_FOREACH_END(ent);
  1019. consensus_cache_delete_pending(cache, 1);
  1020. if (BUG(n_marked < n_to_remove))
  1021. return -1;
  1022. else
  1023. return 0;
  1024. }
  1025. /**
  1026. * Set consensus cache flags on the objects in this consdiffmgr.
  1027. */
  1028. static void
  1029. consdiffmgr_set_cache_flags(void)
  1030. {
  1031. /* Right now, we just mark the consensus objects for aggressive release,
  1032. * so that they get mmapped for as little time as possible. */
  1033. smartlist_t *objects = smartlist_new();
  1034. consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
  1035. DOCTYPE_CONSENSUS);
  1036. SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
  1037. consensus_cache_entry_mark_for_aggressive_release(ent);
  1038. } SMARTLIST_FOREACH_END(ent);
  1039. smartlist_free(objects);
  1040. }
  1041. /**
  1042. * Called before shutdown: drop all storage held by the consdiffmgr.c module.
  1043. */
  1044. void
  1045. consdiffmgr_free_all(void)
  1046. {
  1047. cdm_diff_t **diff, **next;
  1048. for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
  1049. cdm_diff_t *this = *diff;
  1050. next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
  1051. cdm_diff_free(this);
  1052. }
  1053. int i;
  1054. unsigned j;
  1055. for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
  1056. for (j = 0; j < n_consensus_compression_methods(); ++j) {
  1057. consensus_cache_entry_handle_free(latest_consensus[i][j]);
  1058. }
  1059. }
  1060. memset(latest_consensus, 0, sizeof(latest_consensus));
  1061. consensus_cache_free(cons_diff_cache);
  1062. cons_diff_cache = NULL;
  1063. }
  1064. /* =====
  1065. Thread workers
  1066. =====*/
  1067. typedef struct compressed_result_t {
  1068. config_line_t *labels;
  1069. /**
  1070. * Output: Body of the diff, as compressed.
  1071. */
  1072. uint8_t *body;
  1073. /**
  1074. * Output: length of body_out
  1075. */
  1076. size_t bodylen;
  1077. } compressed_result_t;
  1078. /**
  1079. * Compress the bytestring <b>input</b> of length <b>len</b> using the
  1080. * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
  1081. *
  1082. * For each successful compression, set the fields in the <b>results_out</b>
  1083. * array in the position corresponding to the compression method. Use
  1084. * <b>labels_in</b> as a basis for the labels of the result.
  1085. *
  1086. * Return 0 if all compression succeeded; -1 if any failed.
  1087. */
  1088. static int
  1089. compress_multiple(compressed_result_t *results_out, int n_methods,
  1090. const compress_method_t *methods,
  1091. const uint8_t *input, size_t len,
  1092. const config_line_t *labels_in)
  1093. {
  1094. int rv = 0;
  1095. int i;
  1096. for (i = 0; i < n_methods; ++i) {
  1097. compress_method_t method = methods[i];
  1098. const char *methodname = compression_method_get_name(method);
  1099. char *result;
  1100. size_t sz;
  1101. if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
  1102. results_out[i].body = (uint8_t*)result;
  1103. results_out[i].bodylen = sz;
  1104. results_out[i].labels = config_lines_dup(labels_in);
  1105. cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
  1106. results_out[i].body,
  1107. results_out[i].bodylen);
  1108. config_line_prepend(&results_out[i].labels,
  1109. LABEL_COMPRESSION_TYPE,
  1110. methodname);
  1111. } else {
  1112. rv = -1;
  1113. }
  1114. }
  1115. return rv;
  1116. }
  1117. /**
  1118. * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
  1119. * as produced by compress_multiple, store them all into the
  1120. * consdiffmgr, and store handles to them in the <b>handles_out</b>
  1121. * array.
  1122. *
  1123. * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
  1124. * was stored.
  1125. */
  1126. static cdm_diff_status_t
  1127. store_multiple(consensus_cache_entry_handle_t **handles_out,
  1128. int n,
  1129. const compress_method_t *methods,
  1130. const compressed_result_t *results,
  1131. const char *description)
  1132. {
  1133. cdm_diff_status_t status = CDM_DIFF_ERROR;
  1134. consdiffmgr_ensure_space_for_files(n);
  1135. int i;
  1136. for (i = 0; i < n; ++i) {
  1137. compress_method_t method = methods[i];
  1138. uint8_t *body_out = results[i].body;
  1139. size_t bodylen_out = results[i].bodylen;
  1140. config_line_t *labels = results[i].labels;
  1141. const char *methodname = compression_method_get_name(method);
  1142. if (body_out && bodylen_out && labels) {
  1143. /* Success! Store the results */
  1144. log_info(LD_DIRSERV, "Adding %s, compressed with %s",
  1145. description, methodname);
  1146. consensus_cache_entry_t *ent =
  1147. consensus_cache_add(cdm_cache_get(),
  1148. labels,
  1149. body_out,
  1150. bodylen_out);
  1151. if (BUG(ent == NULL))
  1152. continue;
  1153. status = CDM_DIFF_PRESENT;
  1154. handles_out[i] = consensus_cache_entry_handle_new(ent);
  1155. consensus_cache_entry_decref(ent);
  1156. }
  1157. }
  1158. return status;
  1159. }
  1160. /**
  1161. * An object passed to a worker thread that will try to produce a consensus
  1162. * diff.
  1163. */
  1164. typedef struct consensus_diff_worker_job_t {
  1165. /**
  1166. * Input: The consensus to compute the diff from. Holds a reference to the
  1167. * cache entry, which must not be released until the job is passed back to
  1168. * the main thread. The body must be mapped into memory in the main thread.
  1169. */
  1170. consensus_cache_entry_t *diff_from;
  1171. /**
  1172. * Input: The consensus to compute the diff to. Holds a reference to the
  1173. * cache entry, which must not be released until the job is passed back to
  1174. * the main thread. The body must be mapped into memory in the main thread.
  1175. */
  1176. consensus_cache_entry_t *diff_to;
  1177. /** Output: labels and bodies */
  1178. compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
  1179. } consensus_diff_worker_job_t;
  1180. /** Given a consensus_cache_entry_t, check whether it has a label claiming
  1181. * that it was compressed. If so, uncompress its contents into <b>out</b> and
  1182. * set <b>outlen</b> to hold their size. If not, just copy the body into
  1183. * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success,
  1184. * -1 on failure.
  1185. *
  1186. * In all cases, the output is nul-terminated. */
  1187. STATIC int
  1188. uncompress_or_copy(char **out, size_t *outlen,
  1189. consensus_cache_entry_t *ent)
  1190. {
  1191. const uint8_t *body;
  1192. size_t bodylen;
  1193. if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
  1194. return -1;
  1195. const char *lv_compression =
  1196. consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
  1197. compress_method_t method = NO_METHOD;
  1198. if (lv_compression)
  1199. method = compression_method_get_by_name(lv_compression);
  1200. return tor_uncompress(out, outlen, (const char *)body, bodylen,
  1201. method, 1, LOG_WARN);
  1202. }
  1203. /**
  1204. * Worker function. This function runs inside a worker thread and receives
  1205. * a consensus_diff_worker_job_t as its input.
  1206. */
  1207. static workqueue_reply_t
  1208. consensus_diff_worker_threadfn(void *state_, void *work_)
  1209. {
  1210. (void)state_;
  1211. consensus_diff_worker_job_t *job = work_;
  1212. const uint8_t *diff_from, *diff_to;
  1213. size_t len_from, len_to;
  1214. int r;
  1215. /* We need to have the body already mapped into RAM here.
  1216. */
  1217. r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
  1218. if (BUG(r < 0))
  1219. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1220. r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
  1221. if (BUG(r < 0))
  1222. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1223. const char *lv_to_valid_after =
  1224. consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
  1225. const char *lv_from_valid_after =
  1226. consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
  1227. const char *lv_from_digest =
  1228. consensus_cache_entry_get_value(job->diff_from,
  1229. LABEL_SHA3_DIGEST_AS_SIGNED);
  1230. const char *lv_from_flavor =
  1231. consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
  1232. const char *lv_to_flavor =
  1233. consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
  1234. const char *lv_to_digest =
  1235. consensus_cache_entry_get_value(job->diff_to,
  1236. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  1237. if (! lv_from_digest) {
  1238. /* This isn't a bug right now, since it can happen if you're migrating
  1239. * from an older version of master to a newer one. The older ones didn't
  1240. * annotate their stored consensus objects with sha3-digest-as-signed.
  1241. */
  1242. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1243. }
  1244. /* All these values are mandatory on the input */
  1245. if (BUG(!lv_to_valid_after) ||
  1246. BUG(!lv_from_valid_after) ||
  1247. BUG(!lv_from_flavor) ||
  1248. BUG(!lv_to_flavor)) {
  1249. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1250. }
  1251. /* The flavors need to match */
  1252. if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
  1253. return WQ_RPL_REPLY; // LCOV_EXCL_LINE
  1254. }
  1255. char *consensus_diff;
  1256. {
  1257. char *diff_from_nt = NULL, *diff_to_nt = NULL;
  1258. size_t diff_from_nt_len, diff_to_nt_len;
  1259. if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
  1260. job->diff_from) < 0) {
  1261. return WQ_RPL_REPLY;
  1262. }
  1263. if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
  1264. job->diff_to) < 0) {
  1265. tor_free(diff_from_nt);
  1266. return WQ_RPL_REPLY;
  1267. }
  1268. tor_assert(diff_from_nt);
  1269. tor_assert(diff_to_nt);
  1270. // XXXX ugh; this is going to calculate the SHA3 of both its
  1271. // XXXX inputs again, even though we already have that. Maybe it's time
  1272. // XXXX to change the API here?
  1273. consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
  1274. tor_free(diff_from_nt);
  1275. tor_free(diff_to_nt);
  1276. }
  1277. if (!consensus_diff) {
  1278. /* Couldn't generate consensus; we'll leave the reply blank. */
  1279. return WQ_RPL_REPLY;
  1280. }
  1281. /* Compress the results and send the reply */
  1282. tor_assert(compress_diffs_with[0] == NO_METHOD);
  1283. size_t difflen = strlen(consensus_diff);
  1284. job->out[0].body = (uint8_t *) consensus_diff;
  1285. job->out[0].bodylen = difflen;
  1286. config_line_t *common_labels = NULL;
  1287. cdm_labels_prepend_sha3(&common_labels,
  1288. LABEL_SHA3_DIGEST_UNCOMPRESSED,
  1289. job->out[0].body,
  1290. job->out[0].bodylen);
  1291. config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
  1292. lv_from_valid_after);
  1293. config_line_prepend(&common_labels, LABEL_VALID_AFTER,
  1294. lv_to_valid_after);
  1295. config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
  1296. config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
  1297. lv_from_digest);
  1298. config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
  1299. lv_to_digest);
  1300. config_line_prepend(&common_labels, LABEL_DOCTYPE,
  1301. DOCTYPE_CONSENSUS_DIFF);
  1302. job->out[0].labels = config_lines_dup(common_labels);
  1303. cdm_labels_prepend_sha3(&job->out[0].labels,
  1304. LABEL_SHA3_DIGEST,
  1305. job->out[0].body,
  1306. job->out[0].bodylen);
  1307. compress_multiple(job->out+1,
  1308. n_diff_compression_methods()-1,
  1309. compress_diffs_with+1,
  1310. (const uint8_t*)consensus_diff, difflen, common_labels);
  1311. config_free_lines(common_labels);
  1312. return WQ_RPL_REPLY;
  1313. }
  1314. /**
  1315. * Helper: release all storage held in <b>job</b>.
  1316. */
  1317. static void
  1318. consensus_diff_worker_job_free(consensus_diff_worker_job_t *job)
  1319. {
  1320. if (!job)
  1321. return;
  1322. unsigned u;
  1323. for (u = 0; u < n_diff_compression_methods(); ++u) {
  1324. config_free_lines(job->out[u].labels);
  1325. tor_free(job->out[u].body);
  1326. }
  1327. consensus_cache_entry_decref(job->diff_from);
  1328. consensus_cache_entry_decref(job->diff_to);
  1329. tor_free(job);
  1330. }
  1331. /**
  1332. * Worker function: This function runs in the main thread, and receives
  1333. * a consensus_diff_worker_job_t that the worker thread has already
  1334. * processed.
  1335. */
  1336. static void
  1337. consensus_diff_worker_replyfn(void *work_)
  1338. {
  1339. tor_assert(in_main_thread());
  1340. tor_assert(work_);
  1341. consensus_diff_worker_job_t *job = work_;
  1342. const char *lv_from_digest =
  1343. consensus_cache_entry_get_value(job->diff_from,
  1344. LABEL_SHA3_DIGEST_AS_SIGNED);
  1345. const char *lv_to_digest =
  1346. consensus_cache_entry_get_value(job->diff_to,
  1347. LABEL_SHA3_DIGEST_UNCOMPRESSED);
  1348. const char *lv_flavor =
  1349. consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
  1350. if (BUG(lv_from_digest == NULL))
  1351. lv_from_digest = "???"; // LCOV_EXCL_LINE
  1352. if (BUG(lv_to_digest == NULL))
  1353. lv_to_digest = "???"; // LCOV_EXCL_LINE
  1354. uint8_t from_sha3[DIGEST256_LEN];
  1355. uint8_t to_sha3[DIGEST256_LEN];
  1356. int flav = -1;
  1357. int cache = 1;
  1358. if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
  1359. LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
  1360. cache = 0;
  1361. if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
  1362. LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
  1363. cache = 0;
  1364. if (BUG(lv_flavor == NULL)) {
  1365. cache = 0;
  1366. } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
  1367. cache = 0;
  1368. }
  1369. consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
  1370. memset(handles, 0, sizeof(handles));
  1371. char description[128];
  1372. tor_snprintf(description, sizeof(description),
  1373. "consensus diff from %s to %s",
  1374. lv_from_digest, lv_to_digest);
  1375. int status = store_multiple(handles,
  1376. n_diff_compression_methods(),
  1377. compress_diffs_with,
  1378. job->out,
  1379. description);
  1380. if (status != CDM_DIFF_PRESENT) {
  1381. /* Failure! Nothing to do but complain */
  1382. log_warn(LD_DIRSERV,
  1383. "Worker was unable to compute consensus diff "
  1384. "from %s to %s", lv_from_digest, lv_to_digest);
  1385. /* Cache this error so we don't try to compute this one again. */
  1386. status = CDM_DIFF_ERROR;
  1387. }
  1388. unsigned u;
  1389. for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
  1390. compress_method_t method = compress_diffs_with[u];
  1391. if (cache) {
  1392. cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, status,
  1393. handles[u]);
  1394. } else {
  1395. consensus_cache_entry_handle_free(handles[u]);
  1396. }
  1397. }
  1398. consensus_diff_worker_job_free(job);
  1399. }
  1400. /**
  1401. * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
  1402. * in a worker thread.
  1403. */
  1404. static int
  1405. consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
  1406. consensus_cache_entry_t *diff_to)
  1407. {
  1408. tor_assert(in_main_thread());
  1409. consensus_cache_entry_incref(diff_from);
  1410. consensus_cache_entry_incref(diff_to);
  1411. consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
  1412. job->diff_from = diff_from;
  1413. job->diff_to = diff_to;
  1414. /* Make sure body is mapped. */
  1415. const uint8_t *body;
  1416. size_t bodylen;
  1417. int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
  1418. int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
  1419. if (r1 < 0 || r2 < 0)
  1420. goto err;
  1421. workqueue_entry_t *work;
  1422. work = cpuworker_queue_work(consensus_diff_worker_threadfn,
  1423. consensus_diff_worker_replyfn,
  1424. job);
  1425. if (!work)
  1426. goto err;
  1427. return 0;
  1428. err:
  1429. consensus_diff_worker_job_free(job); // includes decrefs.
  1430. return -1;
  1431. }
  1432. /**
  1433. * Holds requests and replies for consensus_compress_workers.
  1434. */
  1435. typedef struct consensus_compress_worker_job_t {
  1436. char *consensus;
  1437. size_t consensus_len;
  1438. consensus_flavor_t flavor;
  1439. time_t valid_after;
  1440. compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
  1441. } consensus_compress_worker_job_t;
  1442. /**
  1443. * Free all resources held in <b>job</b>
  1444. */
  1445. static void
  1446. consensus_compress_worker_job_free(consensus_compress_worker_job_t *job)
  1447. {
  1448. if (!job)
  1449. return;
  1450. tor_free(job->consensus);
  1451. unsigned u;
  1452. for (u = 0; u < n_consensus_compression_methods(); ++u) {
  1453. config_free_lines(job->out[u].labels);
  1454. tor_free(job->out[u].body);
  1455. }
  1456. tor_free(job);
  1457. }
  1458. /**
  1459. * Worker function. This function runs inside a worker thread and receives
  1460. * a consensus_compress_worker_job_t as its input.
  1461. */
  1462. static workqueue_reply_t
  1463. consensus_compress_worker_threadfn(void *state_, void *work_)
  1464. {
  1465. (void)state_;
  1466. consensus_compress_worker_job_t *job = work_;
  1467. consensus_flavor_t flavor = job->flavor;
  1468. const char *consensus = job->consensus;
  1469. size_t bodylen = job->consensus_len;
  1470. time_t valid_after = job->valid_after;
  1471. config_line_t *labels = NULL;
  1472. char formatted_time[ISO_TIME_LEN+1];
  1473. format_iso_time_nospace(formatted_time, valid_after);
  1474. const char *flavname = networkstatus_get_flavor_name(flavor);
  1475. cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
  1476. (const uint8_t *)consensus, bodylen);
  1477. {
  1478. const char *start, *end;
  1479. if (router_get_networkstatus_v3_signed_boundaries(consensus,
  1480. &start, &end) < 0) {
  1481. start = consensus;
  1482. end = consensus+bodylen;
  1483. }
  1484. cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
  1485. (const uint8_t *)start,
  1486. end - start);
  1487. }
  1488. config_line_prepend(&labels, LABEL_FLAVOR, flavname);
  1489. config_line_prepend(&labels, LABEL_VALID_AFTER, formatted_time);
  1490. config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
  1491. compress_multiple(job->out,
  1492. n_consensus_compression_methods(),
  1493. compress_consensus_with,
  1494. (const uint8_t*)consensus, bodylen, labels);
  1495. config_free_lines(labels);
  1496. return WQ_RPL_REPLY;
  1497. }
  1498. /**
  1499. * Worker function: This function runs in the main thread, and receives
  1500. * a consensus_diff_compress_job_t that the worker thread has already
  1501. * processed.
  1502. */
  1503. static void
  1504. consensus_compress_worker_replyfn(void *work_)
  1505. {
  1506. consensus_compress_worker_job_t *job = work_;
  1507. consensus_cache_entry_handle_t *handles[
  1508. ARRAY_LENGTH(compress_consensus_with)];
  1509. memset(handles, 0, sizeof(handles));
  1510. store_multiple(handles,
  1511. n_consensus_compression_methods(),
  1512. compress_consensus_with,
  1513. job->out,
  1514. "consensus");
  1515. cdm_cache_dirty = 1;
  1516. unsigned u;
  1517. consensus_flavor_t f = job->flavor;
  1518. tor_assert((int)f < N_CONSENSUS_FLAVORS);
  1519. for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
  1520. if (handles[u] == NULL)
  1521. continue;
  1522. consensus_cache_entry_handle_free(latest_consensus[f][u]);
  1523. latest_consensus[f][u] = handles[u];
  1524. }
  1525. consensus_compress_worker_job_free(job);
  1526. }
  1527. /**
  1528. * If true, we compress in worker threads.
  1529. */
  1530. static int background_compression = 0;
  1531. /**
  1532. * Queue a job to compress <b>consensus</b> and store its compressed
  1533. * text in the cache.
  1534. */
  1535. static int
  1536. consensus_queue_compression_work(const char *consensus,
  1537. consensus_flavor_t flavor,
  1538. time_t valid_after)
  1539. {
  1540. consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
  1541. job->consensus = tor_strdup(consensus);
  1542. job->consensus_len = strlen(consensus);
  1543. job->flavor = flavor;
  1544. job->valid_after = valid_after;
  1545. if (background_compression) {
  1546. workqueue_entry_t *work;
  1547. work = cpuworker_queue_work(consensus_compress_worker_threadfn,
  1548. consensus_compress_worker_replyfn,
  1549. job);
  1550. if (!work) {
  1551. consensus_compress_worker_job_free(job);
  1552. return -1;
  1553. }
  1554. return 0;
  1555. } else {
  1556. consensus_compress_worker_threadfn(NULL, job);
  1557. consensus_compress_worker_replyfn(job);
  1558. return 0;
  1559. }
  1560. }
  1561. /**
  1562. * Tell the consdiffmgr backend to compress consensuses in worker threads.
  1563. */
  1564. void
  1565. consdiffmgr_enable_background_compression(void)
  1566. {
  1567. // This isn't the default behavior because it would break unit tests.
  1568. background_compression = 1;
  1569. }
  1570. /** Read the set of voters from the cached object <b>ent</b> into
  1571. * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
  1572. * -1 if no signatories were recorded. */
  1573. int
  1574. consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
  1575. smartlist_t *out)
  1576. {
  1577. tor_assert(ent);
  1578. tor_assert(out);
  1579. const char *s;
  1580. s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
  1581. if (s == NULL)
  1582. return -1;
  1583. smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
  1584. return 0;
  1585. }
  1586. /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
  1587. * and return 0, or return -1 if no such time was recorded. */
  1588. int
  1589. consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
  1590. time_t *out)
  1591. {
  1592. tor_assert(ent);
  1593. tor_assert(out);
  1594. const char *s;
  1595. s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
  1596. if (s == NULL || parse_iso_time_nospace(s, out) < 0)
  1597. return -1;
  1598. else
  1599. return 0;
  1600. }
  1601. /** Read the valid until timestamp from the cached object <b>ent</b> into
  1602. * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
  1603. int
  1604. consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
  1605. time_t *out)
  1606. {
  1607. tor_assert(ent);
  1608. tor_assert(out);
  1609. const char *s;
  1610. s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
  1611. if (s == NULL || parse_iso_time_nospace(s, out) < 0)
  1612. return -1;
  1613. else
  1614. return 0;
  1615. }