123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911 |
- /* Copyright (c) 2017, The Tor Project, Inc. */
- /* See LICENSE for licensing information */
- /**
- * \file consdiffmsr.c
- *
- * \brief consensus diff manager functions
- *
- * This module is run by directory authorities and caches in order
- * to remember a number of past consensus documents, and to generate
- * and serve the diffs from those documents to the latest consensus.
- */
- #define CONSDIFFMGR_PRIVATE
- #include "or.h"
- #include "config.h"
- #include "conscache.h"
- #include "consdiff.h"
- #include "consdiffmgr.h"
- #include "cpuworker.h"
- #include "networkstatus.h"
- #include "routerparse.h"
- #include "workqueue.h"
- /**
- * Labels to apply to items in the conscache object.
- *
- * @{
- */
- /* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
- #define LABEL_DOCTYPE "document-type"
- /* The valid-after time for a consensus (or for the target consensus of a
- * diff), encoded as ISO UTC. */
- #define LABEL_VALID_AFTER "consensus-valid-after"
- /* The fresh-until time for a consensus (or for the target consensus of a
- * diff), encoded as ISO UTC. */
- #define LABEL_FRESH_UNTIL "consensus-fresh-until"
- /* The valid-until time for a consensus (or for the target consensus of a
- * diff), encoded as ISO UTC. */
- #define LABEL_VALID_UNTIL "consensus-valid-until"
- /* Comma-separated list of hex-encoded identity digests for the voting
- * authorities. */
- #define LABEL_SIGNATORIES "consensus-signatories"
- /* A hex encoded SHA3 digest of the object, as compressed (if any) */
- #define LABEL_SHA3_DIGEST "sha3-digest"
- /* A hex encoded SHA3 digest of the object before compression. */
- #define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
- /* A hex encoded SHA3 digest-as-signed of a consensus */
- #define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
- /* The flavor of the consensus or consensuses diff */
- #define LABEL_FLAVOR "consensus-flavor"
- /* Diff only: the SHA3 digest-as-signed of the source consensus. */
- #define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
- /* Diff only: the SHA3 digest-in-full of the target consensus. */
- #define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
- /* Diff only: the valid-after date of the source consensus. */
- #define LABEL_FROM_VALID_AFTER "from-valid-after"
- /* What kind of compression was used? */
- #define LABEL_COMPRESSION_TYPE "compression"
- /** @} */
- #define DOCTYPE_CONSENSUS "consensus"
- #define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
- /**
- * Underlying directory that stores consensuses and consensus diffs. Don't
- * use this directly: use cdm_cache_get() instead.
- */
- static consensus_cache_t *cons_diff_cache = NULL;
- /**
- * If true, we have learned at least one new consensus since the
- * consensus cache was last up-to-date.
- */
- static int cdm_cache_dirty = 0;
- /**
- * If true, we have scanned the cache to update our hashtable of diffs.
- */
- static int cdm_cache_loaded = 0;
- /**
- * Possible status values for cdm_diff_t.cdm_diff_status
- **/
- typedef enum cdm_diff_status_t {
- CDM_DIFF_PRESENT=1,
- CDM_DIFF_IN_PROGRESS=2,
- CDM_DIFF_ERROR=3,
- } cdm_diff_status_t;
- /** Which methods do we use for precompressing diffs? */
- static const compress_method_t compress_diffs_with[] = {
- NO_METHOD,
- GZIP_METHOD,
- #ifdef HAVE_LZMA
- LZMA_METHOD,
- #endif
- #ifdef HAVE_ZSTD
- ZSTD_METHOD,
- #endif
- };
- /** How many different methods will we try to use for diff compression? */
- STATIC unsigned
- n_diff_compression_methods(void)
- {
- return ARRAY_LENGTH(compress_diffs_with);
- }
- /** Which methods do we use for precompressing consensuses? */
- static const compress_method_t compress_consensus_with[] = {
- ZLIB_METHOD,
- #ifdef HAVE_LZMA
- LZMA_METHOD,
- #endif
- #ifdef HAVE_ZSTD
- ZSTD_METHOD,
- #endif
- };
- /** How many different methods will we try to use for diff compression? */
- STATIC unsigned
- n_consensus_compression_methods(void)
- {
- return ARRAY_LENGTH(compress_consensus_with);
- }
- /** For which compression method do we retain old consensuses? There's no
- * need to keep all of them, since we won't be serving them. We'll
- * go with ZLIB_METHOD because it's pretty fast and everyone has it.
- */
- #define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
- /** Handles pointing to the latest consensus entries as compressed and
- * stored. */
- static consensus_cache_entry_handle_t *
- latest_consensus[N_CONSENSUS_FLAVORS]
- [ARRAY_LENGTH(compress_consensus_with)];
- /** Hashtable node used to remember the current status of the diff
- * from a given sha3 digest to the current consensus. */
- typedef struct cdm_diff_t {
- HT_ENTRY(cdm_diff_t) node;
- /** Consensus flavor for this diff (part of ht key) */
- consensus_flavor_t flavor;
- /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
- * ht key) */
- uint8_t from_sha3[DIGEST256_LEN];
- /** Method by which the diff is compressed. (part of the ht key */
- compress_method_t compress_method;
- /** One of the CDM_DIFF_* values, depending on whether this diff
- * is available, in progress, or impossible to compute. */
- cdm_diff_status_t cdm_diff_status;
- /** SHA3-256 digest of the consensus that this diff is _to. */
- uint8_t target_sha3[DIGEST256_LEN];
- /** Handle to the cache entry for this diff, if any. We use a handle here
- * to avoid thinking too hard about cache entry lifetime issues. */
- consensus_cache_entry_handle_t *entry;
- } cdm_diff_t;
- /** Hashtable mapping flavor and source consensus digest to status. */
- static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
- /**
- * Configuration for this module
- */
- static consdiff_cfg_t consdiff_cfg = {
- // XXXX I'd like to make this number bigger, but it interferes with the
- // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
- // XXXX rules.
- /* .cache_max_num = */ 128
- };
- static int consdiffmgr_ensure_space_for_files(int n);
- static int consensus_queue_compression_work(const char *consensus,
- const networkstatus_t *as_parsed);
- static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
- consensus_cache_entry_t *diff_to);
- static void consdiffmgr_set_cache_flags(void);
- /* =====
- * Hashtable setup
- * ===== */
- /** Helper: hash the key of a cdm_diff_t. */
- static unsigned
- cdm_diff_hash(const cdm_diff_t *diff)
- {
- uint8_t tmp[DIGEST256_LEN + 2];
- memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
- tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
- tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
- return (unsigned) siphash24g(tmp, sizeof(tmp));
- }
- /** Helper: compare two cdm_diff_t objects for key equality */
- static int
- cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
- {
- return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
- diff1->flavor == diff2->flavor &&
- diff1->compress_method == diff2->compress_method;
- }
- HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
- HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
- 0.6, tor_reallocarray, tor_free_)
- #define cdm_diff_free(diff) \
- FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
- /** Release all storage held in <b>diff</b>. */
- static void
- cdm_diff_free_(cdm_diff_t *diff)
- {
- if (!diff)
- return;
- consensus_cache_entry_handle_free(diff->entry);
- tor_free(diff);
- }
- /** Create and return a new cdm_diff_t with the given values. Does not
- * add it to the hashtable. */
- static cdm_diff_t *
- cdm_diff_new(consensus_flavor_t flav,
- const uint8_t *from_sha3,
- const uint8_t *target_sha3,
- compress_method_t method)
- {
- cdm_diff_t *ent;
- ent = tor_malloc_zero(sizeof(cdm_diff_t));
- ent->flavor = flav;
- memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
- memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
- ent->compress_method = method;
- return ent;
- }
- /**
- * Examine the diff hashtable to see whether we know anything about computing
- * a diff of type <b>flav</b> between consensuses with the two provided
- * SHA3-256 digests. If a computation is in progress, or if the computation
- * has already been tried and failed, return 1. Otherwise, note the
- * computation as "in progress" so that we don't reattempt it later, and
- * return 0.
- */
- static int
- cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
- const uint8_t *from_sha3,
- const uint8_t *target_sha3)
- {
- struct cdm_diff_t search, *ent;
- unsigned u;
- int result = 0;
- for (u = 0; u < n_diff_compression_methods(); ++u) {
- compress_method_t method = compress_diffs_with[u];
- memset(&search, 0, sizeof(cdm_diff_t));
- search.flavor = flav;
- search.compress_method = method;
- memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
- ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
- if (ent) {
- tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
- result = 1;
- continue;
- }
- ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
- ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
- HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
- }
- return result;
- }
- /**
- * Update the status of the diff of type <b>flav</b> between consensuses with
- * the two provided SHA3-256 digests, so that its status becomes
- * <b>status</b>, and its value becomes the <b>handle</b>. If <b>handle</b>
- * is NULL, then the old handle (if any) is freed, and replaced with NULL.
- */
- static void
- cdm_diff_ht_set_status(consensus_flavor_t flav,
- const uint8_t *from_sha3,
- const uint8_t *to_sha3,
- compress_method_t method,
- int status,
- consensus_cache_entry_handle_t *handle)
- {
- if (handle == NULL) {
- tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
- }
- struct cdm_diff_t search, *ent;
- memset(&search, 0, sizeof(cdm_diff_t));
- search.flavor = flav;
- search.compress_method = method,
- memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
- ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
- if (!ent) {
- ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
- ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
- HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
- } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
- // This can happen under certain really pathological conditions
- // if we decide we don't care about a diff before it is actually
- // done computing.
- return;
- }
- tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
- ent->cdm_diff_status = status;
- consensus_cache_entry_handle_free(ent->entry);
- ent->entry = handle;
- }
- /**
- * Helper: Remove from the hash table every present (actually computed) diff
- * of type <b>flav</b> whose target digest does not match
- * <b>unless_target_sha3_matches</b>.
- *
- * This function is used for the hash table to throw away references to diffs
- * that do not lead to the most given consensus of a given flavor.
- */
- static void
- cdm_diff_ht_purge(consensus_flavor_t flav,
- const uint8_t *unless_target_sha3_matches)
- {
- cdm_diff_t **diff, **next;
- for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
- cdm_diff_t *this = *diff;
- if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
- flav == (*diff)->flavor) {
- if (BUG((*diff)->entry == NULL) ||
- consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
- /* the underlying entry has gone away; drop this. */
- next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
- cdm_diff_free(this);
- continue;
- }
- if (unless_target_sha3_matches &&
- fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
- DIGEST256_LEN)) {
- /* target hash doesn't match; drop this. */
- next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
- cdm_diff_free(this);
- continue;
- }
- }
- next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
- }
- }
- /**
- * Helper: initialize <b>cons_diff_cache</b>.
- */
- static void
- cdm_cache_init(void)
- {
- unsigned n_entries = consdiff_cfg.cache_max_num * 2;
- tor_assert(cons_diff_cache == NULL);
- cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
- if (cons_diff_cache == NULL) {
- // LCOV_EXCL_START
- log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
- tor_assert_unreached();
- // LCOV_EXCL_STOP
- } else {
- consdiffmgr_set_cache_flags();
- }
- cdm_cache_dirty = 1;
- cdm_cache_loaded = 0;
- }
- /**
- * Helper: return the consensus_cache_t * that backs this manager,
- * initializing it if needed.
- */
- STATIC consensus_cache_t *
- cdm_cache_get(void)
- {
- if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
- cdm_cache_init();
- }
- return cons_diff_cache;
- }
- /**
- * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
- * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
- * with <b>label</b> as its label.
- */
- static void
- cdm_labels_prepend_sha3(config_line_t **labels,
- const char *label,
- const uint8_t *body,
- size_t bodylen)
- {
- uint8_t sha3_digest[DIGEST256_LEN];
- char hexdigest[HEX_DIGEST256_LEN+1];
- crypto_digest256((char *)sha3_digest,
- (const char *)body, bodylen, DIGEST_SHA3_256);
- base16_encode(hexdigest, sizeof(hexdigest),
- (const char *)sha3_digest, sizeof(sha3_digest));
- config_line_prepend(labels, label, hexdigest);
- }
- /** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
- * given label, set <b>digest_out</b> to that value (decoded), and return 0.
- *
- * Return -1 if there is no such label, and -2 if it is badly formatted. */
- STATIC int
- cdm_entry_get_sha3_value(uint8_t *digest_out,
- consensus_cache_entry_t *ent,
- const char *label)
- {
- if (ent == NULL)
- return -1;
- const char *hex = consensus_cache_entry_get_value(ent, label);
- if (hex == NULL)
- return -1;
- int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
- if (n != DIGEST256_LEN)
- return -2;
- else
- return 0;
- }
- /**
- * Helper: look for a consensus with the given <b>flavor</b> and
- * <b>valid_after</b> time in the cache. Return that consensus if it's
- * present, or NULL if it's missing.
- */
- STATIC consensus_cache_entry_t *
- cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
- {
- char formatted_time[ISO_TIME_LEN+1];
- format_iso_time_nospace(formatted_time, valid_after);
- const char *flavname = networkstatus_get_flavor_name(flavor);
- /* We'll filter by valid-after time first, since that should
- * match the fewest documents. */
- /* We could add an extra hashtable here, but since we only do this scan
- * when adding a new consensus, it probably doesn't matter much. */
- smartlist_t *matches = smartlist_new();
- consensus_cache_find_all(matches, cdm_cache_get(),
- LABEL_VALID_AFTER, formatted_time);
- consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
- consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- consensus_cache_entry_t *result = NULL;
- if (smartlist_len(matches)) {
- result = smartlist_get(matches, 0);
- }
- smartlist_free(matches);
- return result;
- }
- /** Return the maximum age (in seconds) of consensuses that we should consider
- * storing. The available space in the directory may impose additional limits
- * on how much we store. */
- static int32_t
- get_max_age_to_cache(void)
- {
- const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
- const int32_t MIN_MAX_AGE_TO_CACHE = 0;
- const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
- const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
- const or_options_t *options = get_options();
- if (options->MaxConsensusAgeForDiffs) {
- const int v = options->MaxConsensusAgeForDiffs;
- if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
- return MAX_MAX_AGE_TO_CACHE;
- else
- return v;
- }
- /* The parameter is in hours, so we multiply */
- return 3600 * networkstatus_get_param(NULL,
- MAX_AGE_TO_CACHE_NAME,
- DEFAULT_MAX_AGE_TO_CACHE,
- MIN_MAX_AGE_TO_CACHE,
- MAX_MAX_AGE_TO_CACHE);
- }
- /**
- * Given a string containing a networkstatus consensus, and the results of
- * having parsed that consensus, add that consensus to the cache if it is not
- * already present and not too old. Create new consensus diffs from or to
- * that consensus as appropriate.
- *
- * Return 0 on success and -1 on failure.
- */
- int
- consdiffmgr_add_consensus(const char *consensus,
- const networkstatus_t *as_parsed)
- {
- if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
- return -1; // LCOV_EXCL_LINE
- if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
- return -1; // LCOV_EXCL_LINE
- const consensus_flavor_t flavor = as_parsed->flavor;
- const time_t valid_after = as_parsed->valid_after;
- if (valid_after < approx_time() - get_max_age_to_cache()) {
- log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
- "too old.");
- return -1;
- }
- /* Do we already have this one? */
- consensus_cache_entry_t *entry =
- cdm_cache_lookup_consensus(flavor, valid_after);
- if (entry) {
- log_info(LD_DIRSERV, "We already have a copy of that consensus");
- return -1;
- }
- /* We don't have it. Add it to the cache. */
- return consensus_queue_compression_work(consensus, as_parsed);
- }
- /**
- * Helper: used to sort two smartlists of consensus_cache_entry_t by their
- * LABEL_VALID_AFTER labels.
- */
- static int
- compare_by_valid_after_(const void **a, const void **b)
- {
- const consensus_cache_entry_t *e1 = *a;
- const consensus_cache_entry_t *e2 = *b;
- /* We're in luck here: sorting UTC iso-encoded values lexically will work
- * fine (until 9999). */
- return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
- consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
- }
- /**
- * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
- * entry.
- */
- static consensus_cache_entry_t *
- sort_and_find_most_recent(smartlist_t *lst)
- {
- smartlist_sort(lst, compare_by_valid_after_);
- if (smartlist_len(lst)) {
- return smartlist_get(lst, smartlist_len(lst) - 1);
- } else {
- return NULL;
- }
- }
- /** Return i such that compress_consensus_with[i] == method. Return
- * -1 if no such i exists. */
- static int
- consensus_compression_method_pos(compress_method_t method)
- {
- unsigned i;
- for (i = 0; i < n_consensus_compression_methods(); ++i) {
- if (compress_consensus_with[i] == method) {
- return i;
- }
- }
- return -1;
- }
- /**
- * If we know a consensus with the flavor <b>flavor</b> compressed with
- * <b>method</b>, set *<b>entry_out</b> to that value. Return values are as
- * for consdiffmgr_find_diff_from().
- */
- consdiff_status_t
- consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
- consensus_flavor_t flavor,
- compress_method_t method)
- {
- tor_assert(entry_out);
- tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
- int pos = consensus_compression_method_pos(method);
- if (pos < 0) {
- // We don't compress consensuses with this method.
- return CONSDIFF_NOT_FOUND;
- }
- consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
- if (!handle)
- return CONSDIFF_NOT_FOUND;
- *entry_out = consensus_cache_entry_handle_get(handle);
- if (*entry_out)
- return CONSDIFF_AVAILABLE;
- else
- return CONSDIFF_NOT_FOUND;
- }
- /**
- * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
- * from the source consensus with the specified digest (which must be SHA3).
- *
- * If the diff is present, store it into *<b>entry_out</b> and return
- * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
- * CONSDIFF_IN_PROGRESS.
- */
- consdiff_status_t
- consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
- consensus_flavor_t flavor,
- int digest_type,
- const uint8_t *digest,
- size_t digestlen,
- compress_method_t method)
- {
- if (BUG(digest_type != DIGEST_SHA3_256) ||
- BUG(digestlen != DIGEST256_LEN)) {
- return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
- }
- // Try to look up the entry in the hashtable.
- cdm_diff_t search, *ent;
- memset(&search, 0, sizeof(search));
- search.flavor = flavor;
- search.compress_method = method;
- memcpy(search.from_sha3, digest, DIGEST256_LEN);
- ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
- if (ent == NULL ||
- ent->cdm_diff_status == CDM_DIFF_ERROR) {
- return CONSDIFF_NOT_FOUND;
- } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
- return CONSDIFF_IN_PROGRESS;
- } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
- return CONSDIFF_IN_PROGRESS;
- }
- if (BUG(ent->entry == NULL)) {
- return CONSDIFF_NOT_FOUND;
- }
- *entry_out = consensus_cache_entry_handle_get(ent->entry);
- return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
- #if 0
- // XXXX Remove this. I'm keeping it around for now in case we need to
- // XXXX debug issues in the hashtable.
- char hex[HEX_DIGEST256_LEN+1];
- base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
- const char *flavname = networkstatus_get_flavor_name(flavor);
- smartlist_t *matches = smartlist_new();
- consensus_cache_find_all(matches, cdm_cache_get(),
- LABEL_FROM_SHA3_DIGEST, hex);
- consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
- consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
- *entry_out = sort_and_find_most_recent(matches);
- consdiff_status_t result =
- (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
- smartlist_free(matches);
- return result;
- #endif /* 0 */
- }
- /**
- * Perform periodic cleanup tasks on the consensus diff cache. Return
- * the number of objects marked for deletion.
- */
- int
- consdiffmgr_cleanup(void)
- {
- smartlist_t *objects = smartlist_new();
- smartlist_t *consensuses = smartlist_new();
- smartlist_t *diffs = smartlist_new();
- int n_to_delete = 0;
- log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
- // 1. Delete any consensus or diff or anything whose valid_after is too old.
- const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
- consensus_cache_find_all(objects, cdm_cache_get(),
- NULL, NULL);
- SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
- const char *lv_valid_after =
- consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
- if (! lv_valid_after) {
- log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
- LABEL_VALID_AFTER);
- continue;
- }
- time_t valid_after = 0;
- if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
- log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
- "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
- continue;
- }
- if (valid_after < valid_after_cutoff) {
- log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
- "too old", LABEL_VALID_AFTER, lv_valid_after);
- consensus_cache_entry_mark_for_removal(ent);
- ++n_to_delete;
- }
- } SMARTLIST_FOREACH_END(ent);
- // 2. Delete all diffs that lead to a consensus whose valid-after is not the
- // latest.
- for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
- const char *flavname = networkstatus_get_flavor_name(flav);
- /* Determine the most recent consensus of this flavor */
- consensus_cache_find_all(consensuses, cdm_cache_get(),
- LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
- consensus_cache_entry_t *most_recent =
- sort_and_find_most_recent(consensuses);
- if (most_recent == NULL)
- continue;
- const char *most_recent_sha3 =
- consensus_cache_entry_get_value(most_recent,
- LABEL_SHA3_DIGEST_UNCOMPRESSED);
- if (BUG(most_recent_sha3 == NULL))
- continue; // LCOV_EXCL_LINE
- /* consider all such-flavored diffs, and look to see if they match. */
- consensus_cache_find_all(diffs, cdm_cache_get(),
- LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
- consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
- SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
- const char *this_diff_target_sha3 =
- consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
- if (!this_diff_target_sha3)
- continue;
- if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
- consensus_cache_entry_mark_for_removal(diff);
- ++n_to_delete;
- }
- } SMARTLIST_FOREACH_END(diff);
- smartlist_clear(consensuses);
- smartlist_clear(diffs);
- }
- // 3. Delete all consensuses except the most recent that are compressed with
- // an un-preferred method.
- for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
- const char *flavname = networkstatus_get_flavor_name(flav);
- /* Determine the most recent consensus of this flavor */
- consensus_cache_find_all(consensuses, cdm_cache_get(),
- LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
- consensus_cache_entry_t *most_recent =
- sort_and_find_most_recent(consensuses);
- if (most_recent == NULL)
- continue;
- const char *most_recent_sha3_uncompressed =
- consensus_cache_entry_get_value(most_recent,
- LABEL_SHA3_DIGEST_UNCOMPRESSED);
- const char *retain_methodname = compression_method_get_name(
- RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
- if (BUG(most_recent_sha3_uncompressed == NULL))
- continue;
- SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
- const char *lv_sha3_uncompressed =
- consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
- if (BUG(! lv_sha3_uncompressed))
- continue;
- if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
- continue; // This _is_ the most recent.
- const char *lv_methodname =
- consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
- if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
- consensus_cache_entry_mark_for_removal(ent);
- ++n_to_delete;
- }
- } SMARTLIST_FOREACH_END(ent);
- }
- smartlist_free(objects);
- smartlist_free(consensuses);
- smartlist_free(diffs);
- // Actually remove files, if they're not used.
- consensus_cache_delete_pending(cdm_cache_get(), 0);
- return n_to_delete;
- }
- /**
- * Initialize the consensus diff manager and its cache, and configure
- * its parameters based on the latest torrc and networkstatus parameters.
- */
- void
- consdiffmgr_configure(const consdiff_cfg_t *cfg)
- {
- if (cfg)
- memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
- (void) cdm_cache_get();
- }
- /**
- * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
- * operations that the consensus diff manager will need.
- */
- int
- consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
- {
- return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
- }
- /**
- * Scan the consensus diff manager's cache for any grossly malformed entries,
- * and mark them as deletable. Return 0 if no problems were found; 1
- * if problems were found and fixed.
- */
- int
- consdiffmgr_validate(void)
- {
- /* Right now, we only check for entries that have bad sha3 values */
- int problems = 0;
- smartlist_t *objects = smartlist_new();
- consensus_cache_find_all(objects, cdm_cache_get(),
- NULL, NULL);
- SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
- uint8_t sha3_expected[DIGEST256_LEN];
- uint8_t sha3_received[DIGEST256_LEN];
- int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
- if (r == -1) {
- /* digest isn't there; that's allowed */
- continue;
- } else if (r == -2) {
- /* digest is malformed; that's not allowed */
- problems = 1;
- consensus_cache_entry_mark_for_removal(obj);
- continue;
- }
- const uint8_t *body;
- size_t bodylen;
- consensus_cache_entry_incref(obj);
- r = consensus_cache_entry_get_body(obj, &body, &bodylen);
- if (r == 0) {
- crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
- DIGEST_SHA3_256);
- }
- consensus_cache_entry_decref(obj);
- if (r < 0)
- continue;
- // Deconfuse coverity about the possibility of sha3_received being
- // uninitialized
- tor_assert(r <= 0);
- if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
- problems = 1;
- consensus_cache_entry_mark_for_removal(obj);
- continue;
- }
- } SMARTLIST_FOREACH_END(obj);
- smartlist_free(objects);
- return problems;
- }
- /**
- * Helper: build new diffs of <b>flavor</b> as needed
- */
- static void
- consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
- {
- smartlist_t *matches = NULL;
- smartlist_t *diffs = NULL;
- smartlist_t *compute_diffs_from = NULL;
- strmap_t *have_diff_from = NULL;
- // look for the most recent consensus, and for all previous in-range
- // consensuses. Do they all have diffs to it?
- const char *flavname = networkstatus_get_flavor_name(flavor);
- // 1. find the most recent consensus, and the ones that we might want
- // to diff to it.
- const char *methodname = compression_method_get_name(
- RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
- matches = smartlist_new();
- consensus_cache_find_all(matches, cdm_cache_get(),
- LABEL_FLAVOR, flavname);
- consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
- consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
- if (!most_recent) {
- log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
- "not making diffs", flavname);
- goto done;
- }
- tor_assert(smartlist_len(matches));
- smartlist_del(matches, smartlist_len(matches) - 1);
- const char *most_recent_valid_after =
- consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
- if (BUG(most_recent_valid_after == NULL))
- goto done; //LCOV_EXCL_LINE
- uint8_t most_recent_sha3[DIGEST256_LEN];
- if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
- LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
- goto done; //LCOV_EXCL_LINE
- // 2. Find all the relevant diffs _to_ this consensus. These are ones
- // that we don't need to compute.
- diffs = smartlist_new();
- consensus_cache_find_all(diffs, cdm_cache_get(),
- LABEL_VALID_AFTER, most_recent_valid_after);
- consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
- consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
- have_diff_from = strmap_new();
- SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
- const char *va = consensus_cache_entry_get_value(diff,
- LABEL_FROM_VALID_AFTER);
- if (BUG(va == NULL))
- continue; // LCOV_EXCL_LINE
- strmap_set(have_diff_from, va, diff);
- } SMARTLIST_FOREACH_END(diff);
- // 3. See which consensuses in 'matches' don't have diffs yet.
- smartlist_reverse(matches); // from newest to oldest.
- compute_diffs_from = smartlist_new();
- SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
- const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
- if (BUG(va == NULL))
- continue; // LCOV_EXCL_LINE
- if (strmap_get(have_diff_from, va) != NULL)
- continue; /* we already have this one. */
- smartlist_add(compute_diffs_from, ent);
- /* Since we are not going to serve this as the most recent consensus
- * any more, we should stop keeping it mmap'd when it's not in use.
- */
- consensus_cache_entry_mark_for_aggressive_release(ent);
- } SMARTLIST_FOREACH_END(ent);
- log_info(LD_DIRSERV,
- "The most recent %s consensus is valid-after %s. We have diffs to "
- "this consensus for %d/%d older %s consensuses. Generating diffs "
- "for the other %d.",
- flavname,
- most_recent_valid_after,
- smartlist_len(matches) - smartlist_len(compute_diffs_from),
- smartlist_len(matches),
- flavname,
- smartlist_len(compute_diffs_from));
- // 4. Update the hashtable; remove entries in this flavor to other
- // target consensuses.
- cdm_diff_ht_purge(flavor, most_recent_sha3);
- // 5. Actually launch the requests.
- SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
- if (BUG(c == most_recent))
- continue; // LCOV_EXCL_LINE
- uint8_t this_sha3[DIGEST256_LEN];
- if (cdm_entry_get_sha3_value(this_sha3, c,
- LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
- // Not actually a bug, since we might be running with a directory
- // with stale files from before the #22143 fixes.
- continue;
- }
- if (cdm_diff_ht_check_and_note_pending(flavor,
- this_sha3, most_recent_sha3)) {
- // This is already pending, or we encountered an error.
- continue;
- }
- consensus_diff_queue_diff_work(c, most_recent);
- } SMARTLIST_FOREACH_END(c);
- done:
- smartlist_free(matches);
- smartlist_free(diffs);
- smartlist_free(compute_diffs_from);
- strmap_free(have_diff_from, NULL);
- }
- /**
- * Scan the cache for the latest consensuses and add their handles to
- * latest_consensus
- */
- static void
- consdiffmgr_consensus_load(void)
- {
- smartlist_t *matches = smartlist_new();
- for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
- const char *flavname = networkstatus_get_flavor_name(flav);
- smartlist_clear(matches);
- consensus_cache_find_all(matches, cdm_cache_get(),
- LABEL_FLAVOR, flavname);
- consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
- if (! most_recent)
- continue; // no consensuses.
- const char *most_recent_sha3 =
- consensus_cache_entry_get_value(most_recent,
- LABEL_SHA3_DIGEST_UNCOMPRESSED);
- if (BUG(most_recent_sha3 == NULL))
- continue; // LCOV_EXCL_LINE
- consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
- most_recent_sha3);
- // Everything that remains matches the most recent consensus of this
- // flavor.
- SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
- const char *lv_compression =
- consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
- compress_method_t method =
- compression_method_get_by_name(lv_compression);
- int pos = consensus_compression_method_pos(method);
- if (pos < 0)
- continue;
- consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
- latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
- } SMARTLIST_FOREACH_END(ent);
- }
- smartlist_free(matches);
- }
- /**
- * Scan the cache for diffs, and add them to the hashtable.
- */
- static void
- consdiffmgr_diffs_load(void)
- {
- smartlist_t *diffs = smartlist_new();
- consensus_cache_find_all(diffs, cdm_cache_get(),
- LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
- SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
- const char *lv_flavor =
- consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
- if (!lv_flavor)
- continue;
- int flavor = networkstatus_parse_flavor_name(lv_flavor);
- if (flavor < 0)
- continue;
- const char *lv_compression =
- consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
- compress_method_t method = NO_METHOD;
- if (lv_compression) {
- method = compression_method_get_by_name(lv_compression);
- if (method == UNKNOWN_METHOD) {
- continue;
- }
- }
- uint8_t from_sha3[DIGEST256_LEN];
- uint8_t to_sha3[DIGEST256_LEN];
- if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
- continue;
- if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
- continue;
- cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
- method,
- CDM_DIFF_PRESENT,
- consensus_cache_entry_handle_new(diff));
- } SMARTLIST_FOREACH_END(diff);
- smartlist_free(diffs);
- }
- /**
- * Build new diffs as needed.
- */
- void
- consdiffmgr_rescan(void)
- {
- if (cdm_cache_dirty == 0)
- return;
- // Clean up here to make room for new diffs, and to ensure that older
- // consensuses do not have any entries.
- consdiffmgr_cleanup();
- if (cdm_cache_loaded == 0) {
- consdiffmgr_diffs_load();
- consdiffmgr_consensus_load();
- cdm_cache_loaded = 1;
- }
- for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
- consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
- }
- cdm_cache_dirty = 0;
- }
- /**
- * Helper: compare two files by their from-valid-after and valid-after labels,
- * trying to sort in ascending order by from-valid-after (when present) and
- * valid-after (when not). Place everything that has neither label first in
- * the list.
- */
- static int
- compare_by_staleness_(const void **a, const void **b)
- {
- const consensus_cache_entry_t *e1 = *a;
- const consensus_cache_entry_t *e2 = *b;
- const char *va1, *fva1, *va2, *fva2;
- va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
- va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
- fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
- fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
- if (fva1)
- va1 = fva1;
- if (fva2)
- va2 = fva2;
- /* See note about iso-encoded values in compare_by_valid_after_. Also note
- * that missing dates will get placed first. */
- return strcmp_opt(va1, va2);
- }
- /** If there are not enough unused filenames to store <b>n</b> files, then
- * delete old consensuses until there are. (We have to keep track of the
- * number of filenames because of the way that the seccomp2 cache works.)
- *
- * Return 0 on success, -1 on failure.
- **/
- static int
- consdiffmgr_ensure_space_for_files(int n)
- {
- consensus_cache_t *cache = cdm_cache_get();
- if (consensus_cache_get_n_filenames_available(cache) >= n) {
- // there are already enough unused filenames.
- return 0;
- }
- // Try a cheap deletion of stuff that's waiting to get deleted.
- consensus_cache_delete_pending(cache, 0);
- if (consensus_cache_get_n_filenames_available(cache) >= n) {
- // okay, _that_ made enough filenames available.
- return 0;
- }
- // Let's get more assertive: clean out unused stuff, and force-remove
- // the files that we can.
- consdiffmgr_cleanup();
- consensus_cache_delete_pending(cache, 1);
- const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
- if (n_to_remove <= 0) {
- // okay, finally!
- return 0;
- }
- // At this point, we're going to have to throw out objects that will be
- // missed. Too bad!
- smartlist_t *objects = smartlist_new();
- consensus_cache_find_all(objects, cache, NULL, NULL);
- smartlist_sort(objects, compare_by_staleness_);
- int n_marked = 0;
- SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
- consensus_cache_entry_mark_for_removal(ent);
- if (++n_marked >= n_to_remove)
- break;
- } SMARTLIST_FOREACH_END(ent);
- smartlist_free(objects);
- consensus_cache_delete_pending(cache, 1);
- if (consensus_cache_may_overallocate(cache)) {
- /* If we're allowed to throw extra files into the cache, let's do so
- * rather getting upset.
- */
- return 0;
- }
- if (BUG(n_marked < n_to_remove))
- return -1;
- else
- return 0;
- }
- /**
- * Set consensus cache flags on the objects in this consdiffmgr.
- */
- static void
- consdiffmgr_set_cache_flags(void)
- {
- /* Right now, we just mark the consensus objects for aggressive release,
- * so that they get mmapped for as little time as possible. */
- smartlist_t *objects = smartlist_new();
- consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
- DOCTYPE_CONSENSUS);
- SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
- consensus_cache_entry_mark_for_aggressive_release(ent);
- } SMARTLIST_FOREACH_END(ent);
- smartlist_free(objects);
- }
- /**
- * Called before shutdown: drop all storage held by the consdiffmgr.c module.
- */
- void
- consdiffmgr_free_all(void)
- {
- cdm_diff_t **diff, **next;
- for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
- cdm_diff_t *this = *diff;
- next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
- cdm_diff_free(this);
- }
- int i;
- unsigned j;
- for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
- for (j = 0; j < n_consensus_compression_methods(); ++j) {
- consensus_cache_entry_handle_free(latest_consensus[i][j]);
- }
- }
- memset(latest_consensus, 0, sizeof(latest_consensus));
- consensus_cache_free(cons_diff_cache);
- cons_diff_cache = NULL;
- }
- /* =====
- Thread workers
- =====*/
- typedef struct compressed_result_t {
- config_line_t *labels;
- /**
- * Output: Body of the diff, as compressed.
- */
- uint8_t *body;
- /**
- * Output: length of body_out
- */
- size_t bodylen;
- } compressed_result_t;
- /**
- * Compress the bytestring <b>input</b> of length <b>len</b> using the
- * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
- *
- * For each successful compression, set the fields in the <b>results_out</b>
- * array in the position corresponding to the compression method. Use
- * <b>labels_in</b> as a basis for the labels of the result.
- *
- * Return 0 if all compression succeeded; -1 if any failed.
- */
- static int
- compress_multiple(compressed_result_t *results_out, int n_methods,
- const compress_method_t *methods,
- const uint8_t *input, size_t len,
- const config_line_t *labels_in)
- {
- int rv = 0;
- int i;
- for (i = 0; i < n_methods; ++i) {
- compress_method_t method = methods[i];
- const char *methodname = compression_method_get_name(method);
- char *result;
- size_t sz;
- if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
- results_out[i].body = (uint8_t*)result;
- results_out[i].bodylen = sz;
- results_out[i].labels = config_lines_dup(labels_in);
- cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
- results_out[i].body,
- results_out[i].bodylen);
- config_line_prepend(&results_out[i].labels,
- LABEL_COMPRESSION_TYPE,
- methodname);
- } else {
- rv = -1;
- }
- }
- return rv;
- }
- /**
- * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
- * as produced by compress_multiple, store them all into the
- * consdiffmgr, and store handles to them in the <b>handles_out</b>
- * array.
- *
- * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
- * was stored.
- */
- static cdm_diff_status_t
- store_multiple(consensus_cache_entry_handle_t **handles_out,
- int n,
- const compress_method_t *methods,
- const compressed_result_t *results,
- const char *description)
- {
- cdm_diff_status_t status = CDM_DIFF_ERROR;
- consdiffmgr_ensure_space_for_files(n);
- int i;
- for (i = 0; i < n; ++i) {
- compress_method_t method = methods[i];
- uint8_t *body_out = results[i].body;
- size_t bodylen_out = results[i].bodylen;
- config_line_t *labels = results[i].labels;
- const char *methodname = compression_method_get_name(method);
- if (body_out && bodylen_out && labels) {
- /* Success! Store the results */
- log_info(LD_DIRSERV, "Adding %s, compressed with %s",
- description, methodname);
- consensus_cache_entry_t *ent =
- consensus_cache_add(cdm_cache_get(),
- labels,
- body_out,
- bodylen_out);
- if (ent == NULL) {
- static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
- log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
- "Unable to store object %s compressed with %s.",
- description, methodname);
- continue;
- }
- status = CDM_DIFF_PRESENT;
- handles_out[i] = consensus_cache_entry_handle_new(ent);
- consensus_cache_entry_decref(ent);
- }
- }
- return status;
- }
- /**
- * An object passed to a worker thread that will try to produce a consensus
- * diff.
- */
- typedef struct consensus_diff_worker_job_t {
- /**
- * Input: The consensus to compute the diff from. Holds a reference to the
- * cache entry, which must not be released until the job is passed back to
- * the main thread. The body must be mapped into memory in the main thread.
- */
- consensus_cache_entry_t *diff_from;
- /**
- * Input: The consensus to compute the diff to. Holds a reference to the
- * cache entry, which must not be released until the job is passed back to
- * the main thread. The body must be mapped into memory in the main thread.
- */
- consensus_cache_entry_t *diff_to;
- /** Output: labels and bodies */
- compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
- } consensus_diff_worker_job_t;
- /** Given a consensus_cache_entry_t, check whether it has a label claiming
- * that it was compressed. If so, uncompress its contents into <b>out</b> and
- * set <b>outlen</b> to hold their size. If not, just copy the body into
- * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success,
- * -1 on failure.
- *
- * In all cases, the output is nul-terminated. */
- STATIC int
- uncompress_or_copy(char **out, size_t *outlen,
- consensus_cache_entry_t *ent)
- {
- const uint8_t *body;
- size_t bodylen;
- if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
- return -1;
- const char *lv_compression =
- consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
- compress_method_t method = NO_METHOD;
- if (lv_compression)
- method = compression_method_get_by_name(lv_compression);
- return tor_uncompress(out, outlen, (const char *)body, bodylen,
- method, 1, LOG_WARN);
- }
- /**
- * Worker function. This function runs inside a worker thread and receives
- * a consensus_diff_worker_job_t as its input.
- */
- static workqueue_reply_t
- consensus_diff_worker_threadfn(void *state_, void *work_)
- {
- (void)state_;
- consensus_diff_worker_job_t *job = work_;
- const uint8_t *diff_from, *diff_to;
- size_t len_from, len_to;
- int r;
- /* We need to have the body already mapped into RAM here.
- */
- r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
- if (BUG(r < 0))
- return WQ_RPL_REPLY; // LCOV_EXCL_LINE
- r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
- if (BUG(r < 0))
- return WQ_RPL_REPLY; // LCOV_EXCL_LINE
- const char *lv_to_valid_after =
- consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
- const char *lv_to_fresh_until =
- consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
- const char *lv_to_valid_until =
- consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
- const char *lv_to_signatories =
- consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
- const char *lv_from_valid_after =
- consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
- const char *lv_from_digest =
- consensus_cache_entry_get_value(job->diff_from,
- LABEL_SHA3_DIGEST_AS_SIGNED);
- const char *lv_from_flavor =
- consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
- const char *lv_to_flavor =
- consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
- const char *lv_to_digest =
- consensus_cache_entry_get_value(job->diff_to,
- LABEL_SHA3_DIGEST_UNCOMPRESSED);
- if (! lv_from_digest) {
- /* This isn't a bug right now, since it can happen if you're migrating
- * from an older version of master to a newer one. The older ones didn't
- * annotate their stored consensus objects with sha3-digest-as-signed.
- */
- return WQ_RPL_REPLY; // LCOV_EXCL_LINE
- }
- /* All these values are mandatory on the input */
- if (BUG(!lv_to_valid_after) ||
- BUG(!lv_from_valid_after) ||
- BUG(!lv_from_flavor) ||
- BUG(!lv_to_flavor)) {
- return WQ_RPL_REPLY; // LCOV_EXCL_LINE
- }
- /* The flavors need to match */
- if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
- return WQ_RPL_REPLY; // LCOV_EXCL_LINE
- }
- char *consensus_diff;
- {
- char *diff_from_nt = NULL, *diff_to_nt = NULL;
- size_t diff_from_nt_len, diff_to_nt_len;
- if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
- job->diff_from) < 0) {
- return WQ_RPL_REPLY;
- }
- if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
- job->diff_to) < 0) {
- tor_free(diff_from_nt);
- return WQ_RPL_REPLY;
- }
- tor_assert(diff_from_nt);
- tor_assert(diff_to_nt);
- // XXXX ugh; this is going to calculate the SHA3 of both its
- // XXXX inputs again, even though we already have that. Maybe it's time
- // XXXX to change the API here?
- consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
- tor_free(diff_from_nt);
- tor_free(diff_to_nt);
- }
- if (!consensus_diff) {
- /* Couldn't generate consensus; we'll leave the reply blank. */
- return WQ_RPL_REPLY;
- }
- /* Compress the results and send the reply */
- tor_assert(compress_diffs_with[0] == NO_METHOD);
- size_t difflen = strlen(consensus_diff);
- job->out[0].body = (uint8_t *) consensus_diff;
- job->out[0].bodylen = difflen;
- config_line_t *common_labels = NULL;
- if (lv_to_valid_until)
- config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
- if (lv_to_fresh_until)
- config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
- if (lv_to_signatories)
- config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
- cdm_labels_prepend_sha3(&common_labels,
- LABEL_SHA3_DIGEST_UNCOMPRESSED,
- job->out[0].body,
- job->out[0].bodylen);
- config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
- lv_from_valid_after);
- config_line_prepend(&common_labels, LABEL_VALID_AFTER,
- lv_to_valid_after);
- config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
- config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
- lv_from_digest);
- config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
- lv_to_digest);
- config_line_prepend(&common_labels, LABEL_DOCTYPE,
- DOCTYPE_CONSENSUS_DIFF);
- job->out[0].labels = config_lines_dup(common_labels);
- cdm_labels_prepend_sha3(&job->out[0].labels,
- LABEL_SHA3_DIGEST,
- job->out[0].body,
- job->out[0].bodylen);
- compress_multiple(job->out+1,
- n_diff_compression_methods()-1,
- compress_diffs_with+1,
- (const uint8_t*)consensus_diff, difflen, common_labels);
- config_free_lines(common_labels);
- return WQ_RPL_REPLY;
- }
- #define consensus_diff_worker_job_free(job) \
- FREE_AND_NULL(consensus_diff_worker_job_t, \
- consensus_diff_worker_job_free_, (job))
- /**
- * Helper: release all storage held in <b>job</b>.
- */
- static void
- consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
- {
- if (!job)
- return;
- unsigned u;
- for (u = 0; u < n_diff_compression_methods(); ++u) {
- config_free_lines(job->out[u].labels);
- tor_free(job->out[u].body);
- }
- consensus_cache_entry_decref(job->diff_from);
- consensus_cache_entry_decref(job->diff_to);
- tor_free(job);
- }
- /**
- * Worker function: This function runs in the main thread, and receives
- * a consensus_diff_worker_job_t that the worker thread has already
- * processed.
- */
- static void
- consensus_diff_worker_replyfn(void *work_)
- {
- tor_assert(in_main_thread());
- tor_assert(work_);
- consensus_diff_worker_job_t *job = work_;
- const char *lv_from_digest =
- consensus_cache_entry_get_value(job->diff_from,
- LABEL_SHA3_DIGEST_AS_SIGNED);
- const char *lv_to_digest =
- consensus_cache_entry_get_value(job->diff_to,
- LABEL_SHA3_DIGEST_UNCOMPRESSED);
- const char *lv_flavor =
- consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
- if (BUG(lv_from_digest == NULL))
- lv_from_digest = "???"; // LCOV_EXCL_LINE
- if (BUG(lv_to_digest == NULL))
- lv_to_digest = "???"; // LCOV_EXCL_LINE
- uint8_t from_sha3[DIGEST256_LEN];
- uint8_t to_sha3[DIGEST256_LEN];
- int flav = -1;
- int cache = 1;
- if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
- LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
- cache = 0;
- if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
- LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
- cache = 0;
- if (BUG(lv_flavor == NULL)) {
- cache = 0;
- } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
- cache = 0;
- }
- consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
- memset(handles, 0, sizeof(handles));
- char description[128];
- tor_snprintf(description, sizeof(description),
- "consensus diff from %s to %s",
- lv_from_digest, lv_to_digest);
- int status = store_multiple(handles,
- n_diff_compression_methods(),
- compress_diffs_with,
- job->out,
- description);
- if (status != CDM_DIFF_PRESENT) {
- /* Failure! Nothing to do but complain */
- log_warn(LD_DIRSERV,
- "Worker was unable to compute consensus diff "
- "from %s to %s", lv_from_digest, lv_to_digest);
- /* Cache this error so we don't try to compute this one again. */
- status = CDM_DIFF_ERROR;
- }
- unsigned u;
- for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
- compress_method_t method = compress_diffs_with[u];
- if (cache) {
- consensus_cache_entry_handle_t *h = handles[u];
- int this_status = status;
- if (h == NULL) {
- this_status = CDM_DIFF_ERROR;
- }
- tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
- cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
- } else {
- consensus_cache_entry_handle_free(handles[u]);
- }
- }
- consensus_diff_worker_job_free(job);
- }
- /**
- * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
- * in a worker thread.
- */
- static int
- consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
- consensus_cache_entry_t *diff_to)
- {
- tor_assert(in_main_thread());
- consensus_cache_entry_incref(diff_from);
- consensus_cache_entry_incref(diff_to);
- consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
- job->diff_from = diff_from;
- job->diff_to = diff_to;
- /* Make sure body is mapped. */
- const uint8_t *body;
- size_t bodylen;
- int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
- int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
- if (r1 < 0 || r2 < 0)
- goto err;
- workqueue_entry_t *work;
- work = cpuworker_queue_work(WQ_PRI_LOW,
- consensus_diff_worker_threadfn,
- consensus_diff_worker_replyfn,
- job);
- if (!work)
- goto err;
- return 0;
- err:
- consensus_diff_worker_job_free(job); // includes decrefs.
- return -1;
- }
- /**
- * Holds requests and replies for consensus_compress_workers.
- */
- typedef struct consensus_compress_worker_job_t {
- char *consensus;
- size_t consensus_len;
- consensus_flavor_t flavor;
- config_line_t *labels_in;
- compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
- } consensus_compress_worker_job_t;
- #define consensus_compress_worker_job_free(job) \
- FREE_AND_NULL(consensus_compress_worker_job_t, \
- consensus_compress_worker_job_free_, (job))
- /**
- * Free all resources held in <b>job</b>
- */
- static void
- consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
- {
- if (!job)
- return;
- tor_free(job->consensus);
- config_free_lines(job->labels_in);
- unsigned u;
- for (u = 0; u < n_consensus_compression_methods(); ++u) {
- config_free_lines(job->out[u].labels);
- tor_free(job->out[u].body);
- }
- tor_free(job);
- }
- /**
- * Worker function. This function runs inside a worker thread and receives
- * a consensus_compress_worker_job_t as its input.
- */
- static workqueue_reply_t
- consensus_compress_worker_threadfn(void *state_, void *work_)
- {
- (void)state_;
- consensus_compress_worker_job_t *job = work_;
- consensus_flavor_t flavor = job->flavor;
- const char *consensus = job->consensus;
- size_t bodylen = job->consensus_len;
- config_line_t *labels = config_lines_dup(job->labels_in);
- const char *flavname = networkstatus_get_flavor_name(flavor);
- cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
- (const uint8_t *)consensus, bodylen);
- {
- const char *start, *end;
- if (router_get_networkstatus_v3_signed_boundaries(consensus,
- &start, &end) < 0) {
- start = consensus;
- end = consensus+bodylen;
- }
- cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
- (const uint8_t *)start,
- end - start);
- }
- config_line_prepend(&labels, LABEL_FLAVOR, flavname);
- config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
- compress_multiple(job->out,
- n_consensus_compression_methods(),
- compress_consensus_with,
- (const uint8_t*)consensus, bodylen, labels);
- config_free_lines(labels);
- return WQ_RPL_REPLY;
- }
- /**
- * Worker function: This function runs in the main thread, and receives
- * a consensus_diff_compress_job_t that the worker thread has already
- * processed.
- */
- static void
- consensus_compress_worker_replyfn(void *work_)
- {
- consensus_compress_worker_job_t *job = work_;
- consensus_cache_entry_handle_t *handles[
- ARRAY_LENGTH(compress_consensus_with)];
- memset(handles, 0, sizeof(handles));
- store_multiple(handles,
- n_consensus_compression_methods(),
- compress_consensus_with,
- job->out,
- "consensus");
- cdm_cache_dirty = 1;
- unsigned u;
- consensus_flavor_t f = job->flavor;
- tor_assert((int)f < N_CONSENSUS_FLAVORS);
- for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
- if (handles[u] == NULL)
- continue;
- consensus_cache_entry_handle_free(latest_consensus[f][u]);
- latest_consensus[f][u] = handles[u];
- }
- consensus_compress_worker_job_free(job);
- }
- /**
- * If true, we compress in worker threads.
- */
- static int background_compression = 0;
- /**
- * Queue a job to compress <b>consensus</b> and store its compressed
- * text in the cache.
- */
- static int
- consensus_queue_compression_work(const char *consensus,
- const networkstatus_t *as_parsed)
- {
- tor_assert(consensus);
- tor_assert(as_parsed);
- consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
- job->consensus = tor_strdup(consensus);
- job->consensus_len = strlen(consensus);
- job->flavor = as_parsed->flavor;
- char va_str[ISO_TIME_LEN+1];
- char vu_str[ISO_TIME_LEN+1];
- char fu_str[ISO_TIME_LEN+1];
- format_iso_time_nospace(va_str, as_parsed->valid_after);
- format_iso_time_nospace(fu_str, as_parsed->fresh_until);
- format_iso_time_nospace(vu_str, as_parsed->valid_until);
- config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
- config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
- config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
- if (as_parsed->voters) {
- smartlist_t *hexvoters = smartlist_new();
- SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
- networkstatus_voter_info_t *, vi) {
- if (smartlist_len(vi->sigs) == 0)
- continue; // didn't sign.
- char d[HEX_DIGEST_LEN+1];
- base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
- smartlist_add_strdup(hexvoters, d);
- } SMARTLIST_FOREACH_END(vi);
- char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
- config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
- tor_free(signers);
- SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
- smartlist_free(hexvoters);
- }
- if (background_compression) {
- workqueue_entry_t *work;
- work = cpuworker_queue_work(WQ_PRI_LOW,
- consensus_compress_worker_threadfn,
- consensus_compress_worker_replyfn,
- job);
- if (!work) {
- consensus_compress_worker_job_free(job);
- return -1;
- }
- return 0;
- } else {
- consensus_compress_worker_threadfn(NULL, job);
- consensus_compress_worker_replyfn(job);
- return 0;
- }
- }
- /**
- * Tell the consdiffmgr backend to compress consensuses in worker threads.
- */
- void
- consdiffmgr_enable_background_compression(void)
- {
- // This isn't the default behavior because it would break unit tests.
- background_compression = 1;
- }
- /** Read the set of voters from the cached object <b>ent</b> into
- * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
- * -1 if no signatories were recorded. */
- int
- consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
- smartlist_t *out)
- {
- tor_assert(ent);
- tor_assert(out);
- const char *s;
- s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
- if (s == NULL)
- return -1;
- smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
- return 0;
- }
- /** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
- * and return 0, or return -1 if no such time was recorded. */
- int
- consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
- time_t *out)
- {
- tor_assert(ent);
- tor_assert(out);
- const char *s;
- s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
- if (s == NULL || parse_iso_time_nospace(s, out) < 0)
- return -1;
- else
- return 0;
- }
- /** Read the valid until timestamp from the cached object <b>ent</b> into
- * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
- int
- consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
- time_t *out)
- {
- tor_assert(ent);
- tor_assert(out);
- const char *s;
- s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
- if (s == NULL || parse_iso_time_nospace(s, out) < 0)
- return -1;
- else
- return 0;
- }
- /** Read the valid after timestamp from the cached object <b>ent</b> into
- * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
- int
- consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
- time_t *out)
- {
- tor_assert(ent);
- tor_assert(out);
- const char *s;
- s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
- if (s == NULL || parse_iso_time_nospace(s, out) < 0)
- return -1;
- else
- return 0;
- }
|