/* Copyright (c) 2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CONSDIFFMGR_PRIVATE #include "or.h" #include "config.h" #include "conscache.h" #include "consdiff.h" #include "consdiffmgr.h" #include "cpuworker.h" #include "networkstatus.h" #include "workqueue.h" #include "test.h" #include "log_test_helpers.h" // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module static void * consdiffmgr_test_setup(const struct testcase_t *arg) { (void)arg; char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm")); tor_free(get_options_mutable()->DataDirectory); get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer. check_private_dir(ddir_fname, CPD_CREATE, NULL); consdiff_cfg_t consdiff_cfg = { 7200, 300 }; consdiffmgr_configure(&consdiff_cfg); return (void *)1; // must return something non-null. } static int consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore) { (void)arg; (void)ignore; consdiffmgr_free_all(); return 1; } static struct testcase_setup_t setup_diffmgr = { consdiffmgr_test_setup, consdiffmgr_test_teardown }; // ============================== NS faking functions // These functions are for making quick fake consensus objects and // strings that are just good enough for consdiff and consdiffmgr. static networkstatus_t * fake_ns_new(consensus_flavor_t flav, time_t valid_after) { networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t)); ns->type = NS_TYPE_CONSENSUS; ns->flavor = flav; ns->valid_after = valid_after; return ns; } static char * fake_ns_body_new(consensus_flavor_t flav, time_t valid_after) { const char *flavor_string = flav == FLAV_NS ? "" : " microdesc"; char valid_after_string[ISO_TIME_LEN+1]; format_iso_time(valid_after_string, valid_after); char *random_stuff = crypto_random_hostname(3, 25, "junk ", ""); char *consensus; tor_asprintf(&consensus, "network-status-version 3%s\n" "vote-status consensus\n" "valid-after %s\n" "r name ccccccccccccccccc etc\nsample\n" "r name eeeeeeeeeeeeeeeee etc\nbar\n" "%s\n", flavor_string, valid_after_string, random_stuff); tor_free(random_stuff); return consensus; } // ============================== Cpuworker mocking code // These mocking functions and types capture the cpuworker calls // so we can inspect them and run them in the main thread. static smartlist_t *fake_cpuworker_queue = NULL; typedef struct fake_work_queue_ent_t { enum workqueue_reply_t (*fn)(void *, void *); void (*reply_fn)(void *); void *arg; } fake_work_queue_ent_t; static struct workqueue_entry_s * mock_cpuworker_queue_work(enum workqueue_reply_t (*fn)(void *, void *), void (*reply_fn)(void *), void *arg) { if (! fake_cpuworker_queue) fake_cpuworker_queue = smartlist_new(); fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent)); ent->fn = fn; ent->reply_fn = reply_fn; ent->arg = arg; smartlist_add(fake_cpuworker_queue, ent); return (struct workqueue_entry_s *)ent; } static int mock_cpuworker_run_work(void) { if (! fake_cpuworker_queue) return 0; SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, { enum workqueue_reply_t r = ent->fn(NULL, ent->arg); if (r != WQ_RPL_REPLY) return -1; }); return 0; } static void mock_cpuworker_handle_replies(void) { if (! fake_cpuworker_queue) return; SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, { ent->reply_fn(ent->arg); }); smartlist_free(fake_cpuworker_queue); fake_cpuworker_queue = NULL; } // ============================== Beginning of tests #if 0 static int got_failure = 0; static void got_assertion_failure(void) { ++got_failure; } /* XXXX This test won't work, because there is currently no way to actually * XXXX capture a real assertion failure. */ static void test_consdiffmgr_init_failure(void *arg) { (void)arg; // Capture assertions and bugs. /* As in ...test_setup, but do not create the datadir. The missing directory * will cause a failure. */ char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm")); tor_free(get_options_mutable()->DataDirectory); get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer. consdiff_cfg_t consdiff_cfg = { 7200, 300 }; tor_set_failed_assertion_callback(got_assertion_failure); tor_capture_bugs_(1); consdiffmgr_configure(&consdiff_cfg); // This should fail. tt_int_op(got_failure, OP_EQ, 1); const smartlist_t *bugs = tor_get_captured_bug_log_(); tt_int_op(smartlist_len(bugs), OP_EQ, 1); done: tor_end_capture_bugs_(); } #endif static void test_consdiffmgr_add(void *arg) { (void) arg; time_t now = approx_time(); consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); const char *dummy = "foo"; int r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, 0); /* If we add it again, it won't work */ setup_capture_of_logs(LOG_INFO); dummy = "bar"; r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, -1); expect_single_log_msg_containing("We already have a copy of that " "consensus"); mock_clean_saved_logs(); /* But it will work fine if the flavor is different */ dummy = "baz"; ns_tmp->flavor = FLAV_MICRODESC; r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, 0); /* And it will work fine if the time is different */ dummy = "quux"; ns_tmp->flavor = FLAV_NS; ns_tmp->valid_after = now - 60; r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, 0); /* If we add one a long long time ago, it will fail. */ dummy = "xyzzy"; ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */ r = consdiffmgr_add_consensus(dummy, ns_tmp); tt_int_op(r, OP_EQ, -1); expect_single_log_msg_containing("it's too old."); /* Try looking up a consensuses. */ ent = cdm_cache_lookup_consensus(FLAV_NS, now-60); tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; const uint8_t *body; r = consensus_cache_entry_get_body(ent, &body, &s); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); /* Try looking up another entry, but fail */ tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_MICRODESC, now-60)); tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_NS, now-61)); done: networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); } static void test_consdiffmgr_make_diffs(void *arg) { (void)arg; networkstatus_t *ns = NULL; char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL; char *applied = NULL, *diff_text = NULL; time_t now = approx_time(); int r; consensus_cache_entry_t *diff = NULL; uint8_t md_ns_sha3[DIGEST256_LEN]; consdiff_status_t diff_status; MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); // Try rescan with no consensuses: shouldn't crash or queue work. consdiffmgr_rescan(); tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); // Make two consensuses, 1 hour sec ago. ns = fake_ns_new(FLAV_NS, now-3600); ns_body = fake_ns_body_new(FLAV_NS, now-3600); r = consdiffmgr_add_consensus(ns_body, ns); networkstatus_vote_free(ns); tor_free(ns_body); tt_int_op(r, OP_EQ, 0); ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); crypto_digest256((char*)md_ns_sha3, md_ns_body, strlen(md_ns_body), DIGEST_SHA3_256); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); // No diffs will be generated. consdiffmgr_rescan(); tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); // Add a MD consensus from 45 minutes ago. This should cause one diff // worth of work to get queued. ns = fake_ns_new(FLAV_MICRODESC, now-45*60); md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60); r = consdiffmgr_add_consensus(md_ns_body_2, ns); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); // XXXX not working yet /* diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC, DIGEST_SHA3_256, md_ns_sha3, DIGEST256_LEN); tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status); */ // Now run that process and get the diff. r = mock_cpuworker_run_work(); tt_int_op(r, OP_EQ, 0); mock_cpuworker_handle_replies(); // At this point we should be able to get that diff. diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC, DIGEST_SHA3_256, md_ns_sha3, DIGEST256_LEN); tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status); tt_assert(diff); /* Make sure applying the diff actually works */ const uint8_t *diff_body; size_t diff_size; r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); applied = consensus_diff_apply(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); /* Rescan again: no more work to do. */ consdiffmgr_rescan(); tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); done: tor_free(md_ns_body); tor_free(md_ns_body_2); tor_free(diff_text); tor_free(applied); } static void test_consdiffmgr_diff_rules(void *arg) { (void)arg; #define N 6 char *md_body[N], *ns_body[N]; networkstatus_t *md_ns[N], *ns_ns[N]; uint8_t md_ns_sha3[N][DIGEST256_LEN], ns_ns_sha3[N][DIGEST256_LEN]; int i; MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); /* Create a bunch of consensus things at 15-second intervals. */ time_t start = approx_time() - 120; for (i = 0; i < N; ++i) { time_t when = start + i * 15; md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when); ns_body[i] = fake_ns_body_new(FLAV_NS, when); md_ns[i] = fake_ns_new(FLAV_MICRODESC, when); ns_ns[i] = fake_ns_new(FLAV_NS, when); crypto_digest256((char *)md_ns_sha3[i], md_body[i], strlen(md_body[i]), DIGEST_SHA3_256); crypto_digest256((char *)ns_ns_sha3[i], ns_body[i], strlen(ns_body[i]), DIGEST_SHA3_256); } /* For the MD consensuses: add 4 of them, and make sure that * diffs are created to one consensus (the most recent) only. */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4])); consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue)); tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); mock_cpuworker_handle_replies(); tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); /* For the NS consensuses: add 3, generate, and add one older one and * make sure that older one is the only one whose diff is generated */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5])); consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue)); tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); mock_cpuworker_handle_replies(); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2])); consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); mock_cpuworker_handle_replies(); done: for (i = 0; i < N; ++i) { tor_free(md_body[i]); tor_free(ns_body[i]); networkstatus_vote_free(md_ns[i]); networkstatus_vote_free(ns_ns[i]); } UNMOCK(cpuworker_queue_work); #undef N } static void test_consdiffmgr_diff_failure(void *arg) { (void)arg; MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); /* We're going to make sure that if we have a bogus request where * we can't actually compute a diff, the world must not end. */ networkstatus_t *ns1 = NULL; networkstatus_t *ns2 = NULL; int r; ns1 = fake_ns_new(FLAV_NS, approx_time()-100); ns2 = fake_ns_new(FLAV_NS, approx_time()-50); r = consdiffmgr_add_consensus("foo bar baz\n", ns1); tt_int_op(r, OP_EQ, 0); // We refuse to compute a diff to or from a line holding only a single dot. // We can add it here, though. r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2); tt_int_op(r, OP_EQ, 0); consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); setup_capture_of_logs(LOG_WARN); tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue)); tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); expect_single_log_msg_containing("one of the lines to be added is \".\"."); mock_clean_saved_logs(); mock_cpuworker_handle_replies(); expect_single_log_msg_containing("Worker was unable to compute consensus " "diff from "); done: teardown_capture_of_logs(); UNMOCK(cpuworker_queue_work); networkstatus_vote_free(ns1); networkstatus_vote_free(ns2); } static void test_consdiffmgr_cleanup_old(void *arg) { (void)arg; config_line_t *labels = NULL; consensus_cache_entry_t *ent = NULL; consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier /* This item will be will be cleanable because it has a valid-after * time far in the past. */ config_line_prepend(&labels, "document-type", "confribble-blarg"); config_line_prepend(&labels, "consensus-valid-after", "1980-10-10T10:10:10"); ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); tt_assert(ent); consensus_cache_entry_decref(ent); setup_capture_of_logs(LOG_DEBUG); tt_int_op(1, OP_EQ, consdiffmgr_cleanup()); expect_log_msg_containing("Deleting entry because its consensus-valid-" "after value (1980-10-10T10:10:10) was too old"); done: teardown_capture_of_logs(); config_free_lines(labels); } static void test_consdiffmgr_cleanup_bad_valid_after(void *arg) { /* This will seem cleanable, but isn't, because its valid-after time is * misformed. */ (void)arg; config_line_t *labels = NULL; consensus_cache_entry_t *ent = NULL; consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier config_line_prepend(&labels, "document-type", "consensus"); config_line_prepend(&labels, "consensus-valid-after", "whan that aprille with his shoures soote"); // (~1385?) ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); tt_assert(ent); consensus_cache_entry_decref(ent); setup_capture_of_logs(LOG_DEBUG); tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); expect_log_msg_containing("Ignoring entry because its consensus-valid-" "after value (\"whan that aprille with his " "shoures soote\") was unparseable"); done: teardown_capture_of_logs(); config_free_lines(labels); } static void test_consdiffmgr_cleanup_no_valid_after(void *arg) { (void)arg; config_line_t *labels = NULL; consensus_cache_entry_t *ent = NULL; consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier /* This item will be will be uncleanable because it has no recognized * valid-after. */ config_line_prepend(&labels, "document-type", "consensus"); config_line_prepend(&labels, "confrooble-voolid-oofter", "2010-10-10T09:08:07"); ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3); tt_assert(ent); consensus_cache_entry_decref(ent); setup_capture_of_logs(LOG_DEBUG); tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); expect_log_msg_containing("Ignoring entry because it had no consensus-" "valid-after label"); done: teardown_capture_of_logs(); config_free_lines(labels); } static void test_consdiffmgr_cleanup_old_diffs(void *arg) { (void)arg; #define N 4 char *md_body[N]; networkstatus_t *md_ns[N]; uint8_t md_ns_sha3[N][DIGEST256_LEN]; int i; /* Make sure that the cleanup function removes diffs to the not-most-recent * consensus. */ MOCK(cpuworker_queue_work, mock_cpuworker_queue_work); /* Create a bunch of consensus things at 15-second intervals. */ time_t start = approx_time() - 120; for (i = 0; i < N; ++i) { time_t when = start + i * 15; md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when); md_ns[i] = fake_ns_new(FLAV_MICRODESC, when); crypto_digest256((char *)md_ns_sha3[i], md_body[i], strlen(md_body[i]), DIGEST_SHA3_256); } /* add the first 3. */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1])); tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2])); /* Make diffs. */ consdiffmgr_rescan(); tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue); tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue)); tt_int_op(0, OP_EQ, mock_cpuworker_run_work()); mock_cpuworker_handle_replies(); tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue); /* Nothing is deletable now */ tt_int_op(0, OP_EQ, consdiffmgr_cleanup()); /* Now add an even-more-recent consensus; this should make all previous * diffs deletable */ tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3])); tt_int_op(2, OP_EQ, consdiffmgr_cleanup()); done: for (i = 0; i < N; ++i) { tor_free(md_body[i]); networkstatus_vote_free(md_ns[i]); } UNMOCK(cpuworker_queue_work); #undef N } #define TEST(name) \ { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL } struct testcase_t consdiffmgr_tests[] = { #if 0 { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL }, #endif TEST(add), TEST(make_diffs), TEST(diff_rules), TEST(diff_failure), TEST(cleanup_old), TEST(cleanup_bad_valid_after), TEST(cleanup_no_valid_after), TEST(cleanup_old_diffs), END_OF_TESTCASES };