test_consdiffmgr.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899
  1. /* Copyright (c) 2017-2018, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. #define CONSDIFFMGR_PRIVATE
  4. #include "or/or.h"
  5. #include "or/config.h"
  6. #include "or/conscache.h"
  7. #include "or/consdiff.h"
  8. #include "or/consdiffmgr.h"
  9. #include "or/cpuworker.h"
  10. #include "common/crypto_rand.h"
  11. #include "or/networkstatus.h"
  12. #include "or/routerparse.h"
  13. #include "common/workqueue.h"
  14. #include "or/networkstatus_st.h"
  15. #include "test/test.h"
  16. #include "test/log_test_helpers.h"
  17. // ============================== Setup/teardown the consdiffmgr
  18. // These functions get run before/after each test in this module
  19. static void *
  20. consdiffmgr_test_setup(const struct testcase_t *arg)
  21. {
  22. (void)arg;
  23. char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
  24. tor_free(get_options_mutable()->CacheDirectory);
  25. get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
  26. check_private_dir(ddir_fname, CPD_CREATE, NULL);
  27. consdiff_cfg_t consdiff_cfg = { 300 };
  28. consdiffmgr_configure(&consdiff_cfg);
  29. return (void *)1; // must return something non-null.
  30. }
  31. static int
  32. consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
  33. {
  34. (void)arg;
  35. (void)ignore;
  36. consdiffmgr_free_all();
  37. return 1;
  38. }
  39. static struct testcase_setup_t setup_diffmgr = {
  40. consdiffmgr_test_setup,
  41. consdiffmgr_test_teardown
  42. };
  43. // ============================== NS faking functions
  44. // These functions are for making quick fake consensus objects and
  45. // strings that are just good enough for consdiff and consdiffmgr.
  46. static networkstatus_t *
  47. fake_ns_new(consensus_flavor_t flav, time_t valid_after)
  48. {
  49. networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
  50. ns->type = NS_TYPE_CONSENSUS;
  51. ns->flavor = flav;
  52. ns->valid_after = valid_after;
  53. return ns;
  54. }
  55. static char *
  56. fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
  57. {
  58. const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
  59. char valid_after_string[ISO_TIME_LEN+1];
  60. format_iso_time(valid_after_string, valid_after);
  61. char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
  62. char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
  63. char *consensus;
  64. tor_asprintf(&consensus,
  65. "network-status-version 3%s\n"
  66. "vote-status consensus\n"
  67. "valid-after %s\n"
  68. "r name ccccccccccccccccc etc\nsample\n"
  69. "r name eeeeeeeeeeeeeeeee etc\nbar\n"
  70. "%s\n"
  71. "directory-signature hello-there\n"
  72. "directory-signature %s\n",
  73. flavor_string,
  74. valid_after_string,
  75. random_stuff,
  76. random_stuff2);
  77. tor_free(random_stuff);
  78. tor_free(random_stuff2);
  79. return consensus;
  80. }
  81. // ============================== Cpuworker mocking code
  82. // These mocking functions and types capture the cpuworker calls
  83. // so we can inspect them and run them in the main thread.
  84. static smartlist_t *fake_cpuworker_queue = NULL;
  85. typedef struct fake_work_queue_ent_t {
  86. enum workqueue_reply_t (*fn)(void *, void *);
  87. void (*reply_fn)(void *);
  88. void *arg;
  89. } fake_work_queue_ent_t;
  90. static struct workqueue_entry_s *
  91. mock_cpuworker_queue_work(workqueue_priority_t prio,
  92. enum workqueue_reply_t (*fn)(void *, void *),
  93. void (*reply_fn)(void *),
  94. void *arg)
  95. {
  96. (void) prio;
  97. if (! fake_cpuworker_queue)
  98. fake_cpuworker_queue = smartlist_new();
  99. fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
  100. ent->fn = fn;
  101. ent->reply_fn = reply_fn;
  102. ent->arg = arg;
  103. smartlist_add(fake_cpuworker_queue, ent);
  104. return (struct workqueue_entry_s *)ent;
  105. }
  106. static int
  107. mock_cpuworker_run_work(void)
  108. {
  109. if (! fake_cpuworker_queue)
  110. return 0;
  111. SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
  112. enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
  113. if (r != WQ_RPL_REPLY)
  114. return -1;
  115. });
  116. return 0;
  117. }
  118. static void
  119. mock_cpuworker_handle_replies(void)
  120. {
  121. if (! fake_cpuworker_queue)
  122. return;
  123. SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
  124. ent->reply_fn(ent->arg);
  125. tor_free(ent);
  126. });
  127. smartlist_free(fake_cpuworker_queue);
  128. fake_cpuworker_queue = NULL;
  129. }
  130. // ============================== Other helpers
  131. static consdiff_status_t
  132. lookup_diff_from(consensus_cache_entry_t **out,
  133. consensus_flavor_t flav,
  134. const char *str1)
  135. {
  136. uint8_t digest[DIGEST256_LEN];
  137. if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
  138. TT_FAIL(("Unable to compute sha3-as-signed"));
  139. return CONSDIFF_NOT_FOUND;
  140. }
  141. return consdiffmgr_find_diff_from(out, flav,
  142. DIGEST_SHA3_256, digest, sizeof(digest),
  143. NO_METHOD);
  144. }
  145. static int
  146. lookup_apply_and_verify_diff(consensus_flavor_t flav,
  147. const char *str1,
  148. const char *str2)
  149. {
  150. consensus_cache_entry_t *ent = NULL;
  151. consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
  152. if (ent == NULL || status != CONSDIFF_AVAILABLE) {
  153. return -1;
  154. }
  155. consensus_cache_entry_incref(ent);
  156. size_t size;
  157. char *diff_string = NULL;
  158. int r = uncompress_or_copy(&diff_string, &size, ent);
  159. consensus_cache_entry_decref(ent);
  160. if (diff_string == NULL || r < 0)
  161. return -1;
  162. char *applied = consensus_diff_apply(str1, diff_string);
  163. tor_free(diff_string);
  164. if (applied == NULL)
  165. return -1;
  166. int match = !strcmp(applied, str2);
  167. tor_free(applied);
  168. return match ? 0 : -1;
  169. }
  170. static void
  171. cdm_reload(void)
  172. {
  173. consdiffmgr_free_all();
  174. cdm_cache_get();
  175. consdiffmgr_rescan();
  176. }
  177. // ============================== Beginning of tests
  178. #if 0
  179. static int got_failure = 0;
  180. static void
  181. got_assertion_failure(void)
  182. {
  183. ++got_failure;
  184. }
  185. /* XXXX This test won't work, because there is currently no way to actually
  186. * XXXX capture a real assertion failure. */
  187. static void
  188. test_consdiffmgr_init_failure(void *arg)
  189. {
  190. (void)arg;
  191. // Capture assertions and bugs.
  192. /* As in ...test_setup, but do not create the datadir. The missing directory
  193. * will cause a failure. */
  194. char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
  195. tor_free(get_options_mutable()->CacheDirectory);
  196. get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
  197. consdiff_cfg_t consdiff_cfg = { 7200, 300 };
  198. tor_set_failed_assertion_callback(got_assertion_failure);
  199. tor_capture_bugs_(1);
  200. consdiffmgr_configure(&consdiff_cfg); // This should fail.
  201. tt_int_op(got_failure, OP_EQ, 1);
  202. const smartlist_t *bugs = tor_get_captured_bug_log_();
  203. tt_int_op(smartlist_len(bugs), OP_EQ, 1);
  204. done:
  205. tor_end_capture_bugs_();
  206. }
  207. #endif /* 0 */
  208. static void
  209. test_consdiffmgr_sha3_helper(void *arg)
  210. {
  211. (void) arg;
  212. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  213. config_line_t *lines = NULL;
  214. char *mem_op_hex_tmp = NULL;
  215. config_line_prepend(&lines, "good-sha",
  216. "F00DF00DF00DF00DF00DF00DF00DF00D"
  217. "F00DF00DF00DF00DF00DF00DF00DF00D");
  218. config_line_prepend(&lines, "short-sha",
  219. "F00DF00DF00DF00DF00DF00DF00DF00D"
  220. "F00DF00DF00DF00DF00DF00DF00DF0");
  221. config_line_prepend(&lines, "long-sha",
  222. "F00DF00DF00DF00DF00DF00DF00DF00D"
  223. "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
  224. config_line_prepend(&lines, "not-sha",
  225. "F00DF00DF00DF00DF00DF00DF00DF00D"
  226. "F00DF00DF00DF00DF00DF00DF00DXXXX");
  227. consensus_cache_entry_t *ent =
  228. consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  229. uint8_t buf[DIGEST256_LEN];
  230. tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
  231. tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
  232. test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
  233. "F00DF00DF00DF00DF00DF00DF00DF00D");
  234. tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
  235. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
  236. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
  237. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
  238. done:
  239. consensus_cache_entry_decref(ent);
  240. config_free_lines(lines);
  241. tor_free(mem_op_hex_tmp);
  242. }
  243. static void
  244. test_consdiffmgr_add(void *arg)
  245. {
  246. (void) arg;
  247. time_t now = approx_time();
  248. char *body = NULL;
  249. consensus_cache_entry_t *ent = NULL;
  250. networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
  251. const char *dummy = "foo";
  252. int r = consdiffmgr_add_consensus(dummy, ns_tmp);
  253. tt_int_op(r, OP_EQ, 0);
  254. /* If we add it again, it won't work */
  255. setup_capture_of_logs(LOG_INFO);
  256. dummy = "bar";
  257. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  258. tt_int_op(r, OP_EQ, -1);
  259. expect_single_log_msg_containing("We already have a copy of that "
  260. "consensus");
  261. mock_clean_saved_logs();
  262. /* But it will work fine if the flavor is different */
  263. dummy = "baz";
  264. ns_tmp->flavor = FLAV_MICRODESC;
  265. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  266. tt_int_op(r, OP_EQ, 0);
  267. /* And it will work fine if the time is different */
  268. dummy = "quux";
  269. ns_tmp->flavor = FLAV_NS;
  270. ns_tmp->valid_after = now - 60;
  271. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  272. tt_int_op(r, OP_EQ, 0);
  273. /* If we add one a long long time ago, it will fail. */
  274. dummy = "xyzzy";
  275. ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
  276. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  277. tt_int_op(r, OP_EQ, -1);
  278. expect_log_msg_containing("it's too old.");
  279. /* Try looking up a consensuses. */
  280. ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
  281. tt_assert(ent);
  282. consensus_cache_entry_incref(ent);
  283. size_t s;
  284. r = uncompress_or_copy(&body, &s, ent);
  285. tt_int_op(r, OP_EQ, 0);
  286. tt_int_op(s, OP_EQ, 4);
  287. tt_mem_op(body, OP_EQ, "quux", 4);
  288. /* Try looking up another entry, but fail */
  289. tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
  290. tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
  291. done:
  292. networkstatus_vote_free(ns_tmp);
  293. teardown_capture_of_logs();
  294. consensus_cache_entry_decref(ent);
  295. tor_free(body);
  296. }
  297. static void
  298. test_consdiffmgr_make_diffs(void *arg)
  299. {
  300. (void)arg;
  301. networkstatus_t *ns = NULL;
  302. char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
  303. char *applied = NULL, *diff_text = NULL;
  304. time_t now = approx_time();
  305. int r;
  306. consensus_cache_entry_t *diff = NULL;
  307. uint8_t md_ns_sha3[DIGEST256_LEN];
  308. consdiff_status_t diff_status;
  309. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  310. // Try rescan with no consensuses: shouldn't crash or queue work.
  311. consdiffmgr_rescan();
  312. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  313. // Make two consensuses, 1 hour sec ago.
  314. ns = fake_ns_new(FLAV_NS, now-3600);
  315. ns_body = fake_ns_body_new(FLAV_NS, now-3600);
  316. r = consdiffmgr_add_consensus(ns_body, ns);
  317. networkstatus_vote_free(ns);
  318. tor_free(ns_body);
  319. tt_int_op(r, OP_EQ, 0);
  320. ns = fake_ns_new(FLAV_MICRODESC, now-3600);
  321. md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
  322. r = consdiffmgr_add_consensus(md_ns_body, ns);
  323. router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
  324. networkstatus_vote_free(ns);
  325. tt_int_op(r, OP_EQ, 0);
  326. // No diffs will be generated.
  327. consdiffmgr_rescan();
  328. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  329. // Add a MD consensus from 45 minutes ago. This should cause one diff
  330. // worth of work to get queued.
  331. ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
  332. md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
  333. r = consdiffmgr_add_consensus(md_ns_body_2, ns);
  334. networkstatus_vote_free(ns);
  335. tt_int_op(r, OP_EQ, 0);
  336. consdiffmgr_rescan();
  337. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  338. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  339. diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
  340. DIGEST_SHA3_256,
  341. md_ns_sha3, DIGEST256_LEN,
  342. NO_METHOD);
  343. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
  344. // Now run that process and get the diff.
  345. r = mock_cpuworker_run_work();
  346. tt_int_op(r, OP_EQ, 0);
  347. mock_cpuworker_handle_replies();
  348. // At this point we should be able to get that diff.
  349. diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
  350. DIGEST_SHA3_256,
  351. md_ns_sha3, DIGEST256_LEN,
  352. NO_METHOD);
  353. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
  354. tt_assert(diff);
  355. /* Make sure applying the diff actually works */
  356. const uint8_t *diff_body;
  357. size_t diff_size;
  358. r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
  359. tt_int_op(r, OP_EQ, 0);
  360. diff_text = tor_memdup_nulterm(diff_body, diff_size);
  361. applied = consensus_diff_apply(md_ns_body, diff_text);
  362. tt_assert(applied);
  363. tt_str_op(applied, OP_EQ, md_ns_body_2);
  364. /* Rescan again: no more work to do. */
  365. consdiffmgr_rescan();
  366. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  367. done:
  368. tor_free(md_ns_body);
  369. tor_free(md_ns_body_2);
  370. tor_free(diff_text);
  371. tor_free(applied);
  372. }
  373. static void
  374. test_consdiffmgr_diff_rules(void *arg)
  375. {
  376. (void)arg;
  377. #define N 6
  378. char *md_body[N], *ns_body[N];
  379. networkstatus_t *md_ns[N], *ns_ns[N];
  380. int i;
  381. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  382. /* Create a bunch of consensus things at 15-second intervals. */
  383. time_t start = approx_time() - 120;
  384. for (i = 0; i < N; ++i) {
  385. time_t when = start + i * 15;
  386. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  387. ns_body[i] = fake_ns_body_new(FLAV_NS, when);
  388. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  389. ns_ns[i] = fake_ns_new(FLAV_NS, when);
  390. }
  391. /* For the MD consensuses: add 4 of them, and make sure that
  392. * diffs are created to one consensus (the most recent) only. */
  393. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  394. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  395. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
  396. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
  397. consdiffmgr_rescan();
  398. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  399. tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
  400. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  401. mock_cpuworker_handle_replies();
  402. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  403. /* For the NS consensuses: add 3, generate, and add one older one and
  404. * make sure that older one is the only one whose diff is generated */
  405. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
  406. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
  407. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
  408. consdiffmgr_rescan();
  409. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  410. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  411. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  412. mock_cpuworker_handle_replies();
  413. /* At this point, we should actually have working diffs! */
  414. tt_int_op(0, OP_EQ,
  415. lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
  416. tt_int_op(0, OP_EQ,
  417. lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
  418. tt_int_op(0, OP_EQ,
  419. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
  420. tt_int_op(0, OP_EQ,
  421. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
  422. tt_int_op(0, OP_EQ,
  423. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
  424. /* Self-to-self diff won't be present */
  425. consensus_cache_entry_t *ent;
  426. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  427. lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
  428. /* No diff from 2 has been added yet */
  429. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  430. lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
  431. /* No diff arriving at old things. */
  432. tt_int_op(-1, OP_EQ,
  433. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  434. /* No backwards diff */
  435. tt_int_op(-1, OP_EQ,
  436. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
  437. /* Now, an update: add number 2 and make sure it's the only one whose diff
  438. * is regenerated. */
  439. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
  440. consdiffmgr_rescan();
  441. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  442. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  443. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  444. mock_cpuworker_handle_replies();
  445. tt_int_op(0, OP_EQ,
  446. lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
  447. /* Finally: reload, and make sure that the information is still indexed */
  448. cdm_reload();
  449. tt_int_op(0, OP_EQ,
  450. lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
  451. tt_int_op(0, OP_EQ,
  452. lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
  453. tt_int_op(0, OP_EQ,
  454. lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
  455. tt_int_op(0, OP_EQ,
  456. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
  457. tt_int_op(0, OP_EQ,
  458. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
  459. tt_int_op(0, OP_EQ,
  460. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
  461. done:
  462. for (i = 0; i < N; ++i) {
  463. tor_free(md_body[i]);
  464. tor_free(ns_body[i]);
  465. networkstatus_vote_free(md_ns[i]);
  466. networkstatus_vote_free(ns_ns[i]);
  467. }
  468. UNMOCK(cpuworker_queue_work);
  469. #undef N
  470. }
  471. static void
  472. test_consdiffmgr_diff_failure(void *arg)
  473. {
  474. (void)arg;
  475. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  476. /* We're going to make sure that if we have a bogus request where
  477. * we can't actually compute a diff, the world must not end. */
  478. networkstatus_t *ns1 = NULL;
  479. networkstatus_t *ns2 = NULL;
  480. int r;
  481. ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
  482. ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
  483. r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
  484. tt_int_op(r, OP_EQ, 0);
  485. // We refuse to compute a diff to or from a line holding only a single dot.
  486. // We can add it here, though.
  487. r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
  488. tt_int_op(r, OP_EQ, 0);
  489. consdiffmgr_rescan();
  490. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  491. setup_capture_of_logs(LOG_WARN);
  492. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  493. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  494. expect_single_log_msg_containing("one of the lines to be added is \".\".");
  495. mock_clean_saved_logs();
  496. mock_cpuworker_handle_replies();
  497. expect_single_log_msg_containing("Worker was unable to compute consensus "
  498. "diff from ");
  499. /* Make sure the diff is not present */
  500. consensus_cache_entry_t *ent;
  501. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  502. lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
  503. done:
  504. teardown_capture_of_logs();
  505. UNMOCK(cpuworker_queue_work);
  506. networkstatus_vote_free(ns1);
  507. networkstatus_vote_free(ns2);
  508. }
  509. static void
  510. test_consdiffmgr_diff_pending(void *arg)
  511. {
  512. #define N 3
  513. (void)arg;
  514. char *md_body[N];
  515. networkstatus_t *md_ns[N];
  516. time_t start = approx_time() - 120;
  517. int i;
  518. for (i = 0; i < N; ++i) {
  519. time_t when = start + i * 30;
  520. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  521. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  522. }
  523. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  524. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  525. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  526. /* Make a diff */
  527. consdiffmgr_rescan();
  528. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  529. /* Look it up. Is it pending? */
  530. consensus_cache_entry_t *ent = NULL;
  531. consdiff_status_t diff_status;
  532. diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
  533. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
  534. tt_ptr_op(ent, OP_EQ, NULL);
  535. /* Add another old consensus. only one new diff should launch! */
  536. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
  537. consdiffmgr_rescan();
  538. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  539. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  540. mock_cpuworker_handle_replies();
  541. tt_int_op(0, OP_EQ,
  542. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
  543. tt_int_op(0, OP_EQ,
  544. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  545. done:
  546. UNMOCK(cpuworker_queue_work);
  547. for (i = 0; i < N; ++i) {
  548. tor_free(md_body[i]);
  549. networkstatus_vote_free(md_ns[i]);
  550. }
  551. #undef N
  552. }
  553. static void
  554. test_consdiffmgr_cleanup_old(void *arg)
  555. {
  556. (void)arg;
  557. config_line_t *labels = NULL;
  558. consensus_cache_entry_t *ent = NULL;
  559. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  560. /* This item will be will be cleanable because it has a valid-after
  561. * time far in the past. */
  562. config_line_prepend(&labels, "document-type", "confribble-blarg");
  563. config_line_prepend(&labels, "consensus-valid-after",
  564. "1980-10-10T10:10:10");
  565. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  566. tt_assert(ent);
  567. consensus_cache_entry_decref(ent);
  568. setup_capture_of_logs(LOG_DEBUG);
  569. tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
  570. expect_log_msg_containing("Deleting entry because its consensus-valid-"
  571. "after value (1980-10-10T10:10:10) was too old");
  572. done:
  573. teardown_capture_of_logs();
  574. config_free_lines(labels);
  575. }
  576. static void
  577. test_consdiffmgr_cleanup_bad_valid_after(void *arg)
  578. {
  579. /* This will seem cleanable, but isn't, because its valid-after time is
  580. * misformed. */
  581. (void)arg;
  582. config_line_t *labels = NULL;
  583. consensus_cache_entry_t *ent = NULL;
  584. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  585. config_line_prepend(&labels, "document-type", "consensus");
  586. config_line_prepend(&labels, "consensus-valid-after",
  587. "whan that aprille with his shoures soote"); // (~1385?)
  588. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  589. tt_assert(ent);
  590. consensus_cache_entry_decref(ent);
  591. setup_capture_of_logs(LOG_DEBUG);
  592. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  593. expect_log_msg_containing("Ignoring entry because its consensus-valid-"
  594. "after value (\"whan that aprille with his "
  595. "shoures soote\") was unparseable");
  596. done:
  597. teardown_capture_of_logs();
  598. config_free_lines(labels);
  599. }
  600. static void
  601. test_consdiffmgr_cleanup_no_valid_after(void *arg)
  602. {
  603. (void)arg;
  604. config_line_t *labels = NULL;
  605. consensus_cache_entry_t *ent = NULL;
  606. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  607. /* This item will be will be uncleanable because it has no recognized
  608. * valid-after. */
  609. config_line_prepend(&labels, "document-type", "consensus");
  610. config_line_prepend(&labels, "confrooble-voolid-oofter",
  611. "2010-10-10T09:08:07");
  612. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  613. tt_assert(ent);
  614. consensus_cache_entry_decref(ent);
  615. setup_capture_of_logs(LOG_DEBUG);
  616. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  617. expect_log_msg_containing("Ignoring entry because it had no consensus-"
  618. "valid-after label");
  619. done:
  620. teardown_capture_of_logs();
  621. config_free_lines(labels);
  622. }
  623. static void
  624. test_consdiffmgr_cleanup_old_diffs(void *arg)
  625. {
  626. (void)arg;
  627. #define N 4
  628. char *md_body[N];
  629. networkstatus_t *md_ns[N];
  630. int i;
  631. consensus_cache_entry_t *hold_ent = NULL, *ent;
  632. /* Make sure that the cleanup function removes diffs to the not-most-recent
  633. * consensus. */
  634. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  635. /* Create a bunch of consensus things at 15-second intervals. */
  636. time_t start = approx_time() - 120;
  637. for (i = 0; i < N; ++i) {
  638. time_t when = start + i * 15;
  639. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  640. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  641. }
  642. /* add the first 3. */
  643. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
  644. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  645. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  646. /* Make diffs. */
  647. consdiffmgr_rescan();
  648. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  649. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  650. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  651. mock_cpuworker_handle_replies();
  652. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  653. /* Nothing is deletable now */
  654. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  655. tt_int_op(0, OP_EQ,
  656. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
  657. tt_int_op(0, OP_EQ,
  658. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  659. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
  660. lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
  661. consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
  662. /* Now add an even-more-recent consensus; this should make all previous
  663. * diffs deletable, and make delete */
  664. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
  665. tt_int_op(2 * n_diff_compression_methods() +
  666. (n_consensus_compression_methods() - 1) , OP_EQ,
  667. consdiffmgr_cleanup());
  668. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  669. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
  670. /* This one is marked deletable but still in the hashtable */
  671. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
  672. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
  673. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  674. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
  675. /* Everything should be valid at this point */
  676. tt_int_op(0, OP_EQ, consdiffmgr_validate());
  677. /* And if we recan NOW, we'll purge the hashtable of the entries,
  678. * and launch attempts to generate new ones */
  679. consdiffmgr_rescan();
  680. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  681. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
  682. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  683. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
  684. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  685. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
  686. /* We're still holding on to this, though, so we can still map it! */
  687. const uint8_t *t1 = NULL;
  688. size_t s;
  689. int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
  690. tt_int_op(r, OP_EQ, 0);
  691. tt_assert(t1);
  692. done:
  693. for (i = 0; i < N; ++i) {
  694. tor_free(md_body[i]);
  695. networkstatus_vote_free(md_ns[i]);
  696. }
  697. consensus_cache_entry_decref(hold_ent);
  698. UNMOCK(cpuworker_queue_work);
  699. #undef N
  700. }
  701. static void
  702. test_consdiffmgr_validate(void *arg)
  703. {
  704. (void)arg;
  705. config_line_t *lines = NULL;
  706. consensus_cache_entry_t *ent = NULL;
  707. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  708. smartlist_t *vals = smartlist_new();
  709. /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
  710. * one with a wrong sha3, and one with no sha3. */
  711. config_line_prepend(&lines, "id", "wrong sha3");
  712. config_line_prepend(&lines, "sha3-digest",
  713. "F00DF00DF00DF00DF00DF00DF00DF00D"
  714. "F00DF00DF00DF00DF00DF00DF00DF00D");
  715. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  716. consensus_cache_entry_decref(ent);
  717. config_free_lines(lines);
  718. lines = NULL;
  719. config_line_prepend(&lines, "id", "bad sha3");
  720. config_line_prepend(&lines, "sha3-digest",
  721. "now is the winter of our dicotheque");
  722. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  723. consensus_cache_entry_decref(ent);
  724. config_free_lines(lines);
  725. lines = NULL;
  726. config_line_prepend(&lines, "id", "no sha3");
  727. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  728. consensus_cache_entry_decref(ent);
  729. config_free_lines(lines);
  730. lines = NULL;
  731. config_line_prepend(&lines, "id", "good sha3");
  732. config_line_prepend(&lines, "sha3-digest",
  733. "8d8b1998616cd6b4c4055da8d38728dc"
  734. "93c758d4131a53c7d81aa6337dee1c05");
  735. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  736. consensus_cache_entry_decref(ent);
  737. config_free_lines(lines);
  738. lines = NULL;
  739. cdm_reload();
  740. cache = cdm_cache_get();
  741. tt_int_op(1, OP_EQ, consdiffmgr_validate());
  742. consensus_cache_find_all(vals, cache, "id", "good sha3");
  743. tt_int_op(smartlist_len(vals), OP_EQ, 1);
  744. smartlist_clear(vals);
  745. consensus_cache_find_all(vals, cache, "id", "no sha3");
  746. tt_int_op(smartlist_len(vals), OP_EQ, 1);
  747. smartlist_clear(vals);
  748. consensus_cache_find_all(vals, cache, "id", "wrong sha3");
  749. tt_int_op(smartlist_len(vals), OP_EQ, 0);
  750. consensus_cache_find_all(vals, cache, "id", "bad sha3");
  751. tt_int_op(smartlist_len(vals), OP_EQ, 0);
  752. done:
  753. smartlist_free(vals);
  754. }
  755. #define TEST(name) \
  756. { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
  757. struct testcase_t consdiffmgr_tests[] = {
  758. #if 0
  759. { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
  760. #endif
  761. TEST(sha3_helper),
  762. TEST(add),
  763. TEST(make_diffs),
  764. TEST(diff_rules),
  765. TEST(diff_failure),
  766. TEST(diff_pending),
  767. TEST(cleanup_old),
  768. TEST(cleanup_bad_valid_after),
  769. TEST(cleanup_no_valid_after),
  770. TEST(cleanup_old_diffs),
  771. TEST(validate),
  772. // XXXX Test: non-cacheing cases of replyfn().
  773. END_OF_TESTCASES
  774. };