test_consdiffmgr.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900
  1. /* Copyright (c) 2017-2019, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. #define CONSDIFFMGR_PRIVATE
  4. #include "core/or/or.h"
  5. #include "app/config/config.h"
  6. #include "feature/dircache/conscache.h"
  7. #include "feature/dircommon/consdiff.h"
  8. #include "feature/dircache/consdiffmgr.h"
  9. #include "core/mainloop/cpuworker.h"
  10. #include "lib/crypt_ops/crypto_rand.h"
  11. #include "feature/nodelist/networkstatus.h"
  12. #include "feature/dirparse/ns_parse.h"
  13. #include "lib/evloop/workqueue.h"
  14. #include "lib/compress/compress.h"
  15. #include "lib/encoding/confline.h"
  16. #include "feature/nodelist/networkstatus_st.h"
  17. #include "test/test.h"
  18. #include "test/log_test_helpers.h"
  19. // ============================== Setup/teardown the consdiffmgr
  20. // These functions get run before/after each test in this module
  21. static void *
  22. consdiffmgr_test_setup(const struct testcase_t *arg)
  23. {
  24. (void)arg;
  25. char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
  26. tor_free(get_options_mutable()->CacheDirectory);
  27. get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
  28. check_private_dir(ddir_fname, CPD_CREATE, NULL);
  29. consdiff_cfg_t consdiff_cfg = { 300 };
  30. consdiffmgr_configure(&consdiff_cfg);
  31. return (void *)1; // must return something non-null.
  32. }
  33. static int
  34. consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
  35. {
  36. (void)arg;
  37. (void)ignore;
  38. consdiffmgr_free_all();
  39. return 1;
  40. }
  41. static struct testcase_setup_t setup_diffmgr = {
  42. consdiffmgr_test_setup,
  43. consdiffmgr_test_teardown
  44. };
  45. // ============================== NS faking functions
  46. // These functions are for making quick fake consensus objects and
  47. // strings that are just good enough for consdiff and consdiffmgr.
  48. static networkstatus_t *
  49. fake_ns_new(consensus_flavor_t flav, time_t valid_after)
  50. {
  51. networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
  52. ns->type = NS_TYPE_CONSENSUS;
  53. ns->flavor = flav;
  54. ns->valid_after = valid_after;
  55. return ns;
  56. }
  57. static char *
  58. fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
  59. {
  60. const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
  61. char valid_after_string[ISO_TIME_LEN+1];
  62. format_iso_time(valid_after_string, valid_after);
  63. char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
  64. char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
  65. char *consensus;
  66. tor_asprintf(&consensus,
  67. "network-status-version 3%s\n"
  68. "vote-status consensus\n"
  69. "valid-after %s\n"
  70. "r name ccccccccccccccccc etc\nsample\n"
  71. "r name eeeeeeeeeeeeeeeee etc\nbar\n"
  72. "%s\n"
  73. "directory-signature hello-there\n"
  74. "directory-signature %s\n",
  75. flavor_string,
  76. valid_after_string,
  77. random_stuff,
  78. random_stuff2);
  79. tor_free(random_stuff);
  80. tor_free(random_stuff2);
  81. return consensus;
  82. }
  83. // ============================== Cpuworker mocking code
  84. // These mocking functions and types capture the cpuworker calls
  85. // so we can inspect them and run them in the main thread.
  86. static smartlist_t *fake_cpuworker_queue = NULL;
  87. typedef struct fake_work_queue_ent_t {
  88. enum workqueue_reply_t (*fn)(void *, void *);
  89. void (*reply_fn)(void *);
  90. void *arg;
  91. } fake_work_queue_ent_t;
  92. static struct workqueue_entry_s *
  93. mock_cpuworker_queue_work(workqueue_priority_t prio,
  94. enum workqueue_reply_t (*fn)(void *, void *),
  95. void (*reply_fn)(void *),
  96. void *arg)
  97. {
  98. (void) prio;
  99. if (! fake_cpuworker_queue)
  100. fake_cpuworker_queue = smartlist_new();
  101. fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
  102. ent->fn = fn;
  103. ent->reply_fn = reply_fn;
  104. ent->arg = arg;
  105. smartlist_add(fake_cpuworker_queue, ent);
  106. return (struct workqueue_entry_s *)ent;
  107. }
  108. static int
  109. mock_cpuworker_run_work(void)
  110. {
  111. if (! fake_cpuworker_queue)
  112. return 0;
  113. SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
  114. enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
  115. if (r != WQ_RPL_REPLY)
  116. return -1;
  117. });
  118. return 0;
  119. }
  120. static void
  121. mock_cpuworker_handle_replies(void)
  122. {
  123. if (! fake_cpuworker_queue)
  124. return;
  125. SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
  126. ent->reply_fn(ent->arg);
  127. tor_free(ent);
  128. });
  129. smartlist_free(fake_cpuworker_queue);
  130. fake_cpuworker_queue = NULL;
  131. }
  132. // ============================== Other helpers
  133. static consdiff_status_t
  134. lookup_diff_from(consensus_cache_entry_t **out,
  135. consensus_flavor_t flav,
  136. const char *str1)
  137. {
  138. uint8_t digest[DIGEST256_LEN];
  139. if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
  140. TT_FAIL(("Unable to compute sha3-as-signed"));
  141. return CONSDIFF_NOT_FOUND;
  142. }
  143. return consdiffmgr_find_diff_from(out, flav,
  144. DIGEST_SHA3_256, digest, sizeof(digest),
  145. NO_METHOD);
  146. }
  147. static int
  148. lookup_apply_and_verify_diff(consensus_flavor_t flav,
  149. const char *str1,
  150. const char *str2)
  151. {
  152. consensus_cache_entry_t *ent = NULL;
  153. consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
  154. if (ent == NULL || status != CONSDIFF_AVAILABLE) {
  155. return -1;
  156. }
  157. consensus_cache_entry_incref(ent);
  158. size_t size;
  159. char *diff_string = NULL;
  160. int r = uncompress_or_copy(&diff_string, &size, ent);
  161. consensus_cache_entry_decref(ent);
  162. if (diff_string == NULL || r < 0)
  163. return -1;
  164. char *applied = consensus_diff_apply(str1, diff_string);
  165. tor_free(diff_string);
  166. if (applied == NULL)
  167. return -1;
  168. int match = !strcmp(applied, str2);
  169. tor_free(applied);
  170. return match ? 0 : -1;
  171. }
  172. static void
  173. cdm_reload(void)
  174. {
  175. consdiffmgr_free_all();
  176. cdm_cache_get();
  177. consdiffmgr_rescan();
  178. }
  179. // ============================== Beginning of tests
  180. #if 0
  181. static int got_failure = 0;
  182. static void
  183. got_assertion_failure(void)
  184. {
  185. ++got_failure;
  186. }
  187. /* XXXX This test won't work, because there is currently no way to actually
  188. * XXXX capture a real assertion failure. */
  189. static void
  190. test_consdiffmgr_init_failure(void *arg)
  191. {
  192. (void)arg;
  193. // Capture assertions and bugs.
  194. /* As in ...test_setup, but do not create the datadir. The missing directory
  195. * will cause a failure. */
  196. char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
  197. tor_free(get_options_mutable()->CacheDirectory);
  198. get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
  199. consdiff_cfg_t consdiff_cfg = { 7200, 300 };
  200. tor_set_failed_assertion_callback(got_assertion_failure);
  201. tor_capture_bugs_(1);
  202. consdiffmgr_configure(&consdiff_cfg); // This should fail.
  203. tt_int_op(got_failure, OP_EQ, 1);
  204. const smartlist_t *bugs = tor_get_captured_bug_log_();
  205. tt_int_op(smartlist_len(bugs), OP_EQ, 1);
  206. done:
  207. tor_end_capture_bugs_();
  208. }
  209. #endif /* 0 */
  210. static void
  211. test_consdiffmgr_sha3_helper(void *arg)
  212. {
  213. (void) arg;
  214. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  215. config_line_t *lines = NULL;
  216. char *mem_op_hex_tmp = NULL;
  217. config_line_prepend(&lines, "good-sha",
  218. "F00DF00DF00DF00DF00DF00DF00DF00D"
  219. "F00DF00DF00DF00DF00DF00DF00DF00D");
  220. config_line_prepend(&lines, "short-sha",
  221. "F00DF00DF00DF00DF00DF00DF00DF00D"
  222. "F00DF00DF00DF00DF00DF00DF00DF0");
  223. config_line_prepend(&lines, "long-sha",
  224. "F00DF00DF00DF00DF00DF00DF00DF00D"
  225. "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
  226. config_line_prepend(&lines, "not-sha",
  227. "F00DF00DF00DF00DF00DF00DF00DF00D"
  228. "F00DF00DF00DF00DF00DF00DF00DXXXX");
  229. consensus_cache_entry_t *ent =
  230. consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  231. uint8_t buf[DIGEST256_LEN];
  232. tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
  233. tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
  234. test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
  235. "F00DF00DF00DF00DF00DF00DF00DF00D");
  236. tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
  237. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
  238. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
  239. tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
  240. done:
  241. consensus_cache_entry_decref(ent);
  242. config_free_lines(lines);
  243. tor_free(mem_op_hex_tmp);
  244. }
  245. static void
  246. test_consdiffmgr_add(void *arg)
  247. {
  248. (void) arg;
  249. time_t now = approx_time();
  250. char *body = NULL;
  251. consensus_cache_entry_t *ent = NULL;
  252. networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
  253. const char *dummy = "foo";
  254. int r = consdiffmgr_add_consensus(dummy, ns_tmp);
  255. tt_int_op(r, OP_EQ, 0);
  256. /* If we add it again, it won't work */
  257. setup_capture_of_logs(LOG_INFO);
  258. dummy = "bar";
  259. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  260. tt_int_op(r, OP_EQ, -1);
  261. expect_single_log_msg_containing("We already have a copy of that "
  262. "consensus");
  263. mock_clean_saved_logs();
  264. /* But it will work fine if the flavor is different */
  265. dummy = "baz";
  266. ns_tmp->flavor = FLAV_MICRODESC;
  267. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  268. tt_int_op(r, OP_EQ, 0);
  269. /* And it will work fine if the time is different */
  270. dummy = "quux";
  271. ns_tmp->flavor = FLAV_NS;
  272. ns_tmp->valid_after = now - 60;
  273. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  274. tt_int_op(r, OP_EQ, 0);
  275. /* If we add one a long long time ago, it will fail. */
  276. dummy = "xyzzy";
  277. ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
  278. r = consdiffmgr_add_consensus(dummy, ns_tmp);
  279. tt_int_op(r, OP_EQ, -1);
  280. expect_log_msg_containing("it's too old.");
  281. /* Try looking up a consensuses. */
  282. ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
  283. tt_assert(ent);
  284. consensus_cache_entry_incref(ent);
  285. size_t s;
  286. r = uncompress_or_copy(&body, &s, ent);
  287. tt_int_op(r, OP_EQ, 0);
  288. tt_int_op(s, OP_EQ, 4);
  289. tt_mem_op(body, OP_EQ, "quux", 4);
  290. /* Try looking up another entry, but fail */
  291. tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
  292. tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
  293. done:
  294. networkstatus_vote_free(ns_tmp);
  295. teardown_capture_of_logs();
  296. consensus_cache_entry_decref(ent);
  297. tor_free(body);
  298. }
  299. static void
  300. test_consdiffmgr_make_diffs(void *arg)
  301. {
  302. (void)arg;
  303. networkstatus_t *ns = NULL;
  304. char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
  305. char *applied = NULL, *diff_text = NULL;
  306. time_t now = approx_time();
  307. int r;
  308. consensus_cache_entry_t *diff = NULL;
  309. uint8_t md_ns_sha3[DIGEST256_LEN];
  310. consdiff_status_t diff_status;
  311. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  312. // Try rescan with no consensuses: shouldn't crash or queue work.
  313. consdiffmgr_rescan();
  314. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  315. // Make two consensuses, 1 hour sec ago.
  316. ns = fake_ns_new(FLAV_NS, now-3600);
  317. ns_body = fake_ns_body_new(FLAV_NS, now-3600);
  318. r = consdiffmgr_add_consensus(ns_body, ns);
  319. networkstatus_vote_free(ns);
  320. tor_free(ns_body);
  321. tt_int_op(r, OP_EQ, 0);
  322. ns = fake_ns_new(FLAV_MICRODESC, now-3600);
  323. md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
  324. r = consdiffmgr_add_consensus(md_ns_body, ns);
  325. router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
  326. networkstatus_vote_free(ns);
  327. tt_int_op(r, OP_EQ, 0);
  328. // No diffs will be generated.
  329. consdiffmgr_rescan();
  330. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  331. // Add a MD consensus from 45 minutes ago. This should cause one diff
  332. // worth of work to get queued.
  333. ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
  334. md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
  335. r = consdiffmgr_add_consensus(md_ns_body_2, ns);
  336. networkstatus_vote_free(ns);
  337. tt_int_op(r, OP_EQ, 0);
  338. consdiffmgr_rescan();
  339. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  340. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  341. diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
  342. DIGEST_SHA3_256,
  343. md_ns_sha3, DIGEST256_LEN,
  344. NO_METHOD);
  345. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
  346. // Now run that process and get the diff.
  347. r = mock_cpuworker_run_work();
  348. tt_int_op(r, OP_EQ, 0);
  349. mock_cpuworker_handle_replies();
  350. // At this point we should be able to get that diff.
  351. diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
  352. DIGEST_SHA3_256,
  353. md_ns_sha3, DIGEST256_LEN,
  354. NO_METHOD);
  355. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
  356. tt_assert(diff);
  357. /* Make sure applying the diff actually works */
  358. const uint8_t *diff_body;
  359. size_t diff_size;
  360. r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
  361. tt_int_op(r, OP_EQ, 0);
  362. diff_text = tor_memdup_nulterm(diff_body, diff_size);
  363. applied = consensus_diff_apply(md_ns_body, diff_text);
  364. tt_assert(applied);
  365. tt_str_op(applied, OP_EQ, md_ns_body_2);
  366. /* Rescan again: no more work to do. */
  367. consdiffmgr_rescan();
  368. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  369. done:
  370. tor_free(md_ns_body);
  371. tor_free(md_ns_body_2);
  372. tor_free(diff_text);
  373. tor_free(applied);
  374. }
  375. static void
  376. test_consdiffmgr_diff_rules(void *arg)
  377. {
  378. (void)arg;
  379. #define N 6
  380. char *md_body[N], *ns_body[N];
  381. networkstatus_t *md_ns[N], *ns_ns[N];
  382. int i;
  383. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  384. /* Create a bunch of consensus things at 15-second intervals. */
  385. time_t start = approx_time() - 120;
  386. for (i = 0; i < N; ++i) {
  387. time_t when = start + i * 15;
  388. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  389. ns_body[i] = fake_ns_body_new(FLAV_NS, when);
  390. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  391. ns_ns[i] = fake_ns_new(FLAV_NS, when);
  392. }
  393. /* For the MD consensuses: add 4 of them, and make sure that
  394. * diffs are created to one consensus (the most recent) only. */
  395. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  396. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  397. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
  398. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
  399. consdiffmgr_rescan();
  400. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  401. tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
  402. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  403. mock_cpuworker_handle_replies();
  404. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  405. /* For the NS consensuses: add 3, generate, and add one older one and
  406. * make sure that older one is the only one whose diff is generated */
  407. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
  408. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
  409. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
  410. consdiffmgr_rescan();
  411. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  412. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  413. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  414. mock_cpuworker_handle_replies();
  415. /* At this point, we should actually have working diffs! */
  416. tt_int_op(0, OP_EQ,
  417. lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
  418. tt_int_op(0, OP_EQ,
  419. lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
  420. tt_int_op(0, OP_EQ,
  421. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
  422. tt_int_op(0, OP_EQ,
  423. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
  424. tt_int_op(0, OP_EQ,
  425. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
  426. /* Self-to-self diff won't be present */
  427. consensus_cache_entry_t *ent;
  428. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  429. lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
  430. /* No diff from 2 has been added yet */
  431. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  432. lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
  433. /* No diff arriving at old things. */
  434. tt_int_op(-1, OP_EQ,
  435. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  436. /* No backwards diff */
  437. tt_int_op(-1, OP_EQ,
  438. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
  439. /* Now, an update: add number 2 and make sure it's the only one whose diff
  440. * is regenerated. */
  441. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
  442. consdiffmgr_rescan();
  443. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  444. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  445. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  446. mock_cpuworker_handle_replies();
  447. tt_int_op(0, OP_EQ,
  448. lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
  449. /* Finally: reload, and make sure that the information is still indexed */
  450. cdm_reload();
  451. tt_int_op(0, OP_EQ,
  452. lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
  453. tt_int_op(0, OP_EQ,
  454. lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
  455. tt_int_op(0, OP_EQ,
  456. lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
  457. tt_int_op(0, OP_EQ,
  458. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
  459. tt_int_op(0, OP_EQ,
  460. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
  461. tt_int_op(0, OP_EQ,
  462. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
  463. done:
  464. for (i = 0; i < N; ++i) {
  465. tor_free(md_body[i]);
  466. tor_free(ns_body[i]);
  467. networkstatus_vote_free(md_ns[i]);
  468. networkstatus_vote_free(ns_ns[i]);
  469. }
  470. UNMOCK(cpuworker_queue_work);
  471. #undef N
  472. }
  473. static void
  474. test_consdiffmgr_diff_failure(void *arg)
  475. {
  476. (void)arg;
  477. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  478. /* We're going to make sure that if we have a bogus request where
  479. * we can't actually compute a diff, the world must not end. */
  480. networkstatus_t *ns1 = NULL;
  481. networkstatus_t *ns2 = NULL;
  482. int r;
  483. ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
  484. ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
  485. r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
  486. tt_int_op(r, OP_EQ, 0);
  487. // We refuse to compute a diff to or from a line holding only a single dot.
  488. // We can add it here, though.
  489. r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
  490. tt_int_op(r, OP_EQ, 0);
  491. consdiffmgr_rescan();
  492. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  493. setup_capture_of_logs(LOG_WARN);
  494. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  495. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  496. expect_single_log_msg_containing("one of the lines to be added is \".\".");
  497. mock_clean_saved_logs();
  498. mock_cpuworker_handle_replies();
  499. expect_single_log_msg_containing("Worker was unable to compute consensus "
  500. "diff from ");
  501. /* Make sure the diff is not present */
  502. consensus_cache_entry_t *ent;
  503. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  504. lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
  505. done:
  506. teardown_capture_of_logs();
  507. UNMOCK(cpuworker_queue_work);
  508. networkstatus_vote_free(ns1);
  509. networkstatus_vote_free(ns2);
  510. }
  511. static void
  512. test_consdiffmgr_diff_pending(void *arg)
  513. {
  514. #define N 3
  515. (void)arg;
  516. char *md_body[N];
  517. networkstatus_t *md_ns[N];
  518. time_t start = approx_time() - 120;
  519. int i;
  520. for (i = 0; i < N; ++i) {
  521. time_t when = start + i * 30;
  522. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  523. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  524. }
  525. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  526. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  527. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  528. /* Make a diff */
  529. consdiffmgr_rescan();
  530. tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
  531. /* Look it up. Is it pending? */
  532. consensus_cache_entry_t *ent = NULL;
  533. consdiff_status_t diff_status;
  534. diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
  535. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
  536. tt_ptr_op(ent, OP_EQ, NULL);
  537. /* Add another old consensus. only one new diff should launch! */
  538. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
  539. consdiffmgr_rescan();
  540. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  541. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  542. mock_cpuworker_handle_replies();
  543. tt_int_op(0, OP_EQ,
  544. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
  545. tt_int_op(0, OP_EQ,
  546. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  547. done:
  548. UNMOCK(cpuworker_queue_work);
  549. for (i = 0; i < N; ++i) {
  550. tor_free(md_body[i]);
  551. networkstatus_vote_free(md_ns[i]);
  552. }
  553. #undef N
  554. }
  555. static void
  556. test_consdiffmgr_cleanup_old(void *arg)
  557. {
  558. (void)arg;
  559. config_line_t *labels = NULL;
  560. consensus_cache_entry_t *ent = NULL;
  561. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  562. /* This item will be will be cleanable because it has a valid-after
  563. * time far in the past. */
  564. config_line_prepend(&labels, "document-type", "confribble-blarg");
  565. config_line_prepend(&labels, "consensus-valid-after",
  566. "1980-10-10T10:10:10");
  567. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  568. tt_assert(ent);
  569. consensus_cache_entry_decref(ent);
  570. setup_capture_of_logs(LOG_DEBUG);
  571. tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
  572. expect_log_msg_containing("Deleting entry because its consensus-valid-"
  573. "after value (1980-10-10T10:10:10) was too old");
  574. done:
  575. teardown_capture_of_logs();
  576. config_free_lines(labels);
  577. }
  578. static void
  579. test_consdiffmgr_cleanup_bad_valid_after(void *arg)
  580. {
  581. /* This will seem cleanable, but isn't, because its valid-after time is
  582. * misformed. */
  583. (void)arg;
  584. config_line_t *labels = NULL;
  585. consensus_cache_entry_t *ent = NULL;
  586. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  587. config_line_prepend(&labels, "document-type", "consensus");
  588. config_line_prepend(&labels, "consensus-valid-after",
  589. "whan that aprille with his shoures soote"); // (~1385?)
  590. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  591. tt_assert(ent);
  592. consensus_cache_entry_decref(ent);
  593. setup_capture_of_logs(LOG_DEBUG);
  594. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  595. expect_log_msg_containing("Ignoring entry because its consensus-valid-"
  596. "after value (\"whan that aprille with his "
  597. "shoures soote\") was unparseable");
  598. done:
  599. teardown_capture_of_logs();
  600. config_free_lines(labels);
  601. }
  602. static void
  603. test_consdiffmgr_cleanup_no_valid_after(void *arg)
  604. {
  605. (void)arg;
  606. config_line_t *labels = NULL;
  607. consensus_cache_entry_t *ent = NULL;
  608. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  609. /* This item will be will be uncleanable because it has no recognized
  610. * valid-after. */
  611. config_line_prepend(&labels, "document-type", "consensus");
  612. config_line_prepend(&labels, "confrooble-voolid-oofter",
  613. "2010-10-10T09:08:07");
  614. ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
  615. tt_assert(ent);
  616. consensus_cache_entry_decref(ent);
  617. setup_capture_of_logs(LOG_DEBUG);
  618. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  619. expect_log_msg_containing("Ignoring entry because it had no consensus-"
  620. "valid-after label");
  621. done:
  622. teardown_capture_of_logs();
  623. config_free_lines(labels);
  624. }
  625. static void
  626. test_consdiffmgr_cleanup_old_diffs(void *arg)
  627. {
  628. (void)arg;
  629. #define N 4
  630. char *md_body[N];
  631. networkstatus_t *md_ns[N];
  632. int i;
  633. consensus_cache_entry_t *hold_ent = NULL, *ent;
  634. /* Make sure that the cleanup function removes diffs to the not-most-recent
  635. * consensus. */
  636. MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
  637. /* Create a bunch of consensus things at 15-second intervals. */
  638. time_t start = approx_time() - 120;
  639. for (i = 0; i < N; ++i) {
  640. time_t when = start + i * 15;
  641. md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
  642. md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
  643. }
  644. /* add the first 3. */
  645. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
  646. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
  647. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
  648. /* Make diffs. */
  649. consdiffmgr_rescan();
  650. tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
  651. tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
  652. tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
  653. mock_cpuworker_handle_replies();
  654. tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
  655. /* Nothing is deletable now */
  656. tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
  657. tt_int_op(0, OP_EQ,
  658. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
  659. tt_int_op(0, OP_EQ,
  660. lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
  661. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
  662. lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
  663. consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
  664. /* Now add an even-more-recent consensus; this should make all previous
  665. * diffs deletable, and make delete */
  666. tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
  667. tt_int_op(2 * n_diff_compression_methods() +
  668. (n_consensus_compression_methods() - 1) , OP_EQ,
  669. consdiffmgr_cleanup());
  670. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  671. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
  672. /* This one is marked deletable but still in the hashtable */
  673. tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
  674. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
  675. tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
  676. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
  677. /* Everything should be valid at this point */
  678. tt_int_op(0, OP_EQ, consdiffmgr_validate());
  679. /* And if we recan NOW, we'll purge the hashtable of the entries,
  680. * and launch attempts to generate new ones */
  681. consdiffmgr_rescan();
  682. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  683. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
  684. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  685. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
  686. tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
  687. lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
  688. /* We're still holding on to this, though, so we can still map it! */
  689. const uint8_t *t1 = NULL;
  690. size_t s;
  691. int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
  692. tt_int_op(r, OP_EQ, 0);
  693. tt_assert(t1);
  694. done:
  695. for (i = 0; i < N; ++i) {
  696. tor_free(md_body[i]);
  697. networkstatus_vote_free(md_ns[i]);
  698. }
  699. consensus_cache_entry_decref(hold_ent);
  700. UNMOCK(cpuworker_queue_work);
  701. #undef N
  702. }
  703. static void
  704. test_consdiffmgr_validate(void *arg)
  705. {
  706. (void)arg;
  707. config_line_t *lines = NULL;
  708. consensus_cache_entry_t *ent = NULL;
  709. consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
  710. smartlist_t *vals = smartlist_new();
  711. /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
  712. * one with a wrong sha3, and one with no sha3. */
  713. config_line_prepend(&lines, "id", "wrong sha3");
  714. config_line_prepend(&lines, "sha3-digest",
  715. "F00DF00DF00DF00DF00DF00DF00DF00D"
  716. "F00DF00DF00DF00DF00DF00DF00DF00D");
  717. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  718. consensus_cache_entry_decref(ent);
  719. config_free_lines(lines);
  720. lines = NULL;
  721. config_line_prepend(&lines, "id", "bad sha3");
  722. config_line_prepend(&lines, "sha3-digest",
  723. "now is the winter of our dicotheque");
  724. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  725. consensus_cache_entry_decref(ent);
  726. config_free_lines(lines);
  727. lines = NULL;
  728. config_line_prepend(&lines, "id", "no sha3");
  729. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  730. consensus_cache_entry_decref(ent);
  731. config_free_lines(lines);
  732. lines = NULL;
  733. config_line_prepend(&lines, "id", "good sha3");
  734. config_line_prepend(&lines, "sha3-digest",
  735. "8d8b1998616cd6b4c4055da8d38728dc"
  736. "93c758d4131a53c7d81aa6337dee1c05");
  737. ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
  738. consensus_cache_entry_decref(ent);
  739. config_free_lines(lines);
  740. lines = NULL;
  741. cdm_reload();
  742. cache = cdm_cache_get();
  743. tt_int_op(1, OP_EQ, consdiffmgr_validate());
  744. consensus_cache_find_all(vals, cache, "id", "good sha3");
  745. tt_int_op(smartlist_len(vals), OP_EQ, 1);
  746. smartlist_clear(vals);
  747. consensus_cache_find_all(vals, cache, "id", "no sha3");
  748. tt_int_op(smartlist_len(vals), OP_EQ, 1);
  749. smartlist_clear(vals);
  750. consensus_cache_find_all(vals, cache, "id", "wrong sha3");
  751. tt_int_op(smartlist_len(vals), OP_EQ, 0);
  752. consensus_cache_find_all(vals, cache, "id", "bad sha3");
  753. tt_int_op(smartlist_len(vals), OP_EQ, 0);
  754. done:
  755. smartlist_free(vals);
  756. }
  757. #define TEST(name) \
  758. { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
  759. struct testcase_t consdiffmgr_tests[] = {
  760. #if 0
  761. { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
  762. #endif
  763. TEST(sha3_helper),
  764. TEST(add),
  765. TEST(make_diffs),
  766. TEST(diff_rules),
  767. TEST(diff_failure),
  768. TEST(diff_pending),
  769. TEST(cleanup_old),
  770. TEST(cleanup_bad_valid_after),
  771. TEST(cleanup_no_valid_after),
  772. TEST(cleanup_old_diffs),
  773. TEST(validate),
  774. // XXXX Test: non-cacheing cases of replyfn().
  775. END_OF_TESTCASES
  776. };