test_oos.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. /* Copyright (c) 2016-2018, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /* Unit tests for OOS handler */
  4. #define CONNECTION_PRIVATE
  5. #include "core/or/or.h"
  6. #include "app/config/config.h"
  7. #include "core/mainloop/connection.h"
  8. #include "core/or/connection_or.h"
  9. #include "feature/dircommon/directory.h"
  10. #include "core/mainloop/mainloop.h"
  11. #include "test/test.h"
  12. #include "feature/dircommon/dir_connection_st.h"
  13. #include "core/or/or_connection_st.h"
  14. static or_options_t mock_options;
  15. static void
  16. reset_options_mock(void)
  17. {
  18. memset(&mock_options, 0, sizeof(or_options_t));
  19. }
  20. static const or_options_t *
  21. mock_get_options(void)
  22. {
  23. return &mock_options;
  24. }
  25. static int moribund_calls = 0;
  26. static int moribund_conns = 0;
  27. static int
  28. mock_connection_count_moribund(void)
  29. {
  30. ++moribund_calls;
  31. return moribund_conns;
  32. }
  33. /*
  34. * For unit test purposes it's sufficient to tell that
  35. * kill_conn_list_for_oos() was called with an approximately
  36. * sane argument; it's just the thing we returned from the
  37. * mock for pick_oos_victims().
  38. */
  39. static int kill_conn_list_calls = 0;
  40. static int kill_conn_list_killed = 0;
  41. static void
  42. kill_conn_list_mock(smartlist_t *conns)
  43. {
  44. ++kill_conn_list_calls;
  45. tt_ptr_op(conns, OP_NE, NULL);
  46. kill_conn_list_killed += smartlist_len(conns);
  47. done:
  48. return;
  49. }
  50. static int pick_oos_mock_calls = 0;
  51. static int pick_oos_mock_fail = 0;
  52. static int pick_oos_mock_last_n = 0;
  53. static smartlist_t *
  54. pick_oos_victims_mock(int n)
  55. {
  56. smartlist_t *l = NULL;
  57. int i;
  58. ++pick_oos_mock_calls;
  59. tt_int_op(n, OP_GT, 0);
  60. if (!pick_oos_mock_fail) {
  61. /*
  62. * connection_check_oos() just passes the list onto
  63. * kill_conn_list_for_oos(); we don't need to simulate
  64. * its content for this mock, just its existence, but
  65. * we do need to check the parameter.
  66. */
  67. l = smartlist_new();
  68. for (i = 0; i < n; ++i) smartlist_add(l, NULL);
  69. } else {
  70. l = NULL;
  71. }
  72. pick_oos_mock_last_n = n;
  73. done:
  74. return l;
  75. }
  76. /** Unit test for the logic in connection_check_oos(), which is concerned
  77. * with comparing thresholds and connection counts to decide if an OOS has
  78. * occurred and if so, how many connections to try to kill, and then using
  79. * pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim
  80. * duty.
  81. */
  82. static void
  83. test_oos_connection_check_oos(void *arg)
  84. {
  85. (void)arg;
  86. /* Set up mocks */
  87. reset_options_mock();
  88. /* OOS handling is only sensitive to these fields */
  89. mock_options.ConnLimit = 32;
  90. mock_options.ConnLimit_ = 64;
  91. mock_options.ConnLimit_high_thresh = 60;
  92. mock_options.ConnLimit_low_thresh = 50;
  93. MOCK(get_options, mock_get_options);
  94. moribund_calls = 0;
  95. moribund_conns = 0;
  96. MOCK(connection_count_moribund, mock_connection_count_moribund);
  97. kill_conn_list_calls = 0;
  98. kill_conn_list_killed = 0;
  99. MOCK(kill_conn_list_for_oos, kill_conn_list_mock);
  100. pick_oos_mock_calls = 0;
  101. pick_oos_mock_fail = 0;
  102. MOCK(pick_oos_victims, pick_oos_victims_mock);
  103. /* No OOS case */
  104. connection_check_oos(50, 0);
  105. tt_int_op(moribund_calls, OP_EQ, 0);
  106. tt_int_op(pick_oos_mock_calls, OP_EQ, 0);
  107. tt_int_op(kill_conn_list_calls, OP_EQ, 0);
  108. /* OOS from socket count, nothing moribund */
  109. connection_check_oos(62, 0);
  110. tt_int_op(moribund_calls, OP_EQ, 1);
  111. tt_int_op(pick_oos_mock_calls, OP_EQ, 1);
  112. /* 12 == 62 - ConnLimit_low_thresh */
  113. tt_int_op(pick_oos_mock_last_n, OP_EQ, 12);
  114. tt_int_op(kill_conn_list_calls, OP_EQ, 1);
  115. tt_int_op(kill_conn_list_killed, OP_EQ, 12);
  116. /* OOS from socket count, some are moribund */
  117. kill_conn_list_killed = 0;
  118. moribund_conns = 5;
  119. connection_check_oos(62, 0);
  120. tt_int_op(moribund_calls, OP_EQ, 2);
  121. tt_int_op(pick_oos_mock_calls, OP_EQ, 2);
  122. /* 7 == 62 - ConnLimit_low_thresh - moribund_conns */
  123. tt_int_op(pick_oos_mock_last_n, OP_EQ, 7);
  124. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  125. tt_int_op(kill_conn_list_killed, OP_EQ, 7);
  126. /* OOS from socket count, but pick fails */
  127. kill_conn_list_killed = 0;
  128. moribund_conns = 0;
  129. pick_oos_mock_fail = 1;
  130. connection_check_oos(62, 0);
  131. tt_int_op(moribund_calls, OP_EQ, 3);
  132. tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
  133. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  134. tt_int_op(kill_conn_list_killed, OP_EQ, 0);
  135. pick_oos_mock_fail = 0;
  136. /*
  137. * OOS from socket count with so many moribund conns
  138. * we have none to kill.
  139. */
  140. kill_conn_list_killed = 0;
  141. moribund_conns = 15;
  142. connection_check_oos(62, 0);
  143. tt_int_op(moribund_calls, OP_EQ, 4);
  144. tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
  145. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  146. /*
  147. * OOS from socket exhaustion; OOS handler will try to
  148. * kill 1/10 (5) of the connections.
  149. */
  150. kill_conn_list_killed = 0;
  151. moribund_conns = 0;
  152. connection_check_oos(50, 1);
  153. tt_int_op(moribund_calls, OP_EQ, 5);
  154. tt_int_op(pick_oos_mock_calls, OP_EQ, 4);
  155. tt_int_op(kill_conn_list_calls, OP_EQ, 3);
  156. tt_int_op(kill_conn_list_killed, OP_EQ, 5);
  157. /* OOS from socket exhaustion with moribund conns */
  158. kill_conn_list_killed = 0;
  159. moribund_conns = 2;
  160. connection_check_oos(50, 1);
  161. tt_int_op(moribund_calls, OP_EQ, 6);
  162. tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
  163. tt_int_op(kill_conn_list_calls, OP_EQ, 4);
  164. tt_int_op(kill_conn_list_killed, OP_EQ, 3);
  165. /* OOS from socket exhaustion with many moribund conns */
  166. kill_conn_list_killed = 0;
  167. moribund_conns = 7;
  168. connection_check_oos(50, 1);
  169. tt_int_op(moribund_calls, OP_EQ, 7);
  170. tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
  171. tt_int_op(kill_conn_list_calls, OP_EQ, 4);
  172. /* OOS with both socket exhaustion and above-threshold */
  173. kill_conn_list_killed = 0;
  174. moribund_conns = 0;
  175. connection_check_oos(62, 1);
  176. tt_int_op(moribund_calls, OP_EQ, 8);
  177. tt_int_op(pick_oos_mock_calls, OP_EQ, 6);
  178. tt_int_op(kill_conn_list_calls, OP_EQ, 5);
  179. tt_int_op(kill_conn_list_killed, OP_EQ, 12);
  180. /*
  181. * OOS with both socket exhaustion and above-threshold with some
  182. * moribund conns
  183. */
  184. kill_conn_list_killed = 0;
  185. moribund_conns = 5;
  186. connection_check_oos(62, 1);
  187. tt_int_op(moribund_calls, OP_EQ, 9);
  188. tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
  189. tt_int_op(kill_conn_list_calls, OP_EQ, 6);
  190. tt_int_op(kill_conn_list_killed, OP_EQ, 7);
  191. /*
  192. * OOS with both socket exhaustion and above-threshold with many
  193. * moribund conns
  194. */
  195. kill_conn_list_killed = 0;
  196. moribund_conns = 15;
  197. connection_check_oos(62, 1);
  198. tt_int_op(moribund_calls, OP_EQ, 10);
  199. tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
  200. tt_int_op(kill_conn_list_calls, OP_EQ, 6);
  201. done:
  202. UNMOCK(pick_oos_victims);
  203. UNMOCK(kill_conn_list_for_oos);
  204. UNMOCK(connection_count_moribund);
  205. UNMOCK(get_options);
  206. return;
  207. }
  208. static int cfe_calls = 0;
  209. static void
  210. close_for_error_mock(or_connection_t *orconn, int flush)
  211. {
  212. (void)flush;
  213. tt_ptr_op(orconn, OP_NE, NULL);
  214. ++cfe_calls;
  215. done:
  216. return;
  217. }
  218. static int mark_calls = 0;
  219. static void
  220. mark_for_close_oos_mock(connection_t *conn,
  221. int line, const char *file)
  222. {
  223. (void)line;
  224. (void)file;
  225. tt_ptr_op(conn, OP_NE, NULL);
  226. ++mark_calls;
  227. done:
  228. return;
  229. }
  230. static void
  231. test_oos_kill_conn_list(void *arg)
  232. {
  233. connection_t *c1, *c2;
  234. or_connection_t *or_c1 = NULL;
  235. dir_connection_t *dir_c2 = NULL;
  236. smartlist_t *l = NULL;
  237. (void)arg;
  238. /* Set up mocks */
  239. mark_calls = 0;
  240. MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock);
  241. cfe_calls = 0;
  242. MOCK(connection_or_close_for_error, close_for_error_mock);
  243. /* Make fake conns */
  244. or_c1 = tor_malloc_zero(sizeof(*or_c1));
  245. or_c1->base_.magic = OR_CONNECTION_MAGIC;
  246. or_c1->base_.type = CONN_TYPE_OR;
  247. c1 = TO_CONN(or_c1);
  248. dir_c2 = tor_malloc_zero(sizeof(*dir_c2));
  249. dir_c2->base_.magic = DIR_CONNECTION_MAGIC;
  250. dir_c2->base_.type = CONN_TYPE_DIR;
  251. dir_c2->base_.state = DIR_CONN_STATE_MIN_;
  252. dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
  253. c2 = TO_CONN(dir_c2);
  254. tt_ptr_op(c1, OP_NE, NULL);
  255. tt_ptr_op(c2, OP_NE, NULL);
  256. /* Make list */
  257. l = smartlist_new();
  258. smartlist_add(l, c1);
  259. smartlist_add(l, c2);
  260. /* Run kill_conn_list_for_oos() */
  261. kill_conn_list_for_oos(l);
  262. /* Check call counters */
  263. tt_int_op(mark_calls, OP_EQ, 1);
  264. tt_int_op(cfe_calls, OP_EQ, 1);
  265. done:
  266. UNMOCK(connection_or_close_for_error);
  267. UNMOCK(connection_mark_for_close_internal_);
  268. if (l) smartlist_free(l);
  269. tor_free(or_c1);
  270. tor_free(dir_c2);
  271. return;
  272. }
  273. static smartlist_t *conns_for_mock = NULL;
  274. static smartlist_t *
  275. get_conns_mock(void)
  276. {
  277. return conns_for_mock;
  278. }
  279. /*
  280. * For this mock, we pretend all conns have either zero or one circuits,
  281. * depending on if this appears on the list of things to say have a circuit.
  282. */
  283. static smartlist_t *conns_with_circs = NULL;
  284. static int
  285. get_num_circuits_mock(or_connection_t *conn)
  286. {
  287. int circs = 0;
  288. tt_ptr_op(conn, OP_NE, NULL);
  289. if (conns_with_circs &&
  290. smartlist_contains(conns_with_circs, TO_CONN(conn))) {
  291. circs = 1;
  292. }
  293. done:
  294. return circs;
  295. }
  296. static void
  297. test_oos_pick_oos_victims(void *arg)
  298. {
  299. (void)arg;
  300. or_connection_t *ortmp;
  301. dir_connection_t *dirtmp;
  302. smartlist_t *picked;
  303. /* Set up mocks */
  304. conns_for_mock = smartlist_new();
  305. MOCK(get_connection_array, get_conns_mock);
  306. conns_with_circs = smartlist_new();
  307. MOCK(connection_or_get_num_circuits, get_num_circuits_mock);
  308. /* Make some fake connections */
  309. ortmp = tor_malloc_zero(sizeof(*ortmp));
  310. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  311. ortmp->base_.type = CONN_TYPE_OR;
  312. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  313. /* We'll pretend this one has a circuit too */
  314. smartlist_add(conns_with_circs, TO_CONN(ortmp));
  315. /* Next one */
  316. ortmp = tor_malloc_zero(sizeof(*ortmp));
  317. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  318. ortmp->base_.type = CONN_TYPE_OR;
  319. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  320. /* Next one is moribund */
  321. ortmp = tor_malloc_zero(sizeof(*ortmp));
  322. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  323. ortmp->base_.type = CONN_TYPE_OR;
  324. ortmp->base_.marked_for_close = 1;
  325. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  326. /* Last one isn't an orconn */
  327. dirtmp = tor_malloc_zero(sizeof(*dirtmp));
  328. dirtmp->base_.magic = DIR_CONNECTION_MAGIC;
  329. dirtmp->base_.type = CONN_TYPE_DIR;
  330. smartlist_add(conns_for_mock, TO_CONN(dirtmp));
  331. /* Try picking one */
  332. picked = pick_oos_victims(1);
  333. /* It should be the one with circuits */
  334. tt_ptr_op(picked, OP_NE, NULL);
  335. tt_int_op(smartlist_len(picked), OP_EQ, 1);
  336. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  337. smartlist_free(picked);
  338. /* Try picking none */
  339. picked = pick_oos_victims(0);
  340. /* We should get an empty list */
  341. tt_ptr_op(picked, OP_NE, NULL);
  342. tt_int_op(smartlist_len(picked), OP_EQ, 0);
  343. smartlist_free(picked);
  344. /* Try picking two */
  345. picked = pick_oos_victims(2);
  346. /* We should get both active orconns */
  347. tt_ptr_op(picked, OP_NE, NULL);
  348. tt_int_op(smartlist_len(picked), OP_EQ, 2);
  349. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  350. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
  351. smartlist_free(picked);
  352. /* Try picking three - only two are eligible */
  353. picked = pick_oos_victims(3);
  354. tt_int_op(smartlist_len(picked), OP_EQ, 2);
  355. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  356. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
  357. smartlist_free(picked);
  358. done:
  359. /* Free leftover stuff */
  360. if (conns_with_circs) {
  361. smartlist_free(conns_with_circs);
  362. conns_with_circs = NULL;
  363. }
  364. UNMOCK(connection_or_get_num_circuits);
  365. if (conns_for_mock) {
  366. SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c));
  367. smartlist_free(conns_for_mock);
  368. conns_for_mock = NULL;
  369. }
  370. UNMOCK(get_connection_array);
  371. return;
  372. }
  373. struct testcase_t oos_tests[] = {
  374. { "connection_check_oos", test_oos_connection_check_oos,
  375. TT_FORK, NULL, NULL },
  376. { "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL },
  377. { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
  378. END_OF_TESTCASES
  379. };