test_oos.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. /* Copyright (c) 2016-2017, The Tor Project, Inc. */
  2. /* See LICENSE for licensing information */
  3. /* Unit tests for OOS handler */
  4. #define CONNECTION_PRIVATE
  5. #include "or.h"
  6. #include "config.h"
  7. #include "connection.h"
  8. #include "connection_or.h"
  9. #include "main.h"
  10. #include "test.h"
  11. static or_options_t mock_options;
  12. static void
  13. reset_options_mock(void)
  14. {
  15. memset(&mock_options, 0, sizeof(or_options_t));
  16. }
  17. static const or_options_t *
  18. mock_get_options(void)
  19. {
  20. return &mock_options;
  21. }
  22. static int moribund_calls = 0;
  23. static int moribund_conns = 0;
  24. static int
  25. mock_connection_count_moribund(void)
  26. {
  27. ++moribund_calls;
  28. return moribund_conns;
  29. }
  30. /*
  31. * For unit test purposes it's sufficient to tell that
  32. * kill_conn_list_for_oos() was called with an approximately
  33. * sane argument; it's just the thing we returned from the
  34. * mock for pick_oos_victims().
  35. */
  36. static int kill_conn_list_calls = 0;
  37. static int kill_conn_list_killed = 0;
  38. static void
  39. kill_conn_list_mock(smartlist_t *conns)
  40. {
  41. ++kill_conn_list_calls;
  42. tt_ptr_op(conns, OP_NE, NULL);
  43. kill_conn_list_killed += smartlist_len(conns);
  44. done:
  45. return;
  46. }
  47. static int pick_oos_mock_calls = 0;
  48. static int pick_oos_mock_fail = 0;
  49. static int pick_oos_mock_last_n = 0;
  50. static smartlist_t *
  51. pick_oos_victims_mock(int n)
  52. {
  53. smartlist_t *l = NULL;
  54. int i;
  55. ++pick_oos_mock_calls;
  56. tt_int_op(n, OP_GT, 0);
  57. if (!pick_oos_mock_fail) {
  58. /*
  59. * connection_check_oos() just passes the list onto
  60. * kill_conn_list_for_oos(); we don't need to simulate
  61. * its content for this mock, just its existence, but
  62. * we do need to check the parameter.
  63. */
  64. l = smartlist_new();
  65. for (i = 0; i < n; ++i) smartlist_add(l, NULL);
  66. } else {
  67. l = NULL;
  68. }
  69. pick_oos_mock_last_n = n;
  70. done:
  71. return l;
  72. }
  73. /** Unit test for the logic in connection_check_oos(), which is concerned
  74. * with comparing thresholds and connection counts to decide if an OOS has
  75. * occurred and if so, how many connections to try to kill, and then using
  76. * pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim
  77. * duty.
  78. */
  79. static void
  80. test_oos_connection_check_oos(void *arg)
  81. {
  82. (void)arg;
  83. /* Set up mocks */
  84. reset_options_mock();
  85. /* OOS handling is only sensitive to these fields */
  86. mock_options.ConnLimit = 32;
  87. mock_options.ConnLimit_ = 64;
  88. mock_options.ConnLimit_high_thresh = 60;
  89. mock_options.ConnLimit_low_thresh = 50;
  90. MOCK(get_options, mock_get_options);
  91. moribund_calls = 0;
  92. moribund_conns = 0;
  93. MOCK(connection_count_moribund, mock_connection_count_moribund);
  94. kill_conn_list_calls = 0;
  95. kill_conn_list_killed = 0;
  96. MOCK(kill_conn_list_for_oos, kill_conn_list_mock);
  97. pick_oos_mock_calls = 0;
  98. pick_oos_mock_fail = 0;
  99. MOCK(pick_oos_victims, pick_oos_victims_mock);
  100. /* No OOS case */
  101. connection_check_oos(50, 0);
  102. tt_int_op(moribund_calls, OP_EQ, 0);
  103. tt_int_op(pick_oos_mock_calls, OP_EQ, 0);
  104. tt_int_op(kill_conn_list_calls, OP_EQ, 0);
  105. /* OOS from socket count, nothing moribund */
  106. connection_check_oos(62, 0);
  107. tt_int_op(moribund_calls, OP_EQ, 1);
  108. tt_int_op(pick_oos_mock_calls, OP_EQ, 1);
  109. /* 12 == 62 - ConnLimit_low_thresh */
  110. tt_int_op(pick_oos_mock_last_n, OP_EQ, 12);
  111. tt_int_op(kill_conn_list_calls, OP_EQ, 1);
  112. tt_int_op(kill_conn_list_killed, OP_EQ, 12);
  113. /* OOS from socket count, some are moribund */
  114. kill_conn_list_killed = 0;
  115. moribund_conns = 5;
  116. connection_check_oos(62, 0);
  117. tt_int_op(moribund_calls, OP_EQ, 2);
  118. tt_int_op(pick_oos_mock_calls, OP_EQ, 2);
  119. /* 7 == 62 - ConnLimit_low_thresh - moribund_conns */
  120. tt_int_op(pick_oos_mock_last_n, OP_EQ, 7);
  121. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  122. tt_int_op(kill_conn_list_killed, OP_EQ, 7);
  123. /* OOS from socket count, but pick fails */
  124. kill_conn_list_killed = 0;
  125. moribund_conns = 0;
  126. pick_oos_mock_fail = 1;
  127. connection_check_oos(62, 0);
  128. tt_int_op(moribund_calls, OP_EQ, 3);
  129. tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
  130. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  131. tt_int_op(kill_conn_list_killed, OP_EQ, 0);
  132. pick_oos_mock_fail = 0;
  133. /*
  134. * OOS from socket count with so many moribund conns
  135. * we have none to kill.
  136. */
  137. kill_conn_list_killed = 0;
  138. moribund_conns = 15;
  139. connection_check_oos(62, 0);
  140. tt_int_op(moribund_calls, OP_EQ, 4);
  141. tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
  142. tt_int_op(kill_conn_list_calls, OP_EQ, 2);
  143. /*
  144. * OOS from socket exhaustion; OOS handler will try to
  145. * kill 1/10 (5) of the connections.
  146. */
  147. kill_conn_list_killed = 0;
  148. moribund_conns = 0;
  149. connection_check_oos(50, 1);
  150. tt_int_op(moribund_calls, OP_EQ, 5);
  151. tt_int_op(pick_oos_mock_calls, OP_EQ, 4);
  152. tt_int_op(kill_conn_list_calls, OP_EQ, 3);
  153. tt_int_op(kill_conn_list_killed, OP_EQ, 5);
  154. /* OOS from socket exhaustion with moribund conns */
  155. kill_conn_list_killed = 0;
  156. moribund_conns = 2;
  157. connection_check_oos(50, 1);
  158. tt_int_op(moribund_calls, OP_EQ, 6);
  159. tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
  160. tt_int_op(kill_conn_list_calls, OP_EQ, 4);
  161. tt_int_op(kill_conn_list_killed, OP_EQ, 3);
  162. /* OOS from socket exhaustion with many moribund conns */
  163. kill_conn_list_killed = 0;
  164. moribund_conns = 7;
  165. connection_check_oos(50, 1);
  166. tt_int_op(moribund_calls, OP_EQ, 7);
  167. tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
  168. tt_int_op(kill_conn_list_calls, OP_EQ, 4);
  169. /* OOS with both socket exhaustion and above-threshold */
  170. kill_conn_list_killed = 0;
  171. moribund_conns = 0;
  172. connection_check_oos(62, 1);
  173. tt_int_op(moribund_calls, OP_EQ, 8);
  174. tt_int_op(pick_oos_mock_calls, OP_EQ, 6);
  175. tt_int_op(kill_conn_list_calls, OP_EQ, 5);
  176. tt_int_op(kill_conn_list_killed, OP_EQ, 12);
  177. /*
  178. * OOS with both socket exhaustion and above-threshold with some
  179. * moribund conns
  180. */
  181. kill_conn_list_killed = 0;
  182. moribund_conns = 5;
  183. connection_check_oos(62, 1);
  184. tt_int_op(moribund_calls, OP_EQ, 9);
  185. tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
  186. tt_int_op(kill_conn_list_calls, OP_EQ, 6);
  187. tt_int_op(kill_conn_list_killed, OP_EQ, 7);
  188. /*
  189. * OOS with both socket exhaustion and above-threshold with many
  190. * moribund conns
  191. */
  192. kill_conn_list_killed = 0;
  193. moribund_conns = 15;
  194. connection_check_oos(62, 1);
  195. tt_int_op(moribund_calls, OP_EQ, 10);
  196. tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
  197. tt_int_op(kill_conn_list_calls, OP_EQ, 6);
  198. done:
  199. UNMOCK(pick_oos_victims);
  200. UNMOCK(kill_conn_list_for_oos);
  201. UNMOCK(connection_count_moribund);
  202. UNMOCK(get_options);
  203. return;
  204. }
  205. static int cfe_calls = 0;
  206. static void
  207. close_for_error_mock(or_connection_t *orconn, int flush)
  208. {
  209. (void)flush;
  210. tt_ptr_op(orconn, OP_NE, NULL);
  211. ++cfe_calls;
  212. done:
  213. return;
  214. }
  215. static int mark_calls = 0;
  216. static void
  217. mark_for_close_oos_mock(connection_t *conn,
  218. int line, const char *file)
  219. {
  220. (void)line;
  221. (void)file;
  222. tt_ptr_op(conn, OP_NE, NULL);
  223. ++mark_calls;
  224. done:
  225. return;
  226. }
  227. static void
  228. test_oos_kill_conn_list(void *arg)
  229. {
  230. connection_t *c1, *c2;
  231. or_connection_t *or_c1 = NULL;
  232. dir_connection_t *dir_c2 = NULL;
  233. smartlist_t *l = NULL;
  234. (void)arg;
  235. /* Set up mocks */
  236. mark_calls = 0;
  237. MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock);
  238. cfe_calls = 0;
  239. MOCK(connection_or_close_for_error, close_for_error_mock);
  240. /* Make fake conns */
  241. or_c1 = tor_malloc_zero(sizeof(*or_c1));
  242. or_c1->base_.magic = OR_CONNECTION_MAGIC;
  243. or_c1->base_.type = CONN_TYPE_OR;
  244. c1 = TO_CONN(or_c1);
  245. dir_c2 = tor_malloc_zero(sizeof(*dir_c2));
  246. dir_c2->base_.magic = DIR_CONNECTION_MAGIC;
  247. dir_c2->base_.type = CONN_TYPE_DIR;
  248. dir_c2->base_.state = DIR_CONN_STATE_MIN_;
  249. dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
  250. c2 = TO_CONN(dir_c2);
  251. tt_ptr_op(c1, OP_NE, NULL);
  252. tt_ptr_op(c2, OP_NE, NULL);
  253. /* Make list */
  254. l = smartlist_new();
  255. smartlist_add(l, c1);
  256. smartlist_add(l, c2);
  257. /* Run kill_conn_list_for_oos() */
  258. kill_conn_list_for_oos(l);
  259. /* Check call counters */
  260. tt_int_op(mark_calls, OP_EQ, 1);
  261. tt_int_op(cfe_calls, OP_EQ, 1);
  262. done:
  263. UNMOCK(connection_or_close_for_error);
  264. UNMOCK(connection_mark_for_close_internal_);
  265. if (l) smartlist_free(l);
  266. tor_free(or_c1);
  267. tor_free(dir_c2);
  268. return;
  269. }
  270. static smartlist_t *conns_for_mock = NULL;
  271. static smartlist_t *
  272. get_conns_mock(void)
  273. {
  274. return conns_for_mock;
  275. }
  276. /*
  277. * For this mock, we pretend all conns have either zero or one circuits,
  278. * depending on if this appears on the list of things to say have a circuit.
  279. */
  280. static smartlist_t *conns_with_circs = NULL;
  281. static int
  282. get_num_circuits_mock(or_connection_t *conn)
  283. {
  284. int circs = 0;
  285. tt_ptr_op(conn, OP_NE, NULL);
  286. if (conns_with_circs &&
  287. smartlist_contains(conns_with_circs, TO_CONN(conn))) {
  288. circs = 1;
  289. }
  290. done:
  291. return circs;
  292. }
  293. static void
  294. test_oos_pick_oos_victims(void *arg)
  295. {
  296. (void)arg;
  297. or_connection_t *ortmp;
  298. dir_connection_t *dirtmp;
  299. smartlist_t *picked;
  300. /* Set up mocks */
  301. conns_for_mock = smartlist_new();
  302. MOCK(get_connection_array, get_conns_mock);
  303. conns_with_circs = smartlist_new();
  304. MOCK(connection_or_get_num_circuits, get_num_circuits_mock);
  305. /* Make some fake connections */
  306. ortmp = tor_malloc_zero(sizeof(*ortmp));
  307. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  308. ortmp->base_.type = CONN_TYPE_OR;
  309. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  310. /* We'll pretend this one has a circuit too */
  311. smartlist_add(conns_with_circs, TO_CONN(ortmp));
  312. /* Next one */
  313. ortmp = tor_malloc_zero(sizeof(*ortmp));
  314. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  315. ortmp->base_.type = CONN_TYPE_OR;
  316. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  317. /* Next one is moribund */
  318. ortmp = tor_malloc_zero(sizeof(*ortmp));
  319. ortmp->base_.magic = OR_CONNECTION_MAGIC;
  320. ortmp->base_.type = CONN_TYPE_OR;
  321. ortmp->base_.marked_for_close = 1;
  322. smartlist_add(conns_for_mock, TO_CONN(ortmp));
  323. /* Last one isn't an orconn */
  324. dirtmp = tor_malloc_zero(sizeof(*dirtmp));
  325. dirtmp->base_.magic = DIR_CONNECTION_MAGIC;
  326. dirtmp->base_.type = CONN_TYPE_DIR;
  327. smartlist_add(conns_for_mock, TO_CONN(dirtmp));
  328. /* Try picking one */
  329. picked = pick_oos_victims(1);
  330. /* It should be the one with circuits */
  331. tt_ptr_op(picked, OP_NE, NULL);
  332. tt_int_op(smartlist_len(picked), OP_EQ, 1);
  333. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  334. smartlist_free(picked);
  335. /* Try picking none */
  336. picked = pick_oos_victims(0);
  337. /* We should get an empty list */
  338. tt_ptr_op(picked, OP_NE, NULL);
  339. tt_int_op(smartlist_len(picked), OP_EQ, 0);
  340. smartlist_free(picked);
  341. /* Try picking two */
  342. picked = pick_oos_victims(2);
  343. /* We should get both active orconns */
  344. tt_ptr_op(picked, OP_NE, NULL);
  345. tt_int_op(smartlist_len(picked), OP_EQ, 2);
  346. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  347. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
  348. smartlist_free(picked);
  349. /* Try picking three - only two are eligible */
  350. picked = pick_oos_victims(3);
  351. tt_int_op(smartlist_len(picked), OP_EQ, 2);
  352. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
  353. tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
  354. smartlist_free(picked);
  355. done:
  356. /* Free leftover stuff */
  357. if (conns_with_circs) {
  358. smartlist_free(conns_with_circs);
  359. conns_with_circs = NULL;
  360. }
  361. UNMOCK(connection_or_get_num_circuits);
  362. if (conns_for_mock) {
  363. SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c));
  364. smartlist_free(conns_for_mock);
  365. conns_for_mock = NULL;
  366. }
  367. UNMOCK(get_connection_array);
  368. return;
  369. }
  370. struct testcase_t oos_tests[] = {
  371. { "connection_check_oos", test_oos_connection_check_oos,
  372. TT_FORK, NULL, NULL },
  373. { "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL },
  374. { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
  375. END_OF_TESTCASES
  376. };