tinytest.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500
  1. /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
  2. *
  3. * Redistribution and use in source and binary forms, with or without
  4. * modification, are permitted provided that the following conditions
  5. * are met:
  6. * 1. Redistributions of source code must retain the above copyright
  7. * notice, this list of conditions and the following disclaimer.
  8. * 2. Redistributions in binary form must reproduce the above copyright
  9. * notice, this list of conditions and the following disclaimer in the
  10. * documentation and/or other materials provided with the distribution.
  11. * 3. The name of the author may not be used to endorse or promote products
  12. * derived from this software without specific prior written permission.
  13. *
  14. * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15. * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18. * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23. * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24. */
  25. #ifdef TINYTEST_LOCAL
  26. #include "tinytest_local.h"
  27. #endif
  28. #include <stdio.h>
  29. #include <stdlib.h>
  30. #include <string.h>
  31. #include <assert.h>
  32. #ifndef NO_FORKING
  33. #ifdef _WIN32
  34. #include <windows.h>
  35. #else
  36. #include <sys/types.h>
  37. #include <sys/wait.h>
  38. #include <unistd.h>
  39. #endif
  40. #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
  41. #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
  42. __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
  43. /* Workaround for a stupid bug in OSX 10.6 */
  44. #define FORK_BREAKS_GCOV
  45. #include <vproc.h>
  46. #endif
  47. #endif
  48. #endif /* !NO_FORKING */
  49. #ifndef __GNUC__
  50. #define __attribute__(x)
  51. #endif
  52. #include "tinytest.h"
  53. #include "tinytest_macros.h"
  54. #define LONGEST_TEST_NAME 16384
  55. static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  56. static int n_ok = 0; /**< Number of tests that have passed */
  57. static int n_bad = 0; /**< Number of tests that have failed. */
  58. static int n_skipped = 0; /**< Number of tests that have been skipped. */
  59. static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  60. static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  61. static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  62. const char *verbosity_flag = "";
  63. const struct testlist_alias_t *cfg_aliases=NULL;
  64. enum outcome { SKIP=2, OK=1, FAIL=0 };
  65. static enum outcome cur_test_outcome = 0;
  66. const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  67. /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  68. const char *cur_test_name = NULL;
  69. #ifdef _WIN32
  70. /* Copy of argv[0] for win32. */
  71. static char commandname[MAX_PATH+1];
  72. #endif
  73. static void usage(struct testgroup_t *groups, int list_groups)
  74. __attribute__((noreturn));
  75. static int process_test_option(struct testgroup_t *groups, const char *test);
  76. static enum outcome
  77. testcase_run_bare_(const struct testcase_t *testcase)
  78. {
  79. void *env = NULL;
  80. int outcome;
  81. if (testcase->setup) {
  82. env = testcase->setup->setup_fn(testcase);
  83. if (!env)
  84. return FAIL;
  85. else if (env == (void*)TT_SKIP)
  86. return SKIP;
  87. }
  88. cur_test_outcome = OK;
  89. testcase->fn(env);
  90. outcome = cur_test_outcome;
  91. if (testcase->setup) {
  92. if (testcase->setup->cleanup_fn(testcase, env) == 0)
  93. outcome = FAIL;
  94. }
  95. return outcome;
  96. }
  97. #define MAGIC_EXITCODE 42
  98. #ifndef NO_FORKING
  99. static enum outcome
  100. testcase_run_forked_(const struct testgroup_t *group,
  101. const struct testcase_t *testcase)
  102. {
  103. #ifdef _WIN32
  104. /* Fork? On Win32? How primitive! We'll do what the smart kids do:
  105. we'll invoke our own exe (whose name we recall from the command
  106. line) with a command line that tells it to run just the test we
  107. want, and this time without forking.
  108. (No, threads aren't an option. The whole point of forking is to
  109. share no state between tests.)
  110. */
  111. int ok;
  112. char buffer[LONGEST_TEST_NAME+256];
  113. STARTUPINFOA si;
  114. PROCESS_INFORMATION info;
  115. DWORD exitcode;
  116. if (!in_tinytest_main) {
  117. printf("\nERROR. On Windows, testcase_run_forked_ must be"
  118. " called from within tinytest_main.\n");
  119. abort();
  120. }
  121. if (opt_verbosity>0)
  122. printf("[forking] ");
  123. snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
  124. commandname, verbosity_flag, group->prefix, testcase->name);
  125. memset(&si, 0, sizeof(si));
  126. memset(&info, 0, sizeof(info));
  127. si.cb = sizeof(si);
  128. ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
  129. 0, NULL, NULL, &si, &info);
  130. if (!ok) {
  131. printf("CreateProcess failed!\n");
  132. return 0;
  133. }
  134. WaitForSingleObject(info.hProcess, INFINITE);
  135. GetExitCodeProcess(info.hProcess, &exitcode);
  136. CloseHandle(info.hProcess);
  137. CloseHandle(info.hThread);
  138. if (exitcode == 0)
  139. return OK;
  140. else if (exitcode == MAGIC_EXITCODE)
  141. return SKIP;
  142. else
  143. return FAIL;
  144. #else
  145. int outcome_pipe[2];
  146. pid_t pid;
  147. (void)group;
  148. if (pipe(outcome_pipe))
  149. perror("opening pipe");
  150. if (opt_verbosity>0)
  151. printf("[forking] ");
  152. pid = fork();
  153. #ifdef FORK_BREAKS_GCOV
  154. vproc_transaction_begin(0);
  155. #endif
  156. if (!pid) {
  157. /* child. */
  158. int test_r, write_r;
  159. char b[1];
  160. close(outcome_pipe[0]);
  161. test_r = testcase_run_bare_(testcase);
  162. assert(0<=(int)test_r && (int)test_r<=2);
  163. b[0] = "NYS"[test_r];
  164. write_r = (int)write(outcome_pipe[1], b, 1);
  165. if (write_r != 1) {
  166. perror("write outcome to pipe");
  167. exit(1);
  168. }
  169. exit(0);
  170. return FAIL; /* unreachable */
  171. } else {
  172. /* parent */
  173. int status, r;
  174. char b[1];
  175. /* Close this now, so that if the other side closes it,
  176. * our read fails. */
  177. close(outcome_pipe[1]);
  178. r = (int)read(outcome_pipe[0], b, 1);
  179. if (r == 0) {
  180. printf("[Lost connection!] ");
  181. return 0;
  182. } else if (r != 1) {
  183. perror("read outcome from pipe");
  184. }
  185. waitpid(pid, &status, 0);
  186. close(outcome_pipe[0]);
  187. return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
  188. }
  189. #endif
  190. }
  191. #endif /* !NO_FORKING */
  192. int
  193. testcase_run_one(const struct testgroup_t *group,
  194. const struct testcase_t *testcase)
  195. {
  196. enum outcome outcome;
  197. if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
  198. if (opt_verbosity>0)
  199. printf("%s%s: %s\n",
  200. group->prefix, testcase->name,
  201. (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
  202. ++n_skipped;
  203. return SKIP;
  204. }
  205. if (opt_verbosity>0 && !opt_forked) {
  206. printf("%s%s: ", group->prefix, testcase->name);
  207. } else {
  208. if (opt_verbosity==0) printf(".");
  209. cur_test_prefix = group->prefix;
  210. cur_test_name = testcase->name;
  211. }
  212. #ifndef NO_FORKING
  213. if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
  214. outcome = testcase_run_forked_(group, testcase);
  215. } else {
  216. #else
  217. {
  218. #endif
  219. outcome = testcase_run_bare_(testcase);
  220. }
  221. if (outcome == OK) {
  222. ++n_ok;
  223. if (opt_verbosity>0 && !opt_forked)
  224. puts(opt_verbosity==1?"OK":"");
  225. } else if (outcome == SKIP) {
  226. ++n_skipped;
  227. if (opt_verbosity>0 && !opt_forked)
  228. puts("SKIPPED");
  229. } else {
  230. ++n_bad;
  231. if (!opt_forked)
  232. printf("\n [%s FAILED]\n", testcase->name);
  233. }
  234. if (opt_forked) {
  235. exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
  236. return 1; /* unreachable */
  237. } else {
  238. return (int)outcome;
  239. }
  240. }
  241. int
  242. tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
  243. {
  244. int i, j;
  245. size_t length = LONGEST_TEST_NAME;
  246. char fullname[LONGEST_TEST_NAME];
  247. int found=0;
  248. if (strstr(arg, ".."))
  249. length = strstr(arg,"..")-arg;
  250. for (i=0; groups[i].prefix; ++i) {
  251. for (j=0; groups[i].cases[j].name; ++j) {
  252. struct testcase_t *testcase = &groups[i].cases[j];
  253. snprintf(fullname, sizeof(fullname), "%s%s",
  254. groups[i].prefix, testcase->name);
  255. if (!flag) { /* Hack! */
  256. printf(" %s", fullname);
  257. if (testcase->flags & TT_OFF_BY_DEFAULT)
  258. puts(" (Off by default)");
  259. else if (testcase->flags & TT_SKIP)
  260. puts(" (DISABLED)");
  261. else
  262. puts("");
  263. }
  264. if (!strncmp(fullname, arg, length)) {
  265. if (set)
  266. testcase->flags |= flag;
  267. else
  268. testcase->flags &= ~flag;
  269. ++found;
  270. }
  271. }
  272. }
  273. return found;
  274. }
  275. static void
  276. usage(struct testgroup_t *groups, int list_groups)
  277. {
  278. puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
  279. puts(" Specify tests by name, or using a prefix ending with '..'");
  280. puts(" To skip a test, prefix its name with a colon.");
  281. puts(" To enable a disabled test, prefix its name with a plus.");
  282. puts(" Use --list-tests for a list of tests.");
  283. if (list_groups) {
  284. puts("Known tests are:");
  285. tinytest_set_flag_(groups, "..", 1, 0);
  286. }
  287. exit(0);
  288. }
  289. static int
  290. process_test_alias(struct testgroup_t *groups, const char *test)
  291. {
  292. int i, j, n, r;
  293. for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
  294. if (!strcmp(cfg_aliases[i].name, test)) {
  295. n = 0;
  296. for (j = 0; cfg_aliases[i].tests[j]; ++j) {
  297. r = process_test_option(groups, cfg_aliases[i].tests[j]);
  298. if (r<0)
  299. return -1;
  300. n += r;
  301. }
  302. return n;
  303. }
  304. }
  305. printf("No such test alias as @%s!",test);
  306. return -1;
  307. }
  308. static int
  309. process_test_option(struct testgroup_t *groups, const char *test)
  310. {
  311. int flag = TT_ENABLED_;
  312. int n = 0;
  313. if (test[0] == '@') {
  314. return process_test_alias(groups, test + 1);
  315. } else if (test[0] == ':') {
  316. ++test;
  317. flag = TT_SKIP;
  318. } else if (test[0] == '+') {
  319. ++test;
  320. ++n;
  321. if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
  322. printf("No such test as %s!\n", test);
  323. return -1;
  324. }
  325. } else {
  326. ++n;
  327. }
  328. if (!tinytest_set_flag_(groups, test, 1, flag)) {
  329. printf("No such test as %s!\n", test);
  330. return -1;
  331. }
  332. return n;
  333. }
  334. void
  335. tinytest_set_aliases(const struct testlist_alias_t *aliases)
  336. {
  337. cfg_aliases = aliases;
  338. }
  339. int
  340. tinytest_main(int c, const char **v, struct testgroup_t *groups)
  341. {
  342. int i, j, n=0;
  343. #ifdef _WIN32
  344. const char *sp = strrchr(v[0], '.');
  345. const char *extension = "";
  346. if (!sp || stricmp(sp, ".exe"))
  347. extension = ".exe"; /* Add an exe so CreateProcess will work */
  348. snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
  349. commandname[MAX_PATH]='\0';
  350. #endif
  351. for (i=1; i<c; ++i) {
  352. if (v[i][0] == '-') {
  353. if (!strcmp(v[i], "--RUNNING-FORKED")) {
  354. opt_forked = 1;
  355. } else if (!strcmp(v[i], "--no-fork")) {
  356. opt_nofork = 1;
  357. } else if (!strcmp(v[i], "--quiet")) {
  358. opt_verbosity = -1;
  359. verbosity_flag = "--quiet";
  360. } else if (!strcmp(v[i], "--verbose")) {
  361. opt_verbosity = 2;
  362. verbosity_flag = "--verbose";
  363. } else if (!strcmp(v[i], "--terse")) {
  364. opt_verbosity = 0;
  365. verbosity_flag = "--terse";
  366. } else if (!strcmp(v[i], "--help")) {
  367. usage(groups, 0);
  368. } else if (!strcmp(v[i], "--list-tests")) {
  369. usage(groups, 1);
  370. } else {
  371. printf("Unknown option %s. Try --help\n",v[i]);
  372. return -1;
  373. }
  374. } else {
  375. int r = process_test_option(groups, v[i]);
  376. if (r<0)
  377. return -1;
  378. n += r;
  379. }
  380. }
  381. if (!n)
  382. tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
  383. #ifdef _IONBF
  384. setvbuf(stdout, NULL, _IONBF, 0);
  385. #endif
  386. ++in_tinytest_main;
  387. for (i=0; groups[i].prefix; ++i)
  388. for (j=0; groups[i].cases[j].name; ++j)
  389. if (groups[i].cases[j].flags & TT_ENABLED_)
  390. testcase_run_one(&groups[i],
  391. &groups[i].cases[j]);
  392. --in_tinytest_main;
  393. if (opt_verbosity==0)
  394. puts("");
  395. if (n_bad)
  396. printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
  397. n_bad+n_ok,n_skipped);
  398. else if (opt_verbosity >= 1)
  399. printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
  400. return (n_bad == 0) ? 0 : 1;
  401. }
  402. int
  403. tinytest_get_verbosity_(void)
  404. {
  405. return opt_verbosity;
  406. }
  407. void
  408. tinytest_set_test_failed_(void)
  409. {
  410. if (opt_verbosity <= 0 && cur_test_name) {
  411. if (opt_verbosity==0) puts("");
  412. printf("%s%s: ", cur_test_prefix, cur_test_name);
  413. cur_test_name = NULL;
  414. }
  415. cur_test_outcome = 0;
  416. }
  417. void
  418. tinytest_set_test_skipped_(void)
  419. {
  420. if (cur_test_outcome==OK)
  421. cur_test_outcome = SKIP;
  422. }
  423. char *
  424. tinytest_format_hex_(const void *val_, unsigned long len)
  425. {
  426. const unsigned char *val = val_;
  427. char *result, *cp;
  428. size_t i;
  429. int ellipses = 0;
  430. if (!val)
  431. return strdup("null");
  432. if (len > 1024) {
  433. ellipses = 3;
  434. len = 1024;
  435. }
  436. if (!(result = malloc(len*2+4)))
  437. return strdup("<allocation failure>");
  438. cp = result;
  439. for (i=0;i<len;++i) {
  440. *cp++ = "0123456789ABCDEF"[(val[i] >> 4)&0x0f];
  441. *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
  442. }
  443. while (ellipses--)
  444. *cp++ = '.';
  445. *cp = 0;
  446. return result;
  447. }