shim_internal.h 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851
  1. /* Copyright (C) 2014 Stony Brook University
  2. This file is part of Graphene Library OS.
  3. Graphene Library OS is free software: you can redistribute it and/or
  4. modify it under the terms of the GNU Lesser General Public License
  5. as published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. Graphene Library OS is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Lesser General Public License for more details.
  11. You should have received a copy of the GNU Lesser General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  13. /*
  14. * shim_internal.h
  15. */
  16. #ifndef _SHIM_INTERNAL_H_
  17. #define _SHIM_INTERNAL_H_
  18. #ifndef IN_SHIM
  19. #error "this header file can only be used inside SHIM"
  20. #endif
  21. #define attribute_hidden __attribute__ ((visibility ("hidden")))
  22. #define ALIAS_STR(name) #name
  23. #define EXTERN_ALIAS(name) \
  24. extern __typeof__(name) shim_##name __attribute ((alias (ALIAS_STR(name))))
  25. #define static_always_inline static inline __attribute__((always_inline))
  26. #include <api.h>
  27. #include <assert.h>
  28. #include <shim_types.h>
  29. #include <shim_defs.h>
  30. #include <atomic.h>
  31. #include <shim_tcb.h>
  32. noreturn void shim_clean_and_exit(int exit_code);
  33. /* important macros and static inline functions */
  34. static inline unsigned int get_cur_tid(void) {
  35. return SHIM_TCB_GET(tid);
  36. }
  37. #define PAL_NATIVE_ERRNO (SHIM_TCB_GET(pal_errno))
  38. #define INTERNAL_TID_BASE ((IDTYPE) 1 << (sizeof(IDTYPE) * 8 - 1))
  39. static inline bool is_internal_tid(unsigned int tid)
  40. {
  41. return tid >= INTERNAL_TID_BASE;
  42. }
  43. struct debug_buf {
  44. int start;
  45. int end;
  46. char buf[DEBUGBUF_SIZE];
  47. };
  48. #include <pal.h>
  49. #include <pal_debug.h>
  50. #include <pal_error.h>
  51. extern PAL_HANDLE debug_handle;
  52. #include <stdarg.h>
  53. void debug_printf (const char * fmt, ...) __attribute__((format (printf, 1, 2)));
  54. void debug_puts (const char * str);
  55. void debug_putch (int ch);
  56. void debug_vprintf (const char * fmt, va_list ap) __attribute__((format (printf, 1, 0)));
  57. #define VMID_PREFIX "[P%05u] "
  58. #define TID_PREFIX "[%-6u] "
  59. #define NOID_PREFIX "[ ] "
  60. #define debug(fmt, ...) \
  61. do { \
  62. if (debug_handle) \
  63. debug_printf(fmt, ##__VA_ARGS__); \
  64. } while (0)
  65. /* print system messages */
  66. #define SYSPRINT_BUFFER_SIZE 256
  67. void handle_printf (PAL_HANDLE hdl, const char * fmt, ...) __attribute__((format (printf, 2, 3)));
  68. void handle_vprintf (PAL_HANDLE hdl, const char * fmt, va_list ap) __attribute__((format (printf, 2, 0)));
  69. #define __SYS_PRINTF(fmt, ...) \
  70. do { \
  71. PAL_HANDLE _hdl = __open_shim_stdio(); \
  72. if (_hdl) \
  73. handle_printf(_hdl, fmt, ##__VA_ARGS__); \
  74. } while (0)
  75. #define __SYS_VPRINTF(fmt, va) \
  76. do { \
  77. PAL_HANDLE _hdl = __open_shim_stdio(); \
  78. if (_hdl) \
  79. handle_vprintf(_hdl, fmt, va); \
  80. } while (0)
  81. #define __SYS_FPRINTF(hdl, fmt, ...) \
  82. do { \
  83. handle_printf(hdl, fmt, ##__VA_ARGS__); \
  84. } while (0)
  85. #define SYS_PRINTF(fmt, ...) \
  86. do { \
  87. MASTER_LOCK(); \
  88. __SYS_PRINTF(fmt, ##__VA_ARGS__); \
  89. MASTER_UNLOCK(); \
  90. } while (0)
  91. #define SYS_FPRINTF(hdl, fmt, ...) \
  92. do { \
  93. MASTER_LOCK(); \
  94. __SYS_FPRINTF(hdl, fmt, ##__VA_ARGS__); \
  95. MASTER_UNLOCK(); \
  96. } while (0)
  97. extern PAL_HANDLE shim_stdio;
  98. static inline PAL_HANDLE __open_shim_stdio (void)
  99. {
  100. if (shim_stdio == (PAL_HANDLE) -1)
  101. return NULL;
  102. if (shim_stdio)
  103. return shim_stdio;
  104. shim_stdio = DkStreamOpen(URI_PREFIX_DEV "tty", PAL_ACCESS_RDWR, 0, 0, 0);
  105. if (!shim_stdio) {
  106. shim_stdio = (PAL_HANDLE) -1;
  107. return NULL;
  108. }
  109. return shim_stdio;
  110. }
  111. #define USE_PAUSE 0
  112. static inline void do_pause (void);
  113. #if USE_PAUSE == 1
  114. # define PAUSE() do { do_pause(); } while (0)
  115. #else
  116. # define PAUSE() do { __asm__ volatile ("int $3"); } while (0)
  117. #endif
  118. #define BUG() \
  119. do { \
  120. __SYS_PRINTF("BUG() " __FILE__ ":%d\n", __LINE__); \
  121. PAUSE(); \
  122. shim_clean_and_exit(-ENOTRECOVERABLE); \
  123. } while (0)
  124. #define DEBUG_HERE() \
  125. do { debug("%s (" __FILE__ ":%d)\n", __func__, __LINE__); } while (0)
  126. /* definition for syscall table */
  127. void handle_signal (void);
  128. long convert_pal_errno (long err);
  129. void syscall_wrapper(void);
  130. void syscall_wrapper_after_syscalldb(void);
  131. #define PAL_ERRNO convert_pal_errno(PAL_NATIVE_ERRNO)
  132. #define SHIM_ARG_TYPE long
  133. #ifdef PROFILE
  134. # define ENTER_TIME shim_get_tcb()->context.enter_time
  135. # define BEGIN_SYSCALL_PROFILE() \
  136. do { ENTER_TIME = GET_PROFILE_INTERVAL(); } while (0)
  137. # define END_SYSCALL_PROFILE(name) \
  138. do { unsigned long _interval = GET_PROFILE_INTERVAL(); \
  139. if (_interval - ENTER_TIME > 1000) \
  140. SAVE_PROFILE_INTERVAL_SET(syscall_##name##_slow, ENTER_TIME, _interval); \
  141. else \
  142. SAVE_PROFILE_INTERVAL_SET(syscall_##name, ENTER_TIME, _interval); \
  143. ENTER_TIME = 0; } while (0)
  144. #else
  145. # define BEGIN_SYSCALL_PROFILE() do {} while (0)
  146. # define END_SYSCALL_PROFILE(name) do {} while (0)
  147. #endif
  148. void check_stack_hook (void);
  149. static inline int64_t get_cur_preempt (void) {
  150. shim_tcb_t* tcb = shim_get_tcb();
  151. assert(tcb);
  152. return atomic_read(&tcb->context.preempt);
  153. }
  154. #define BEGIN_SHIM(name, args ...) \
  155. SHIM_ARG_TYPE __shim_##name(args) { \
  156. SHIM_ARG_TYPE ret = 0; \
  157. int64_t preempt = get_cur_preempt(); \
  158. __UNUSED(preempt); \
  159. /* handle_signal(); */ \
  160. /* check_stack_hook(); */ \
  161. BEGIN_SYSCALL_PROFILE();
  162. #define END_SHIM(name) \
  163. END_SYSCALL_PROFILE(name); \
  164. handle_signal(); \
  165. assert(preempt == get_cur_preempt()); \
  166. return ret; \
  167. }
  168. #define DEFINE_SHIM_SYSCALL(name, n, func, ...) \
  169. DEFINE_PROFILE_INTERVAL(syscall_##name##_slow, syscall); \
  170. DEFINE_PROFILE_INTERVAL(syscall_##name, syscall); \
  171. SHIM_SYSCALL_##n (name, func, __VA_ARGS__) \
  172. EXPORT_SHIM_SYSCALL (name, n, __VA_ARGS__)
  173. #define PROTO_ARGS_0() void
  174. #define PROTO_ARGS_1(t, a) t a
  175. #define PROTO_ARGS_2(t, a, rest ...) t a, PROTO_ARGS_1(rest)
  176. #define PROTO_ARGS_3(t, a, rest ...) t a, PROTO_ARGS_2(rest)
  177. #define PROTO_ARGS_4(t, a, rest ...) t a, PROTO_ARGS_3(rest)
  178. #define PROTO_ARGS_5(t, a, rest ...) t a, PROTO_ARGS_4(rest)
  179. #define PROTO_ARGS_6(t, a, rest ...) t a, PROTO_ARGS_5(rest)
  180. #define CAST_ARGS_0()
  181. #define CAST_ARGS_1(t, a) (SHIM_ARG_TYPE) a
  182. #define CAST_ARGS_2(t, a, rest ...) (SHIM_ARG_TYPE) a, CAST_ARGS_1(rest)
  183. #define CAST_ARGS_3(t, a, rest ...) (SHIM_ARG_TYPE) a, CAST_ARGS_2(rest)
  184. #define CAST_ARGS_4(t, a, rest ...) (SHIM_ARG_TYPE) a, CAST_ARGS_3(rest)
  185. #define CAST_ARGS_5(t, a, rest ...) (SHIM_ARG_TYPE) a, CAST_ARGS_4(rest)
  186. #define CAST_ARGS_6(t, a, rest ...) (SHIM_ARG_TYPE) a, CAST_ARGS_5(rest)
  187. #define DEFINE_SHIM_FUNC(func, n, r, args ...) \
  188. r func (PROTO_ARGS_##n (args));
  189. #define TYPE_HASH(t) ({ const char * _s = #t; \
  190. ((uint16_t) _s[0] << 8) + _s[1]; })
  191. #define POINTER_TYPE(t) ({ int _h = TYPE_HASH(t); \
  192. _h == TYPE_HASH(void *) || _h == TYPE_HASH(char *) || \
  193. _h == TYPE_HASH(const); })
  194. #define EXPORT_SHIM_SYSCALL(name, n, r, args ...) \
  195. r shim_##name (PROTO_ARGS_##n (args)) { \
  196. SHIM_ARG_TYPE ret = __shim_##name (CAST_ARGS_##n (args)); \
  197. if (POINTER_TYPE(r)) { \
  198. if ((uint64_t) ret >= (uint64_t) -4095L) return (r) 0; \
  199. } else { \
  200. if ((int) ret < 0) return (r) -1; \
  201. } \
  202. return (r) ret; \
  203. }
  204. #define PARSE_SYSCALL1(name, ...) \
  205. if (debug_handle) \
  206. parse_syscall_before(__NR_##name, #name, ##__VA_ARGS__);
  207. #define PARSE_SYSCALL2(name, ...) \
  208. if (debug_handle) \
  209. parse_syscall_after(__NR_##name, #name, ##__VA_ARGS__);
  210. void parse_syscall_before (int sysno, const char * name, int nr, ...);
  211. void parse_syscall_after (int sysno, const char * name, int nr, ...);
  212. #define SHIM_SYSCALL_0(name, func, r) \
  213. BEGIN_SHIM(name, void) \
  214. PARSE_SYSCALL1(name, 0); \
  215. r __ret = (func)(); \
  216. PARSE_SYSCALL2(name, 0, #r, __ret); \
  217. ret = (SHIM_ARG_TYPE) __ret; \
  218. END_SHIM(name)
  219. #define SHIM_SYSCALL_1(name, func, r, t1, a1) \
  220. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1) \
  221. t1 a1 = (t1) __arg1; \
  222. PARSE_SYSCALL1(name, 1, #t1, a1); \
  223. r __ret = (func)(a1); \
  224. PARSE_SYSCALL2(name, 1, #r, __ret, #t1, a1); \
  225. ret = (SHIM_ARG_TYPE) __ret; \
  226. END_SHIM(name)
  227. #define SHIM_SYSCALL_2(name, func, r, t1, a1, t2, a2) \
  228. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1, SHIM_ARG_TYPE __arg2) \
  229. t1 a1 = (t1) __arg1; \
  230. t2 a2 = (t2) __arg2; \
  231. PARSE_SYSCALL1(name, 2, #t1, a1, #t2, a2); \
  232. r __ret = (func)(a1, a2); \
  233. PARSE_SYSCALL2(name, 2, #r, __ret, #t1, a1, #t2, a2); \
  234. ret = (SHIM_ARG_TYPE) __ret; \
  235. END_SHIM(name)
  236. #define SHIM_SYSCALL_3(name, func, r, t1, a1, t2, a2, t3, a3) \
  237. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1, SHIM_ARG_TYPE __arg2, \
  238. SHIM_ARG_TYPE __arg3) \
  239. t1 a1 = (t1) __arg1; \
  240. t2 a2 = (t2) __arg2; \
  241. t3 a3 = (t3) __arg3; \
  242. PARSE_SYSCALL1(name, 3, #t1, a1, #t2, a2, #t3, a3); \
  243. r __ret = (func)(a1, a2, a3); \
  244. PARSE_SYSCALL2(name, 3, #r, __ret, #t1, a1, #t2, a2, #t3, a3); \
  245. ret = (SHIM_ARG_TYPE) __ret; \
  246. END_SHIM(name)
  247. #define SHIM_SYSCALL_4(name, func, r, t1, a1, t2, a2, t3, a3, t4, a4) \
  248. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1, SHIM_ARG_TYPE __arg2, \
  249. SHIM_ARG_TYPE __arg3, SHIM_ARG_TYPE __arg4) \
  250. t1 a1 = (t1) __arg1; \
  251. t2 a2 = (t2) __arg2; \
  252. t3 a3 = (t3) __arg3; \
  253. t4 a4 = (t4) __arg4; \
  254. PARSE_SYSCALL1(name, 4, #t1, a1, #t2, a2, #t3, a3, #t4, a4); \
  255. r __ret = (func)(a1, a2, a3, a4); \
  256. PARSE_SYSCALL2(name, 4, #r, __ret, #t1, a1, #t2, a2, #t3, a3, \
  257. #t4, a4); \
  258. ret = (SHIM_ARG_TYPE) __ret; \
  259. END_SHIM(name)
  260. #define SHIM_SYSCALL_5(name, func, r, t1, a1, t2, a2, t3, a3, t4, a4, t5, a5) \
  261. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1, SHIM_ARG_TYPE __arg2, \
  262. SHIM_ARG_TYPE __arg3, SHIM_ARG_TYPE __arg4, \
  263. SHIM_ARG_TYPE __arg5) \
  264. t1 a1 = (t1) __arg1; \
  265. t2 a2 = (t2) __arg2; \
  266. t3 a3 = (t3) __arg3; \
  267. t4 a4 = (t4) __arg4; \
  268. t5 a5 = (t5) __arg5; \
  269. PARSE_SYSCALL1(name, 5, #t1, a1, #t2, a2, #t3, a3, #t4, a4, \
  270. #t5, a5); \
  271. r __ret = (func)(a1, a2, a3, a4, a5); \
  272. PARSE_SYSCALL2(name, 5, #r, __ret, #t1, a1, #t2, a2, #t3, a3, \
  273. #t4, a4, #t5, a5); \
  274. ret = (SHIM_ARG_TYPE) __ret; \
  275. END_SHIM(name)
  276. #define SHIM_SYSCALL_6(name, func, r, t1, a1, t2, a2, t3, a3, t4, a4, t5, a5, t6, a6) \
  277. BEGIN_SHIM(name, SHIM_ARG_TYPE __arg1, SHIM_ARG_TYPE __arg2, \
  278. SHIM_ARG_TYPE __arg3, SHIM_ARG_TYPE __arg4, \
  279. SHIM_ARG_TYPE __arg5, SHIM_ARG_TYPE __arg6) \
  280. t1 a1 = (t1) __arg1; \
  281. t2 a2 = (t2) __arg2; \
  282. t3 a3 = (t3) __arg3; \
  283. t4 a4 = (t4) __arg4; \
  284. t5 a5 = (t5) __arg5; \
  285. t6 a6 = (t6) __arg6; \
  286. PARSE_SYSCALL1(name, 6, #t1, a1, #t2, a2, #t3, a3, #t4, a4, \
  287. #t5, a5, #t6, a6); \
  288. r __ret = (func)(a1, a2, a3, a4, a5, a6); \
  289. PARSE_SYSCALL2(name, 6, #r, __ret, #t1, a1, #t2, a2, #t3, a3, \
  290. #t4, a4, #t5, a5, #t6, a6); \
  291. ret = (SHIM_ARG_TYPE) __ret; \
  292. END_SHIM(name)
  293. #define SHIM_PROTO_ARGS_0 void
  294. #define SHIM_PROTO_ARGS_1 SHIM_ARG_TYPE __arg1
  295. #define SHIM_PROTO_ARGS_2 SHIM_PROTO_ARGS_1, SHIM_ARG_TYPE __arg2
  296. #define SHIM_PROTO_ARGS_3 SHIM_PROTO_ARGS_2, SHIM_ARG_TYPE __arg3
  297. #define SHIM_PROTO_ARGS_4 SHIM_PROTO_ARGS_3, SHIM_ARG_TYPE __arg4
  298. #define SHIM_PROTO_ARGS_5 SHIM_PROTO_ARGS_4, SHIM_ARG_TYPE __arg5
  299. #define SHIM_PROTO_ARGS_6 SHIM_PROTO_ARGS_5, SHIM_ARG_TYPE __arg6
  300. #define SHIM_PASS_ARGS_1 __arg1
  301. #define SHIM_PASS_ARGS_2 SHIM_PASS_ARGS_1, __arg2
  302. #define SHIM_PASS_ARGS_3 SHIM_PASS_ARGS_2, __arg3
  303. #define SHIM_PASS_ARGS_4 SHIM_PASS_ARGS_3, __arg4
  304. #define SHIM_PASS_ARGS_5 SHIM_PASS_ARGS_4, __arg5
  305. #define SHIM_PASS_ARGS_6 SHIM_PASS_ARGS_5, __arg6
  306. #define SHIM_UNUSED_ARGS_0()
  307. #define SHIM_UNUSED_ARGS_1() do { \
  308. __UNUSED(__arg1); \
  309. } while (0)
  310. #define SHIM_UNUSED_ARGS_2() do { \
  311. __UNUSED(__arg1); \
  312. __UNUSED(__arg2); \
  313. } while (0)
  314. #define SHIM_UNUSED_ARGS_3() do { \
  315. __UNUSED(__arg1); \
  316. __UNUSED(__arg2); \
  317. __UNUSED(__arg3); \
  318. } while (0)
  319. #define SHIM_UNUSED_ARGS_4() do { \
  320. __UNUSED(__arg1); \
  321. __UNUSED(__arg2); \
  322. __UNUSED(__arg3); \
  323. __UNUSED(__arg4); \
  324. } while (0)
  325. #define SHIM_UNUSED_ARGS_5() do { \
  326. __UNUSED(__arg1); \
  327. __UNUSED(__arg2); \
  328. __UNUSED(__arg3); \
  329. __UNUSED(__arg4); \
  330. __UNUSED(__arg5); \
  331. } while (0)
  332. #define SHIM_UNUSED_ARGS_6() do { \
  333. __UNUSED(__arg1); \
  334. __UNUSED(__arg2); \
  335. __UNUSED(__arg3); \
  336. __UNUSED(__arg4); \
  337. __UNUSED(__arg5); \
  338. __UNUSED(__arg6); \
  339. } while (0)
  340. #define DO_SYSCALL(...) DO_SYSCALL2(__VA_ARGS__)
  341. #define DO_SYSCALL2(n, ...) -ENOSYS
  342. #define DO_SYSCALL_0(sysno) -ENOSYS
  343. #define DO_SYSCALL_1(sysno, ...) DO_SYSCALL(1, sysno, SHIM_PASS_ARGS_1)
  344. #define DO_SYSCALL_2(sysno, ...) DO_SYSCALL(2, sysno, SHIM_PASS_ARGS_2)
  345. #define DO_SYSCALL_3(sysno, ...) DO_SYSCALL(3, sysno, SHIM_PASS_ARGS_3)
  346. #define DO_SYSCALL_4(sysno, ...) DO_SYSCALL(4, sysno, SHIM_PASS_ARGS_4)
  347. #define DO_SYSCALL_5(sysno, ...) DO_SYSCALL(5, sysno, SHIM_PASS_ARGS_5)
  348. #define DO_SYSCALL_6(sysno, ...) DO_SYSCALL(6, sysno, SHIM_PASS_ARGS_6)
  349. #define SHIM_SYSCALL_PASSTHROUGH(name, n, ...) \
  350. DEFINE_PROFILE_INTERVAL(syscall_##name##_slow, syscall); \
  351. DEFINE_PROFILE_INTERVAL(syscall_##name, syscall); \
  352. BEGIN_SHIM(name, SHIM_PROTO_ARGS_##n) \
  353. debug("WARNING: shim_" #name " not implemented\n"); \
  354. SHIM_UNUSED_ARGS_##n(); \
  355. ret = DO_SYSCALL_##n(__NR_##name); \
  356. END_SHIM(name) \
  357. EXPORT_SHIM_SYSCALL(name, n, __VA_ARGS__)
  358. #define CONCAT2(t1, t2) __CONCAT2(t1, t2)
  359. #define __CONCAT2(t1, t2) t1##_##t2
  360. #define CONCAT3(t1, t2, t3) __CONCAT3(t1, t2, t3)
  361. #define __CONCAT3(t1, t2, t3) t1##_##t2##_##t3
  362. /* Some SHIM internal errno */
  363. #define EISLINK 141 /* the path is a link */
  364. #define ECONTAINLINK 142 /* part of path contains a link */
  365. #define ENOTLINK 143 /* the path is not a link */
  366. #define ESKIPPED 144 /* skip looking up current path */
  367. #define PAL_CB(member) (pal_control.member)
  368. #define LOCK_FREE ((IDTYPE) -1)
  369. extern bool lock_enabled;
  370. static inline void enable_locking (void)
  371. {
  372. if (!lock_enabled)
  373. lock_enabled = true;
  374. }
  375. static inline PAL_HANDLE thread_create (void * func, void * arg)
  376. {
  377. assert(lock_enabled);
  378. return DkThreadCreate(func, arg);
  379. }
  380. static inline int64_t __disable_preempt (shim_tcb_t * tcb)
  381. {
  382. //tcb->context.syscall_nr += SYSCALL_NR_PREEMPT_INC;
  383. int64_t preempt = atomic_inc_return(&tcb->context.preempt);
  384. /* Assert if this counter overflows */
  385. assert(preempt != 0);
  386. //debug("disable preempt: %d\n", preempt);
  387. return preempt;
  388. }
  389. static inline void disable_preempt (shim_tcb_t * tcb)
  390. {
  391. if (!tcb && !(tcb = shim_get_tcb()))
  392. return;
  393. __disable_preempt(tcb);
  394. }
  395. static inline void __enable_preempt (shim_tcb_t * tcb)
  396. {
  397. int64_t preempt = atomic_add_return(-1, &tcb->context.preempt);
  398. /* Assert if this counter underflows */
  399. __UNUSED(preempt);
  400. assert(preempt >= 0);
  401. //debug("enable preempt: %d\n", preempt);
  402. }
  403. void __handle_signal (shim_tcb_t * tcb, int sig);
  404. static inline void enable_preempt (shim_tcb_t * tcb)
  405. {
  406. if (!tcb && !(tcb = shim_get_tcb()))
  407. return;
  408. int64_t preempt = atomic_read(&tcb->context.preempt);
  409. if (!preempt)
  410. return;
  411. if (preempt == 1)
  412. __handle_signal(tcb, 0);
  413. __enable_preempt(tcb);
  414. }
  415. static inline bool lock_created(struct shim_lock* l)
  416. {
  417. return l->lock != NULL;
  418. }
  419. static inline void clear_lock(struct shim_lock* l)
  420. {
  421. l->lock = NULL;
  422. l->owner = 0;
  423. }
  424. static inline bool create_lock(struct shim_lock* l) {
  425. l->owner = 0;
  426. l->lock = DkMutexCreate(0);
  427. return l->lock != NULL;
  428. }
  429. static inline void destroy_lock(struct shim_lock* l) {
  430. DkObjectClose(l->lock);
  431. l->lock = NULL;
  432. l->owner = 0;
  433. }
  434. #ifdef DEBUG
  435. #define lock(l) __lock(l, __FILE__, __LINE__)
  436. static void __lock(struct shim_lock* l, const char* file, int line) {
  437. #else
  438. static void lock(struct shim_lock* l) {
  439. #endif
  440. if (!lock_enabled) {
  441. return;
  442. }
  443. /* TODO: This whole if should be just an assert. Change it once we are sure that it does not
  444. * trigger (previous code allowed for this case). Same in unlock below. */
  445. if (!l->lock) {
  446. #ifdef DEBUG
  447. debug("Trying to lock an uninitialized lock at %s:%d!\n", file, line);
  448. #endif // DEBUG
  449. __abort();
  450. }
  451. shim_tcb_t * tcb = shim_get_tcb();
  452. disable_preempt(tcb);
  453. while (!DkSynchronizationObjectWait(l->lock, NO_TIMEOUT))
  454. /* nop */;
  455. l->owner = tcb->tid;
  456. }
  457. #ifdef DEBUG
  458. #define unlock(l) __unlock(l, __FILE__, __LINE__)
  459. static inline void __unlock(struct shim_lock* l, const char* file, int line) {
  460. #else
  461. static inline void unlock(struct shim_lock* l) {
  462. #endif
  463. if (!lock_enabled) {
  464. return;
  465. }
  466. if (!l->lock) {
  467. #ifdef DEBUG
  468. debug("Trying to unlock an uninitialized lock at %s:%d!\n", file, line);
  469. #endif // DEBUG
  470. __abort();
  471. }
  472. shim_tcb_t* tcb = shim_get_tcb();
  473. l->owner = 0;
  474. DkMutexRelease(l->lock);
  475. enable_preempt(tcb);
  476. }
  477. static inline bool locked(struct shim_lock* l) {
  478. if (!lock_enabled) {
  479. return true;
  480. }
  481. if (!l->lock) {
  482. return false;
  483. }
  484. return get_cur_tid() == l->owner;
  485. }
  486. #define DEBUG_MASTER_LOCK 0
  487. extern struct shim_lock __master_lock;
  488. #if DEBUG_MASTER_LOCK == 1
  489. # define MASTER_LOCK() \
  490. do { \
  491. lock(&__master_lock); \
  492. pal_printf("master lock " __FILE__ ":%d\n", __LINE__); \
  493. } while (0)
  494. # define MASTER_UNLOCK() \
  495. do { \
  496. pal_printf("master unlock " __FILE__ ":%d\n", __LINE__); \
  497. unlock(&__master_lock); \
  498. } while (0)
  499. #else
  500. # define MASTER_LOCK() do { lock(&__master_lock); } while (0)
  501. # define MASTER_UNLOCK() do { unlock(&__master_lock); } while (0)
  502. #endif
  503. static inline bool create_lock_runtime(struct shim_lock* l) {
  504. bool ret = true;
  505. if (!lock_created(l)) {
  506. MASTER_LOCK();
  507. if (!lock_created(l))
  508. ret = create_lock(l);
  509. MASTER_UNLOCK();
  510. }
  511. return ret;
  512. }
  513. static inline void create_event (AEVENTTYPE * e)
  514. {
  515. if (!e->event)
  516. e->event = DkStreamOpen(URI_PREFIX_PIPE, PAL_ACCESS_RDWR, 0, 0,
  517. PAL_OPTION_NONBLOCK);
  518. }
  519. static inline bool event_created (AEVENTTYPE * e)
  520. {
  521. return e->event != NULL;
  522. }
  523. static inline PAL_HANDLE event_handle (AEVENTTYPE * e)
  524. {
  525. return e->event;
  526. }
  527. static inline void destroy_event (AEVENTTYPE * e)
  528. {
  529. if (e->event) {
  530. DkObjectClose(e->event);
  531. e->event = NULL;
  532. }
  533. }
  534. static inline void set_event (AEVENTTYPE * e, int n)
  535. {
  536. if (e->event) {
  537. char bytes[n];
  538. DkStreamWrite(e->event, 0, n, bytes, NULL);
  539. }
  540. }
  541. static inline void wait_event (AEVENTTYPE * e)
  542. {
  543. if (e->event) {
  544. char byte;
  545. int n = 0;
  546. do {
  547. if (!DkSynchronizationObjectWait(e->event, NO_TIMEOUT))
  548. continue;
  549. n = DkStreamRead(e->event, 0, 1, &byte, NULL, 0);
  550. } while (!n);
  551. }
  552. }
  553. static inline void clear_event (AEVENTTYPE * e)
  554. {
  555. if (e->event) {
  556. char bytes[100];
  557. int n;
  558. do {
  559. n = DkStreamRead(e->event, 0, 100, bytes, NULL, 0);
  560. } while (n == 100);
  561. }
  562. }
  563. static inline void do_pause (void)
  564. {
  565. bool go = false;
  566. while (!go)
  567. DkThreadDelayExecution(60 * 60 * 1000000ULL);
  568. }
  569. /* reference counter APIs */
  570. #define REF_GET(ref) atomic_read(&(ref))
  571. #define REF_SET(ref, count) atomic_set(&(ref), count)
  572. static inline int __ref_inc (REFTYPE * ref)
  573. {
  574. register int _c;
  575. do {
  576. _c = atomic_read(ref);
  577. assert(_c >= 0);
  578. } while (atomic_cmpxchg(ref, _c, _c + 1) != _c);
  579. return _c + 1;
  580. }
  581. #define REF_INC(ref) __ref_inc(&(ref))
  582. static inline int __ref_dec (REFTYPE * ref)
  583. {
  584. register int _c;
  585. do {
  586. _c = atomic_read(ref);
  587. if (!_c) {
  588. debug("Fail: Trying to drop reference count below 0\n");
  589. BUG();
  590. return 0;
  591. }
  592. } while (atomic_cmpxchg(ref, _c, _c - 1) != _c);
  593. return _c - 1;
  594. }
  595. #define REF_DEC(ref) __ref_dec(&(ref))
  596. /* integer hash functions */
  597. static inline uint32_t hash32 (uint32_t key)
  598. {
  599. key = ~key + (key << 15);
  600. key = key ^ (key >> 12);
  601. key = key + (key << 2);
  602. key = key ^ (key >> 4);
  603. key = (key + (key << 3)) + (key << 11);
  604. key = key ^ (key >> 16);
  605. return key;
  606. }
  607. static inline uint64_t hash64 (uint64_t key)
  608. {
  609. key = (~key) + (key << 21);
  610. key = key ^ (key >> 24);
  611. key = (key + (key << 3)) + (key << 8);
  612. key = key ^ (key >> 14);
  613. key = (key + (key << 2)) + (key << 4);
  614. key = key ^ (key >> 28);
  615. key = key + (key << 31);
  616. return key;
  617. }
  618. #ifndef __alloca
  619. # define __alloca __builtin_alloca
  620. #endif
  621. extern size_t g_pal_alloc_align;
  622. #define ALLOC_ALIGNMENT g_pal_alloc_align
  623. #define IS_ALLOC_ALIGNED(x) IS_ALIGNED_POW2(x, g_pal_alloc_align)
  624. #define IS_ALLOC_ALIGNED_PTR(x) IS_ALIGNED_PTR_POW2(x, g_pal_alloc_align)
  625. #define ALLOC_ALIGN_DOWN(x) ALIGN_DOWN_POW2(x, g_pal_alloc_align)
  626. #define ALLOC_ALIGN_UP(x) ALIGN_UP_POW2(x, g_pal_alloc_align)
  627. #define ALLOC_ALIGN_DOWN_PTR(x) ALIGN_DOWN_PTR_POW2(x, g_pal_alloc_align)
  628. #define ALLOC_ALIGN_UP_PTR(x) ALIGN_UP_PTR_POW2(x, g_pal_alloc_align)
  629. void * __system_malloc (size_t size);
  630. void __system_free (void * addr, size_t size);
  631. #define system_malloc __system_malloc
  632. #define system_free __system_free
  633. extern void * migrated_memory_start;
  634. extern void * migrated_memory_end;
  635. static inline bool memory_migrated(void * mem)
  636. {
  637. return mem >= migrated_memory_start && mem < migrated_memory_end;
  638. }
  639. extern void * __load_address, * __load_address_end;
  640. extern void * __code_address, * __code_address_end;
  641. unsigned long parse_int (const char * str);
  642. extern void * initial_stack;
  643. extern const char ** initial_envp;
  644. void get_brk_region (void ** start, void ** end, void ** current);
  645. int reset_brk (void);
  646. struct shim_handle;
  647. int init_brk_from_executable (struct shim_handle * exec);
  648. int init_brk_region(void* brk_region, size_t data_segment_size);
  649. int init_heap (void);
  650. int init_internal_map (void);
  651. int init_loader (void);
  652. int init_manifest (PAL_HANDLE manifest_handle);
  653. int init_rlimit(void);
  654. bool test_user_memory (void * addr, size_t size, bool write);
  655. bool test_user_string (const char * addr);
  656. uint64_t get_rlimit_cur(int resource);
  657. void set_rlimit_cur(int resource, uint64_t rlim);
  658. int object_wait_with_retry(PAL_HANDLE handle);
  659. void release_clear_child_tid(int* clear_child_tid);
  660. void delete_from_epoll_handles(struct shim_handle* handle);
  661. #ifdef __x86_64__
  662. #define __SWITCH_STACK(stack_top, func, arg) \
  663. do { \
  664. /* 16 Bytes align of stack */ \
  665. uintptr_t __stack_top = (uintptr_t)(stack_top); \
  666. __stack_top &= ~0xf; \
  667. __stack_top -= 8; \
  668. __asm__ volatile ( \
  669. "movq %0, %%rbp\n" \
  670. "movq %0, %%rsp\n" \
  671. "jmpq *%1\n" \
  672. ::"r"(__stack_top), "r"(func), "D"(arg): "memory"); \
  673. } while (0)
  674. static_always_inline void * current_stack(void)
  675. {
  676. void * _rsp;
  677. __asm__ volatile ("movq %%rsp, %0" : "=r"(_rsp) :: "memory");
  678. return _rsp;
  679. }
  680. static_always_inline bool __range_not_ok(unsigned long addr, unsigned long size) {
  681. addr += size;
  682. if (addr < size) {
  683. /* pointer arithmetic overflow, this check is x86-64 specific */
  684. return true;
  685. }
  686. return false;
  687. }
  688. /* Check if pointer to memory region is valid. Return true if the memory
  689. * region may be valid, false if it is definitely invalid. */
  690. static inline bool access_ok(const volatile void* addr, size_t size) {
  691. return !__range_not_ok((unsigned long)addr, (unsigned long)size);
  692. }
  693. #else
  694. # error "Unsupported architecture"
  695. #endif /* __x86_64__ */
  696. static inline IDTYPE hashtype_to_idtype(HASHTYPE hash) {
  697. static_assert(sizeof(HASHTYPE) == 8, "Unsupported HASHTYPE size");
  698. static_assert(sizeof(IDTYPE) == 4, "Unsupported IDTYPE size");
  699. return ((IDTYPE)hash) ^ ((IDTYPE)(hash >> 32));
  700. }
  701. #endif /* _PAL_INTERNAL_H_ */