shim_init.c 36 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216
  1. /* Copyright (C) 2014 Stony Brook University
  2. This file is part of Graphene Library OS.
  3. Graphene Library OS is free software: you can redistribute it and/or
  4. modify it under the terms of the GNU Lesser General Public License
  5. as published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. Graphene Library OS is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Lesser General Public License for more details.
  11. You should have received a copy of the GNU Lesser General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  13. /*
  14. * shim_init.c
  15. *
  16. * This file contains entry and exit functions of library OS.
  17. */
  18. #include <shim_internal.h>
  19. #include <shim_table.h>
  20. #include <shim_tls.h>
  21. #include <shim_thread.h>
  22. #include <shim_handle.h>
  23. #include <shim_vma.h>
  24. #include <shim_checkpoint.h>
  25. #include <shim_fs.h>
  26. #include <shim_ipc.h>
  27. #include <shim_profile.h>
  28. #include <shim_vdso.h>
  29. #include <pal.h>
  30. #include <pal_debug.h>
  31. #include <pal_error.h>
  32. #include <sys/mman.h>
  33. #include <asm/unistd.h>
  34. #include <asm/fcntl.h>
  35. unsigned long allocsize;
  36. unsigned long allocshift;
  37. unsigned long allocmask;
  38. /* The following constants will help matching glibc version with compatible
  39. SHIM libraries */
  40. #include "glibc-version.h"
  41. const unsigned int glibc_version = GLIBC_VERSION;
  42. static void handle_failure (PAL_PTR event, PAL_NUM arg, PAL_CONTEXT * context)
  43. {
  44. __UNUSED(event);
  45. __UNUSED(context);
  46. shim_get_tls()->pal_errno = (arg <= PAL_ERROR_BOUND) ? arg : 0;
  47. }
  48. noreturn void __abort(void) {
  49. PAUSE();
  50. shim_terminate(-ENOTRECOVERABLE);
  51. }
  52. void warn (const char *format, ...)
  53. {
  54. va_list args;
  55. va_start (args, format);
  56. __SYS_VPRINTF(format, args);
  57. va_end (args);
  58. }
  59. void __stack_chk_fail (void)
  60. {
  61. }
  62. static int pal_errno_to_unix_errno [PAL_ERROR_BOUND + 1] = {
  63. /* reserved */ 0,
  64. /* PAL_ERROR_NOTIMPLEMENTED */ ENOSYS,
  65. /* PAL_ERROR_NOTDEFINED */ ENOSYS,
  66. /* PAL_ERROR_NOTSUPPORT */ EACCES,
  67. /* PAL_ERROR_INVAL */ EINVAL,
  68. /* PAL_ERROR_TOOLONG */ ENAMETOOLONG,
  69. /* PAL_ERROR_DENIED */ EACCES,
  70. /* PAL_ERROR_BADHANDLE */ EFAULT,
  71. /* PAL_ERROR_STREAMEXIST */ EEXIST,
  72. /* PAL_ERROR_STREAMNOTEXIST */ ENOENT,
  73. /* PAL_ERROR_STREAMISFILE */ ENOTDIR,
  74. /* PAL_ERROR_STREAMISDIR */ EISDIR,
  75. /* PAL_ERROR_STREAMISDEVICE */ ESPIPE,
  76. /* PAL_ERROR_INTERRUPTED */ EINTR,
  77. /* PAL_ERROR_OVERFLOW */ EFAULT,
  78. /* PAL_ERROR_BADADDR */ EFAULT,
  79. /* PAL_ERROR_NOMEM */ ENOMEM,
  80. /* PAL_ERROR_NOTKILLABLE */ EACCES,
  81. /* PAL_ERROR_INCONSIST */ EFAULT,
  82. /* PAL_ERROR_TRYAGAIN */ EAGAIN,
  83. /* PAL_ERROR_ENDOFSTREAM */ 0,
  84. /* PAL_ERROR_NOTSERVER */ EINVAL,
  85. /* PAL_ERROR_NOTCONNECTION */ ENOTCONN,
  86. /* PAL_ERROR_ZEROSIZE */ 0,
  87. /* PAL_ERROR_CONNFAILED */ ECONNRESET,
  88. /* PAL_ERROR_ADDRNOTEXIST */ EADDRNOTAVAIL,
  89. };
  90. long convert_pal_errno (long err)
  91. {
  92. return (err >= 0 && err <= PAL_ERROR_BOUND) ?
  93. pal_errno_to_unix_errno[err] : 0;
  94. }
  95. unsigned long parse_int (const char * str)
  96. {
  97. unsigned long num = 0;
  98. int radix = 10;
  99. char c;
  100. if (str[0] == '0') {
  101. str++;
  102. radix = 8;
  103. if (str[0] == 'x') {
  104. str++;
  105. radix = 16;
  106. }
  107. }
  108. while ((c = *(str++))) {
  109. int val;
  110. if (c >= 'A' && c <= 'F')
  111. val = c - 'A' + 10;
  112. else if (c >= 'a' && c <= 'f')
  113. val = c - 'a' + 10;
  114. else if (c >= '0' && c <= '9')
  115. val = c - '0';
  116. else
  117. break;
  118. if (val >= radix)
  119. break;
  120. num = num * radix + val;
  121. }
  122. if (c == 'G' || c == 'g')
  123. num *= 1024 * 1024 * 1024;
  124. else if (c == 'M' || c == 'm')
  125. num *= 1024 * 1024;
  126. else if (c == 'K' || c == 'k')
  127. num *= 1024;
  128. return num;
  129. }
  130. long int glibc_option (const char * opt)
  131. {
  132. char cfg[CONFIG_MAX];
  133. if (strcmp_static(opt, "heap_size")) {
  134. ssize_t ret = get_config(root_config, "glibc.heap_size", cfg, CONFIG_MAX);
  135. if (ret <= 0) {
  136. debug("no glibc option: %s (err=%ld)\n", opt, ret);
  137. return -ENOENT;
  138. }
  139. long int heap_size = parse_int(cfg);
  140. debug("glibc option: heap_size = %ld\n", heap_size);
  141. return (long int) heap_size;
  142. }
  143. return -EINVAL;
  144. }
  145. void * migrated_memory_start;
  146. void * migrated_memory_end;
  147. const char ** initial_envp __attribute_migratable;
  148. /* library_paths is populated with LD_PRELOAD entries once during LibOS
  149. * initialization and is used in __load_interp_object() to search for ELF
  150. * program interpreter in specific paths. Once allocated, its memory is
  151. * never freed or updated. */
  152. char ** library_paths = NULL;
  153. struct shim_lock __master_lock;
  154. bool lock_enabled;
  155. void init_tcb (shim_tcb_t * tcb)
  156. {
  157. tcb->canary = SHIM_TLS_CANARY;
  158. tcb->self = tcb;
  159. }
  160. void copy_tcb (shim_tcb_t * new_tcb, const shim_tcb_t * old_tcb)
  161. {
  162. memset(new_tcb, 0, sizeof(shim_tcb_t));
  163. new_tcb->canary = SHIM_TLS_CANARY;
  164. new_tcb->self = new_tcb;
  165. new_tcb->tp = old_tcb->tp;
  166. memcpy(&new_tcb->context, &old_tcb->context, sizeof(struct shim_context));
  167. new_tcb->tid = old_tcb->tid;
  168. new_tcb->debug_buf = old_tcb->debug_buf;
  169. }
  170. /* This function is used to allocate tls before interpreter start running */
  171. void allocate_tls (__libc_tcb_t * tcb, bool user, struct shim_thread * thread)
  172. {
  173. assert(tcb);
  174. tcb->tcb = tcb;
  175. init_tcb(&tcb->shim_tcb);
  176. if (thread) {
  177. thread->tcb = tcb;
  178. thread->user_tcb = user;
  179. tcb->shim_tcb.tp = thread;
  180. tcb->shim_tcb.tid = thread->tid;
  181. } else {
  182. tcb->shim_tcb.tp = NULL;
  183. tcb->shim_tcb.tid = 0;
  184. }
  185. DkSegmentRegister(PAL_SEGMENT_FS, tcb);
  186. assert(shim_tls_check_canary());
  187. }
  188. void populate_tls (__libc_tcb_t * tcb, bool user)
  189. {
  190. assert(tcb);
  191. tcb->tcb = tcb;
  192. copy_tcb(&tcb->shim_tcb, shim_get_tls());
  193. struct shim_thread * thread = (struct shim_thread *) tcb->shim_tcb.tp;
  194. if (thread) {
  195. thread->tcb = tcb;
  196. thread->user_tcb = user;
  197. }
  198. DkSegmentRegister(PAL_SEGMENT_FS, tcb);
  199. assert(shim_tls_check_canary());
  200. }
  201. DEFINE_PROFILE_OCCURENCE(alloc_stack, memory);
  202. DEFINE_PROFILE_OCCURENCE(alloc_stack_count, memory);
  203. #define STACK_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
  204. void * allocate_stack (size_t size, size_t protect_size, bool user)
  205. {
  206. size = ALIGN_UP(size);
  207. protect_size = ALIGN_UP(protect_size);
  208. /* preserve a non-readable, non-writable page below the user
  209. stack to stop user program to clobber other vmas */
  210. void * stack = NULL;
  211. int flags = STACK_FLAGS|(user ? 0 : VMA_INTERNAL);
  212. if (user) {
  213. stack = bkeep_unmapped_heap(size + protect_size, PROT_NONE,
  214. flags, NULL, 0, "stack");
  215. if (!stack)
  216. return NULL;
  217. stack = (void *)
  218. DkVirtualMemoryAlloc(stack, size + protect_size,
  219. 0, PAL_PROT_NONE);
  220. } else {
  221. stack = system_malloc(size + protect_size);
  222. }
  223. if (!stack)
  224. return NULL;
  225. ADD_PROFILE_OCCURENCE(alloc_stack, size + protect_size);
  226. INC_PROFILE_OCCURENCE(alloc_stack_count);
  227. stack += protect_size;
  228. // Ensure proper alignment for process' initial stack pointer value.
  229. stack += (16 - (uintptr_t)stack % 16) % 16;
  230. DkVirtualMemoryProtect(stack, size, PAL_PROT_READ|PAL_PROT_WRITE);
  231. if (bkeep_mprotect(stack, size, PROT_READ|PROT_WRITE, flags) < 0)
  232. return NULL;
  233. debug("allocated stack at %p (size = %ld)\n", stack, size);
  234. return stack;
  235. }
  236. static int populate_user_stack (void * stack, size_t stack_size,
  237. elf_auxv_t ** auxpp, int ** argcpp,
  238. const char *** argvp, const char *** envpp)
  239. {
  240. const int argc = **argcpp;
  241. const char ** argv = *argvp, ** envp = *envpp;
  242. const char ** new_argv = NULL, ** new_envp = NULL;
  243. elf_auxv_t *new_auxp = NULL;
  244. void * stack_bottom = stack;
  245. void * stack_top = stack + stack_size;
  246. #define ALLOCATE_TOP(size) \
  247. ({ if ((stack_top -= (size)) < stack_bottom) return -ENOMEM; \
  248. stack_top; })
  249. #define ALLOCATE_BOTTOM(size) \
  250. ({ if ((stack_bottom += (size)) > stack_top) return -ENOMEM; \
  251. stack_bottom - (size); })
  252. /* ld.so expects argc as long on stack, not int. */
  253. long * argcp = ALLOCATE_BOTTOM(sizeof(long));
  254. *argcp = **argcpp;
  255. if (!argv) {
  256. *(const char **) ALLOCATE_BOTTOM(sizeof(const char *)) = NULL;
  257. goto copy_envp;
  258. }
  259. new_argv = stack_bottom;
  260. while (argv) {
  261. for (const char ** a = argv ; *a ; a++) {
  262. const char ** t = ALLOCATE_BOTTOM(sizeof(const char *));
  263. int len = strlen(*a) + 1;
  264. char * abuf = ALLOCATE_TOP(len);
  265. memcpy(abuf, *a, len);
  266. *t = abuf;
  267. }
  268. *((const char **) ALLOCATE_BOTTOM(sizeof(const char *))) = NULL;
  269. copy_envp:
  270. if (!envp)
  271. break;
  272. new_envp = stack_bottom;
  273. argv = envp;
  274. envp = NULL;
  275. }
  276. if (!new_envp)
  277. *(const char **) ALLOCATE_BOTTOM(sizeof(const char *)) = NULL;
  278. /* reserve space for ELF aux vectors, populated later by LibOS */
  279. new_auxp = ALLOCATE_BOTTOM(REQUIRED_ELF_AUXV * sizeof(elf_auxv_t) +
  280. REQUIRED_ELF_AUXV_SPACE);
  281. /* x86_64 ABI requires 16 bytes alignment on stack on every function
  282. call. */
  283. size_t move_size = stack_bottom - stack;
  284. *argcpp = stack_top - move_size;
  285. *argcpp = ALIGN_DOWN_PTR(*argcpp, 16UL);
  286. **argcpp = argc;
  287. size_t shift = (void*)(*argcpp) - stack;
  288. memmove(*argcpp, stack, move_size);
  289. *argvp = new_argv ? (void *) new_argv + shift : NULL;
  290. *envpp = new_envp ? (void *) new_envp + shift : NULL;
  291. *auxpp = new_auxp ? (void *) new_auxp + shift : NULL;
  292. /* clear working area at the bottom */
  293. memset(stack, 0, shift);
  294. return 0;
  295. }
  296. unsigned long sys_stack_size = 0;
  297. int init_stack (const char ** argv, const char ** envp,
  298. int ** argcpp, const char *** argpp,
  299. elf_auxv_t ** auxpp)
  300. {
  301. if (!sys_stack_size) {
  302. sys_stack_size = DEFAULT_SYS_STACK_SIZE;
  303. if (root_config) {
  304. char stack_cfg[CONFIG_MAX];
  305. if (get_config(root_config, "sys.stack.size", stack_cfg,
  306. CONFIG_MAX) > 0)
  307. sys_stack_size = ALIGN_UP(parse_int(stack_cfg));
  308. }
  309. }
  310. struct shim_thread * cur_thread = get_cur_thread();
  311. if (!cur_thread || cur_thread->stack)
  312. return 0;
  313. void * stack = allocate_stack(sys_stack_size, allocsize, true);
  314. if (!stack)
  315. return -ENOMEM;
  316. if (initial_envp)
  317. envp = initial_envp;
  318. int ret = populate_user_stack(stack, sys_stack_size, auxpp, argcpp, &argv, &envp);
  319. if (ret < 0)
  320. return ret;
  321. *argpp = argv;
  322. initial_envp = envp;
  323. cur_thread->stack_top = stack + sys_stack_size;
  324. cur_thread->stack = stack;
  325. cur_thread->stack_red = stack - allocsize;
  326. return 0;
  327. }
  328. int read_environs (const char ** envp)
  329. {
  330. for (const char ** e = envp ; *e ; e++) {
  331. if (strpartcmp_static(*e, "LD_LIBRARY_PATH=")) {
  332. /* populate library_paths with entries from LD_LIBRARY_PATH envvar */
  333. const char * s = *e + static_strlen("LD_LIBRARY_PATH=");
  334. size_t npaths = 2; // One for the first entry, one for the last
  335. // NULL.
  336. for (const char * tmp = s ; *tmp ; tmp++)
  337. if (*tmp == ':')
  338. npaths++;
  339. char** paths = malloc(sizeof(const char *) *
  340. npaths);
  341. if (!paths)
  342. return -ENOMEM;
  343. size_t cnt = 0;
  344. while (*s) {
  345. const char * next;
  346. for (next = s ; *next && *next != ':' ; next++);
  347. size_t len = next - s;
  348. char * str = malloc(len + 1);
  349. if (!str) {
  350. for (size_t i = 0; i < cnt; i++)
  351. free(paths[i]);
  352. free(paths);
  353. return -ENOMEM;
  354. }
  355. memcpy(str, s, len);
  356. str[len] = 0;
  357. paths[cnt++] = str;
  358. s = *next ? next + 1 : next;
  359. }
  360. paths[cnt] = NULL;
  361. assert(!library_paths);
  362. library_paths = paths;
  363. return 0;
  364. }
  365. }
  366. return 0;
  367. }
  368. struct config_store * root_config = NULL;
  369. static void * __malloc (size_t size)
  370. {
  371. return malloc(size);
  372. }
  373. static void __free (void * mem)
  374. {
  375. free(mem);
  376. }
  377. int init_manifest (PAL_HANDLE manifest_handle)
  378. {
  379. int ret = 0;
  380. void * addr = NULL;
  381. size_t size = 0, map_size = 0;
  382. #define MAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL)
  383. if (PAL_CB(manifest_preload.start)) {
  384. addr = PAL_CB(manifest_preload.start);
  385. size = PAL_CB(manifest_preload.end) - PAL_CB(manifest_preload.start);
  386. } else {
  387. PAL_STREAM_ATTR attr;
  388. if (!DkStreamAttributesQueryByHandle(manifest_handle, &attr))
  389. return -PAL_ERRNO;
  390. size = attr.pending_size;
  391. map_size = ALIGN_UP(size);
  392. addr = bkeep_unmapped_any(map_size, PROT_READ, MAP_FLAGS,
  393. NULL, 0, "manifest");
  394. if (!addr)
  395. return -ENOMEM;
  396. void * ret_addr = DkStreamMap(manifest_handle, addr,
  397. PAL_PROT_READ, 0,
  398. ALIGN_UP(size));
  399. if (!ret_addr) {
  400. bkeep_munmap(addr, map_size, MAP_FLAGS);
  401. return -ENOMEM;
  402. } else {
  403. assert(addr == ret_addr);
  404. }
  405. }
  406. struct config_store * new_root_config = malloc(sizeof(struct config_store));
  407. if (!new_root_config) {
  408. ret = -ENOMEM;
  409. goto fail;
  410. }
  411. new_root_config->raw_data = addr;
  412. new_root_config->raw_size = size;
  413. new_root_config->malloc = __malloc;
  414. new_root_config->free = __free;
  415. const char * errstring = "Unexpected error";
  416. if ((ret = read_config(new_root_config, NULL, &errstring)) < 0) {
  417. SYS_PRINTF("Unable to read manifest file: %s\n", errstring);
  418. goto fail;
  419. }
  420. root_config = new_root_config;
  421. return 0;
  422. fail:
  423. if (map_size) {
  424. DkStreamUnmap(addr, map_size);
  425. if (bkeep_munmap(addr, map_size, MAP_FLAGS) < 0)
  426. BUG();
  427. }
  428. free(new_root_config);
  429. return ret;
  430. }
  431. #ifdef PROFILE
  432. struct shim_profile profile_root;
  433. #endif
  434. # define FIND_ARG_COMPONENTS(cookie, argc, argv, envp, auxp) \
  435. do { \
  436. void *_tmp = (cookie); \
  437. (argv) = _tmp; \
  438. _tmp += sizeof(char *) * ((argc) + 1); \
  439. (envp) = _tmp; \
  440. for ( ; *(char **) _tmp; _tmp += sizeof(char *)); \
  441. (auxp) = _tmp + sizeof(char *); \
  442. } while (0)
  443. #ifdef PROFILE
  444. static void set_profile_enabled (const char ** envp)
  445. {
  446. const char ** p;
  447. for (p = envp ; (*p) ; p++)
  448. if (strpartcmp_static(*p, "PROFILE_ENABLED="))
  449. break;
  450. if (!(*p))
  451. return;
  452. for (size_t i = 0 ; i < N_PROFILE ; i++)
  453. PROFILES[i].disabled = true;
  454. const char * str = (*p) + 16;
  455. bool enabled = false;
  456. while (*str) {
  457. const char * next = str;
  458. for ( ; (*next) && (*next) != ',' ; next++);
  459. if (next > str) {
  460. size_t len = next - str;
  461. for (size_t i = 0 ; i < N_PROFILE ; i++) {
  462. struct shim_profile * profile = &PROFILES[i];
  463. if (!memcmp(profile->name, str, len) && !profile->name[len]) {
  464. profile->disabled = false;
  465. if (profile->type == CATEGORY)
  466. enabled = true;
  467. }
  468. }
  469. }
  470. str = (*next) ? next + 1 : next;
  471. }
  472. while (enabled) {
  473. enabled = false;
  474. for (size_t i = 0 ; i < N_PROFILE ; i++) {
  475. struct shim_profile * profile = &PROFILES[i];
  476. if (!profile->disabled || profile->root == &profile_)
  477. continue;
  478. if (!profile->root->disabled) {
  479. profile->disabled = false;
  480. if (profile->type == CATEGORY)
  481. enabled = true;
  482. }
  483. }
  484. }
  485. for (size_t i = 0 ; i < N_PROFILE ; i++) {
  486. struct shim_profile * profile = &PROFILES[i];
  487. if (profile->type == CATEGORY || profile->disabled)
  488. continue;
  489. for (profile = profile->root ;
  490. profile != &profile_ && profile->disabled ;
  491. profile = profile->root)
  492. profile->disabled = false;
  493. }
  494. }
  495. #endif
  496. static int init_newproc (struct newproc_header * hdr)
  497. {
  498. BEGIN_PROFILE_INTERVAL();
  499. int bytes = DkStreamRead(PAL_CB(parent_process), 0,
  500. sizeof(struct newproc_header), hdr,
  501. NULL, 0);
  502. if (!bytes)
  503. return -PAL_ERRNO;
  504. SAVE_PROFILE_INTERVAL(child_wait_header);
  505. SAVE_PROFILE_INTERVAL_SINCE(child_receive_header, hdr->write_proc_time);
  506. return hdr->failure;
  507. }
  508. DEFINE_PROFILE_CATEGORY(pal, );
  509. DEFINE_PROFILE_INTERVAL(pal_startup_time, pal);
  510. DEFINE_PROFILE_INTERVAL(pal_host_specific_startup_time, pal);
  511. DEFINE_PROFILE_INTERVAL(pal_relocation_time, pal);
  512. DEFINE_PROFILE_INTERVAL(pal_linking_time, pal);
  513. DEFINE_PROFILE_INTERVAL(pal_manifest_loading_time, pal);
  514. DEFINE_PROFILE_INTERVAL(pal_allocation_time, pal);
  515. DEFINE_PROFILE_INTERVAL(pal_tail_startup_time, pal);
  516. DEFINE_PROFILE_INTERVAL(pal_child_creation_time, pal);
  517. DEFINE_PROFILE_CATEGORY(init, );
  518. DEFINE_PROFILE_INTERVAL(init_vma, init);
  519. DEFINE_PROFILE_INTERVAL(init_slab, init);
  520. DEFINE_PROFILE_INTERVAL(init_str_mgr, init);
  521. DEFINE_PROFILE_INTERVAL(init_internal_map, init);
  522. DEFINE_PROFILE_INTERVAL(init_fs, init);
  523. DEFINE_PROFILE_INTERVAL(init_dcache, init);
  524. DEFINE_PROFILE_INTERVAL(init_handle, init);
  525. DEFINE_PROFILE_INTERVAL(read_from_checkpoint, init);
  526. DEFINE_PROFILE_INTERVAL(read_from_file, init);
  527. DEFINE_PROFILE_INTERVAL(init_newproc, init);
  528. DEFINE_PROFILE_INTERVAL(init_mount_root, init);
  529. DEFINE_PROFILE_INTERVAL(init_from_checkpoint_file, init);
  530. DEFINE_PROFILE_INTERVAL(restore_from_file, init);
  531. DEFINE_PROFILE_INTERVAL(init_manifest, init);
  532. DEFINE_PROFILE_INTERVAL(init_ipc, init);
  533. DEFINE_PROFILE_INTERVAL(init_thread, init);
  534. DEFINE_PROFILE_INTERVAL(init_important_handles, init);
  535. DEFINE_PROFILE_INTERVAL(init_mount, init);
  536. DEFINE_PROFILE_INTERVAL(init_async, init);
  537. DEFINE_PROFILE_INTERVAL(init_stack, init);
  538. DEFINE_PROFILE_INTERVAL(read_environs, init);
  539. DEFINE_PROFILE_INTERVAL(init_loader, init);
  540. DEFINE_PROFILE_INTERVAL(init_ipc_helper, init);
  541. DEFINE_PROFILE_INTERVAL(init_signal, init);
  542. #define CALL_INIT(func, args ...) func(args)
  543. #define RUN_INIT(func, ...) \
  544. do { \
  545. int _err = CALL_INIT(func, ##__VA_ARGS__); \
  546. if (_err < 0) { \
  547. SYS_PRINTF("shim_init() in " #func " (%d)\n", _err); \
  548. shim_terminate(_err); \
  549. } \
  550. SAVE_PROFILE_INTERVAL(func); \
  551. } while (0)
  552. extern PAL_HANDLE thread_start_event;
  553. noreturn void* shim_init (int argc, void * args)
  554. {
  555. debug_handle = PAL_CB(debug_stream);
  556. cur_process.vmid = (IDTYPE) PAL_CB(process_id);
  557. /* create the initial TCB, shim can not be run without a tcb */
  558. __libc_tcb_t tcb;
  559. memset(&tcb, 0, sizeof(__libc_tcb_t));
  560. allocate_tls(&tcb, false, NULL);
  561. __disable_preempt(&tcb.shim_tcb); // Temporarily disable preemption for delaying any signal
  562. // that arrives during initialization
  563. debug_setbuf(&tcb.shim_tcb, true);
  564. debug("set tcb to %p\n", &tcb);
  565. #ifdef PROFILE
  566. unsigned long begin_time = GET_PROFILE_INTERVAL();
  567. #endif
  568. debug("host: %s\n", PAL_CB(host_type));
  569. DkSetExceptionHandler(&handle_failure, PAL_EVENT_FAILURE);
  570. allocsize = PAL_CB(alloc_align);
  571. allocshift = allocsize - 1;
  572. allocmask = ~allocshift;
  573. create_lock(&__master_lock);
  574. int * argcp = &argc;
  575. const char ** argv, ** envp, ** argp = NULL;
  576. elf_auxv_t * auxp;
  577. /* call to figure out where the arguments are */
  578. FIND_ARG_COMPONENTS(args, argc, argv, envp, auxp);
  579. #ifdef PROFILE
  580. set_profile_enabled(envp);
  581. #endif
  582. struct newproc_header hdr;
  583. void * cpaddr = NULL;
  584. #ifdef PROFILE
  585. unsigned long begin_create_time = 0;
  586. #endif
  587. BEGIN_PROFILE_INTERVAL();
  588. RUN_INIT(init_vma);
  589. RUN_INIT(init_slab);
  590. RUN_INIT(read_environs, envp);
  591. RUN_INIT(init_str_mgr);
  592. RUN_INIT(init_internal_map);
  593. RUN_INIT(init_fs);
  594. RUN_INIT(init_dcache);
  595. RUN_INIT(init_handle);
  596. debug("shim loaded at %p, ready to initialize\n", &__load_address);
  597. if (argc && argv[0][0] == '-') {
  598. if (strcmp_static(argv[0], "-resume") && argc >= 2) {
  599. const char * filename = *(argv + 1);
  600. argc -= 2;
  601. argv += 2;
  602. RUN_INIT(init_mount_root);
  603. RUN_INIT(init_from_checkpoint_file, filename, &hdr.checkpoint,
  604. &cpaddr);
  605. goto restore;
  606. }
  607. }
  608. if (PAL_CB(parent_process)) {
  609. RUN_INIT(init_newproc, &hdr);
  610. SAVE_PROFILE_INTERVAL_SET(child_created_in_new_process,
  611. hdr.create_time, begin_time);
  612. #ifdef PROFILE
  613. begin_create_time = hdr.begin_create_time;
  614. #endif
  615. if (hdr.checkpoint.hdr.size)
  616. RUN_INIT(do_migration, &hdr.checkpoint, &cpaddr);
  617. }
  618. if (cpaddr) {
  619. restore:
  620. thread_start_event = DkNotificationEventCreate(PAL_FALSE);
  621. RUN_INIT(restore_checkpoint,
  622. &hdr.checkpoint.hdr, &hdr.checkpoint.mem,
  623. (ptr_t) cpaddr, 0);
  624. }
  625. if (PAL_CB(manifest_handle))
  626. RUN_INIT(init_manifest, PAL_CB(manifest_handle));
  627. RUN_INIT(init_mount_root);
  628. RUN_INIT(init_ipc);
  629. RUN_INIT(init_thread);
  630. RUN_INIT(init_mount);
  631. RUN_INIT(init_important_handles);
  632. RUN_INIT(init_async);
  633. RUN_INIT(init_stack, argv, envp, &argcp, &argp, &auxp);
  634. RUN_INIT(init_loader);
  635. RUN_INIT(init_ipc_helper);
  636. RUN_INIT(init_signal);
  637. if (PAL_CB(parent_process)) {
  638. /* Notify the parent process */
  639. struct newproc_response res;
  640. res.child_vmid = cur_process.vmid;
  641. res.failure = 0;
  642. if (!DkStreamWrite(PAL_CB(parent_process), 0,
  643. sizeof(struct newproc_response),
  644. &res, NULL))
  645. shim_do_exit(-PAL_ERRNO);
  646. }
  647. debug("shim process initialized\n");
  648. #ifdef PROFILE
  649. if (begin_create_time)
  650. SAVE_PROFILE_INTERVAL_SINCE(child_total_migration_time,
  651. begin_create_time);
  652. #endif
  653. SAVE_PROFILE_INTERVAL_SET(pal_startup_time, 0, pal_control.startup_time);
  654. SAVE_PROFILE_INTERVAL_SET(pal_host_specific_startup_time, 0,
  655. pal_control.host_specific_startup_time);
  656. SAVE_PROFILE_INTERVAL_SET(pal_relocation_time, 0,
  657. pal_control.relocation_time);
  658. SAVE_PROFILE_INTERVAL_SET(pal_linking_time, 0, pal_control.linking_time);
  659. SAVE_PROFILE_INTERVAL_SET(pal_manifest_loading_time, 0,
  660. pal_control.manifest_loading_time);
  661. SAVE_PROFILE_INTERVAL_SET(pal_allocation_time, 0,
  662. pal_control.allocation_time);
  663. SAVE_PROFILE_INTERVAL_SET(pal_tail_startup_time, 0,
  664. pal_control.tail_startup_time);
  665. SAVE_PROFILE_INTERVAL_SET(pal_child_creation_time, 0,
  666. pal_control.child_creation_time);
  667. if (thread_start_event)
  668. DkEventSet(thread_start_event);
  669. shim_tcb_t * cur_tcb = shim_get_tls();
  670. struct shim_thread * cur_thread = (struct shim_thread *) cur_tcb->tp;
  671. if (cur_tcb->context.regs && cur_tcb->context.regs->rsp) {
  672. vdso_map_migrate();
  673. restore_context(&cur_tcb->context);
  674. }
  675. if (cur_thread->exec)
  676. execute_elf_object(cur_thread->exec, argcp, argp, auxp);
  677. shim_do_exit(0);
  678. }
  679. static int create_unique (int (*mkname) (char *, size_t, void *),
  680. int (*create) (const char *, void *),
  681. int (*output) (char *, size_t, const void *,
  682. struct shim_qstr *),
  683. char * name, size_t size, void * id, void * obj,
  684. struct shim_qstr * qstr)
  685. {
  686. int ret, len;
  687. while (1) {
  688. len = mkname(name, size, id);
  689. if (len < 0)
  690. return len;
  691. if ((ret = create(name, obj)) < 0)
  692. return ret;
  693. if (ret)
  694. continue;
  695. if (output)
  696. return output(name, size, id, qstr);
  697. if (qstr)
  698. qstrsetstr(qstr, name, len);
  699. return len;
  700. }
  701. }
  702. static int name_pipe (char * uri, size_t size, void * id)
  703. {
  704. IDTYPE pipeid;
  705. size_t len;
  706. int ret = DkRandomBitsRead(&pipeid, sizeof(pipeid));
  707. if (ret < 0)
  708. return -convert_pal_errno(-ret);
  709. debug("creating pipe: pipe.srv:%u\n", pipeid);
  710. if ((len = snprintf(uri, size, "pipe.srv:%u", pipeid)) == size)
  711. return -ERANGE;
  712. *((IDTYPE *) id) = pipeid;
  713. return len;
  714. }
  715. static int open_pipe (const char * uri, void * obj)
  716. {
  717. PAL_HANDLE pipe = DkStreamOpen(uri, 0, 0, 0, 0);
  718. if (!pipe)
  719. return PAL_NATIVE_ERRNO == PAL_ERROR_STREAMEXIST ? 1 :
  720. -PAL_ERRNO;
  721. if (obj)
  722. *((PAL_HANDLE *) obj) = pipe;
  723. else
  724. DkObjectClose(pipe);
  725. return 0;
  726. }
  727. static int pipe_addr (char * uri, size_t size, const void * id,
  728. struct shim_qstr * qstr)
  729. {
  730. IDTYPE pipeid = *((IDTYPE *) id);
  731. size_t len;
  732. if ((len = snprintf(uri, size, "pipe:%u", pipeid)) == size)
  733. return -ERANGE;
  734. if (qstr)
  735. qstrsetstr(qstr, uri, len);
  736. return len;
  737. }
  738. int create_pipe (IDTYPE * id, char * uri, size_t size, PAL_HANDLE * hdl,
  739. struct shim_qstr * qstr)
  740. {
  741. IDTYPE pipeid;
  742. int ret = create_unique(&name_pipe, &open_pipe, &pipe_addr,
  743. uri, size, &pipeid, hdl, qstr);
  744. if (ret > 0 && id)
  745. *id = pipeid;
  746. return ret;
  747. }
  748. static int name_path (char * path, size_t size, void * id)
  749. {
  750. unsigned int suffix;
  751. int prefix_len = strlen(path);
  752. size_t len;
  753. int ret = DkRandomBitsRead(&suffix, sizeof(suffix));
  754. if (ret < 0)
  755. return -convert_pal_errno(-ret);
  756. len = snprintf(path + prefix_len, size - prefix_len, "%08x", suffix);
  757. if (len == size)
  758. return -ERANGE;
  759. *((unsigned int *) id) = suffix;
  760. return prefix_len + len;
  761. }
  762. static int open_dir (const char * path, void * obj)
  763. {
  764. struct shim_handle * dir = NULL;
  765. if (obj) {
  766. dir = get_new_handle();
  767. if (!dir)
  768. return -ENOMEM;
  769. }
  770. int ret = open_namei(dir, NULL, path, O_CREAT|O_EXCL|O_DIRECTORY, 0700,
  771. NULL);
  772. if (ret < 0)
  773. return ret = -EEXIST ? 1 : ret;
  774. if (obj)
  775. *((struct shim_handle **) obj) = dir;
  776. return 0;
  777. }
  778. static int open_file (const char * path, void * obj)
  779. {
  780. struct shim_handle * file = NULL;
  781. if (obj) {
  782. file = get_new_handle();
  783. if (!file)
  784. return -ENOMEM;
  785. }
  786. int ret = open_namei(file, NULL, path, O_CREAT|O_EXCL|O_RDWR, 0600,
  787. NULL);
  788. if (ret < 0)
  789. return ret = -EEXIST ? 1 : ret;
  790. if (obj)
  791. *((struct shim_handle **) obj) = file;
  792. return 0;
  793. }
  794. static int open_pal_handle (const char * uri, void * obj)
  795. {
  796. PAL_HANDLE hdl;
  797. if (strpartcmp_static(uri, "dev:"))
  798. hdl = DkStreamOpen(uri, 0,
  799. PAL_SHARE_OWNER_X|PAL_SHARE_OWNER_W|
  800. PAL_SHARE_OWNER_R,
  801. PAL_CREATE_TRY|PAL_CREATE_ALWAYS,
  802. 0);
  803. else
  804. hdl = DkStreamOpen(uri, PAL_ACCESS_RDWR,
  805. PAL_SHARE_OWNER_W|PAL_SHARE_OWNER_R,
  806. PAL_CREATE_TRY|PAL_CREATE_ALWAYS,
  807. 0);
  808. if (!hdl) {
  809. if (PAL_NATIVE_ERRNO == PAL_ERROR_STREAMEXIST)
  810. return 0;
  811. else
  812. return -PAL_ERRNO;
  813. }
  814. if (obj) {
  815. *((PAL_HANDLE *) obj) = hdl;
  816. } else {
  817. DkObjectClose(hdl);
  818. }
  819. return 0;
  820. }
  821. static int output_path (char * path, size_t size, const void * id,
  822. struct shim_qstr * qstr)
  823. {
  824. size_t len = strlen(path);
  825. // API compatibility
  826. __UNUSED(size);
  827. __UNUSED(id);
  828. if (qstr)
  829. qstrsetstr(qstr, path, len);
  830. return len;
  831. }
  832. int create_dir (const char * prefix, char * path, size_t size,
  833. struct shim_handle ** hdl)
  834. {
  835. unsigned int suffix;
  836. if (prefix) {
  837. size_t len = strlen(prefix);
  838. if (len >= size)
  839. return -ERANGE;
  840. memcpy(path, prefix, len + 1);
  841. }
  842. return create_unique(&name_path, &open_dir, &output_path, path, size,
  843. &suffix, hdl, NULL);
  844. }
  845. int create_file (const char * prefix, char * path, size_t size,
  846. struct shim_handle ** hdl)
  847. {
  848. unsigned int suffix;
  849. if (prefix) {
  850. size_t len = strlen(prefix);
  851. if (len >= size)
  852. return -ERANGE;
  853. memcpy(path, prefix, len + 1);
  854. }
  855. return create_unique(&name_path, &open_file, &output_path, path, size,
  856. &suffix, hdl, NULL);
  857. }
  858. int create_handle (const char * prefix, char * uri, size_t size,
  859. PAL_HANDLE * hdl, unsigned int * id)
  860. {
  861. unsigned int suffix;
  862. if (prefix) {
  863. size_t len = strlen(prefix);
  864. if (len >= size)
  865. return -ERANGE;
  866. memcpy(uri, prefix, len + 1);
  867. }
  868. return create_unique(&name_path, &open_pal_handle, &output_path, uri, size,
  869. id ? : &suffix, hdl, NULL);
  870. }
  871. void check_stack_hook (void)
  872. {
  873. struct shim_thread * cur_thread = get_cur_thread();
  874. void * rsp;
  875. __asm__ volatile ("movq %%rsp, %0" : "=r"(rsp) :: "memory");
  876. if (rsp <= cur_thread->stack_top && rsp > cur_thread->stack) {
  877. if ((uintptr_t) rsp - (uintptr_t) cur_thread->stack < PAL_CB(pagesize))
  878. SYS_PRINTF("*** stack is almost drained (RSP = %p, stack = %p-%p) ***\n",
  879. rsp, cur_thread->stack, cur_thread->stack_top);
  880. } else {
  881. SYS_PRINTF("*** context dismatched with thread stack (RSP = %p, stack = %p-%p) ***\n",
  882. rsp, cur_thread->stack, cur_thread->stack_top);
  883. }
  884. }
  885. #ifdef PROFILE
  886. static void print_profile_result (PAL_HANDLE hdl, struct shim_profile * root,
  887. int level)
  888. {
  889. unsigned long total_interval_time = 0;
  890. unsigned long total_interval_count = 0;
  891. for (size_t i = 0 ; i < N_PROFILE ; i++) {
  892. struct shim_profile * profile = &PROFILES[i];
  893. if (profile->root != root || profile->disabled)
  894. continue;
  895. switch (profile->type) {
  896. case OCCURENCE: {
  897. unsigned int count =
  898. atomic_read(&profile->val.occurence.count);
  899. if (count) {
  900. for (int j = 0 ; j < level ; j++)
  901. __SYS_FPRINTF(hdl, " ");
  902. __SYS_FPRINTF(hdl, "- %s: %u times\n", profile->name, count);
  903. }
  904. break;
  905. }
  906. case INTERVAL: {
  907. unsigned int count =
  908. atomic_read(&profile->val.interval.count);
  909. if (count) {
  910. unsigned long time =
  911. atomic_read(&profile->val.interval.time);
  912. unsigned long ind_time = time / count;
  913. total_interval_time += time;
  914. total_interval_count += count;
  915. for (int j = 0 ; j < level ; j++)
  916. __SYS_FPRINTF(hdl, " ");
  917. __SYS_FPRINTF(hdl, "- (%11.11lu) %s: %u times, %lu msec\n",
  918. time, profile->name, count, ind_time);
  919. }
  920. break;
  921. }
  922. case CATEGORY:
  923. for (int j = 0 ; j < level ; j++)
  924. __SYS_FPRINTF(hdl, " ");
  925. __SYS_FPRINTF(hdl, "- %s:\n", profile->name);
  926. print_profile_result(hdl, profile, level + 1);
  927. break;
  928. }
  929. }
  930. if (total_interval_count) {
  931. __SYS_FPRINTF(hdl, " - (%11.11lu) total: %lu times, %lu msec\n",
  932. total_interval_time, total_interval_count,
  933. total_interval_time / total_interval_count);
  934. }
  935. }
  936. #endif /* PROFILE */
  937. static struct atomic_int in_terminate = { .counter = 0, };
  938. noreturn void shim_terminate (int err)
  939. {
  940. debug("teminating the whole process (%d)\n", err);
  941. /* do last clean-up of the process */
  942. shim_clean(err);
  943. DkProcessExit(err);
  944. }
  945. /* cleanup and terminate process, preserve exit code if err == 0 */
  946. int shim_clean (int err)
  947. {
  948. /* preventing multiple cleanup, this is mostly caused by
  949. assertion in shim_clean */
  950. if (atomic_inc_return(&in_terminate) > 1)
  951. return 0;
  952. if (err != 0)
  953. cur_process.exit_code = err;
  954. store_all_msg_persist();
  955. #ifdef PROFILE
  956. if (ENTER_TIME) {
  957. switch (shim_get_tls()->context.orig_rax) {
  958. case __NR_exit_group:
  959. SAVE_PROFILE_INTERVAL_SINCE(syscall_exit_group, ENTER_TIME);
  960. break;
  961. case __NR_exit:
  962. SAVE_PROFILE_INTERVAL_SINCE(syscall_exit, ENTER_TIME);
  963. break;
  964. }
  965. }
  966. if (ipc_cld_profile_send()) {
  967. MASTER_LOCK();
  968. PAL_HANDLE hdl = __open_shim_stdio();
  969. if (hdl) {
  970. __SYS_FPRINTF(hdl, "******************************\n");
  971. __SYS_FPRINTF(hdl, "profiling:\n");
  972. print_profile_result(hdl, &profile_root, 0);
  973. __SYS_FPRINTF(hdl, "******************************\n");
  974. }
  975. MASTER_UNLOCK();
  976. DkObjectClose(hdl);
  977. }
  978. #endif
  979. del_all_ipc_ports(0);
  980. if (shim_stdio && shim_stdio != (PAL_HANDLE) -1)
  981. DkObjectClose(shim_stdio);
  982. shim_stdio = NULL;
  983. debug("process %u exited with status %d\n", cur_process.vmid & 0xFFFF, cur_process.exit_code);
  984. MASTER_LOCK();
  985. DkProcessExit(cur_process.exit_code);
  986. return 0;
  987. }
  988. int message_confirm (const char * message, const char * options)
  989. {
  990. char answer;
  991. int noptions = strlen(options);
  992. char * option_str = __alloca(noptions * 2 + 3), * str = option_str;
  993. int ret = 0;
  994. *(str++) = ' ';
  995. *(str++) = '[';
  996. for (int i = 0 ; i < noptions ; i++) {
  997. *(str++) = options[i];
  998. *(str++) = '/';
  999. }
  1000. str--;
  1001. *(str++) = ']';
  1002. *(str++) = ' ';
  1003. MASTER_LOCK();
  1004. PAL_HANDLE hdl = __open_shim_stdio();
  1005. if (!hdl) {
  1006. MASTER_UNLOCK();
  1007. return -EACCES;
  1008. }
  1009. #define WRITE(buf, len) \
  1010. ({ int _ret = DkStreamWrite(hdl, 0, len, (void*)(buf), NULL); \
  1011. _ret ? : -PAL_ERRNO; })
  1012. #define READ(buf, len) \
  1013. ({ int _ret = DkStreamRead(hdl, 0, len, buf, NULL, 0); \
  1014. _ret ? : -PAL_ERRNO; })
  1015. if ((ret = WRITE(message, strlen(message))) < 0)
  1016. goto out;
  1017. if ((ret = WRITE(option_str, noptions * 2 + 3)) < 0)
  1018. goto out;
  1019. if ((ret = READ(&answer, 1)) < 0)
  1020. goto out;
  1021. out:
  1022. DkObjectClose(hdl);
  1023. MASTER_UNLOCK();
  1024. return (ret < 0) ? ret : answer;
  1025. }