shim_checkpoint.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201
  1. /* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
  2. /* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
  3. /* Copyright (C) 2014 Stony Brook University
  4. This file is part of Graphene Library OS.
  5. Graphene Library OS is free software: you can redistribute it and/or
  6. modify it under the terms of the GNU Lesser General Public License
  7. as published by the Free Software Foundation, either version 3 of the
  8. License, or (at your option) any later version.
  9. Graphene Library OS is distributed in the hope that it will be useful,
  10. but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. GNU Lesser General Public License for more details.
  13. You should have received a copy of the GNU Lesser General Public License
  14. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  15. /*
  16. * shim_checkpoint.c
  17. *
  18. * This file contains codes for checkpoint / migration scheme of library OS.
  19. */
  20. #include <shim_internal.h>
  21. #include <shim_utils.h>
  22. #include <shim_thread.h>
  23. #include <shim_handle.h>
  24. #include <shim_vma.h>
  25. #include <shim_fs.h>
  26. #include <shim_checkpoint.h>
  27. #include <shim_ipc.h>
  28. #include <shim_profile.h>
  29. #include <pal.h>
  30. #include <pal_error.h>
  31. #include <list.h>
  32. #include <stdarg.h>
  33. #include <asm/fcntl.h>
  34. #include <asm/mman.h>
  35. DEFINE_PROFILE_CATAGORY(migrate, );
  36. DEFINE_PROFILE_CATAGORY(checkpoint, migrate);
  37. DEFINE_PROFILE_INTERVAL(checkpoint_create_map, checkpoint);
  38. DEFINE_PROFILE_INTERVAL(checkpoint_copy, checkpoint);
  39. DEFINE_PROFILE_CATAGORY(checkpoint_func, checkpoint);
  40. DEFINE_PROFILE_INTERVAL(checkpoint_destroy_map, checkpoint);
  41. DEFINE_PROFILE_OCCURENCE(checkpoint_count, checkpoint);
  42. DEFINE_PROFILE_OCCURENCE(checkpoint_total_size, checkpoint);
  43. DEFINE_PROFILE_CATAGORY(resume, migrate);
  44. DEFINE_PROFILE_INTERVAL(child_created_in_new_process, resume);
  45. DEFINE_PROFILE_INTERVAL(child_wait_header, resume);
  46. DEFINE_PROFILE_INTERVAL(child_receive_header, resume);
  47. DEFINE_PROFILE_INTERVAL(do_migration, resume);
  48. DEFINE_PROFILE_INTERVAL(child_load_checkpoint_by_gipc, resume);
  49. DEFINE_PROFILE_INTERVAL(child_load_memory_by_gipc, resume);
  50. DEFINE_PROFILE_INTERVAL(child_load_checkpoint_on_pipe, resume);
  51. DEFINE_PROFILE_INTERVAL(child_receive_handles, resume);
  52. DEFINE_PROFILE_INTERVAL(restore_checkpoint, resume);
  53. DEFINE_PROFILE_CATAGORY(resume_func, resume);
  54. DEFINE_PROFILE_INTERVAL(child_total_migration_time, resume);
  55. #define CP_HASH_SIZE 256
  56. #define CP_HASH(addr) ((hashfunc((ptr_t)(addr))) & (CP_HASH_SIZE - 1))
  57. typedef uint16_t FASTHASHTYPE;
  58. #define CP_MAP_ENTRY_NUM 64
  59. DEFINE_LIST(cp_map_entry);
  60. struct cp_map_entry
  61. {
  62. LIST_TYPE(cp_map_entry) hlist;
  63. struct shim_cp_map_entry entry;
  64. };
  65. DEFINE_LISTP(cp_map_entry);
  66. struct cp_map {
  67. struct cp_map_buffer {
  68. struct cp_map_buffer * next;
  69. int num, cnt;
  70. struct cp_map_entry entries[0];
  71. } * buffers;
  72. struct hash_map {
  73. LISTP_TYPE(cp_map_entry) head[CP_HASH_SIZE];
  74. } map;
  75. };
  76. void * create_cp_map (void)
  77. {
  78. void * data = malloc(sizeof(struct cp_map) + sizeof(struct cp_map_buffer) +
  79. sizeof(struct cp_map_entry) * CP_MAP_ENTRY_NUM);
  80. if (!data)
  81. return NULL;
  82. struct cp_map * map = (struct cp_map *) data;
  83. struct cp_map_buffer * buffer =
  84. (struct cp_map_buffer *) (data + sizeof(struct cp_map));
  85. memset(map, 0, sizeof(*map));
  86. map->buffers = buffer;
  87. buffer->next = NULL;
  88. buffer->num = CP_MAP_ENTRY_NUM;
  89. buffer->cnt = 0;
  90. return (void *) map;
  91. }
  92. void destroy_cp_map (void * map)
  93. {
  94. struct cp_map * m = (struct cp_map *) map;
  95. struct cp_map_buffer * buffer = m->buffers, * next;
  96. for (next = buffer ? buffer->next : NULL ;
  97. buffer && next ;
  98. buffer = next, next = next ? next->next : NULL)
  99. free(buffer);
  100. free(m);
  101. }
  102. static inline
  103. struct cp_map_buffer * extend_cp_map (struct cp_map * map)
  104. {
  105. struct cp_map_buffer * buffer =
  106. malloc(sizeof(struct cp_map_buffer) +
  107. sizeof(struct cp_map_entry) * CP_MAP_ENTRY_NUM);
  108. if (!buffer)
  109. return NULL;
  110. buffer->next = map->buffers;
  111. map->buffers = buffer;
  112. buffer->num = CP_MAP_ENTRY_NUM;
  113. buffer->cnt = 0;
  114. return buffer;
  115. }
  116. struct shim_cp_map_entry *
  117. get_cp_map_entry (void * map, void * addr, bool create)
  118. {
  119. struct cp_map * m = (struct cp_map *) map;
  120. FASTHASHTYPE hash = CP_HASH(addr);
  121. LISTP_TYPE(cp_map_entry) * head = &m->map.head[hash];
  122. struct cp_map_entry * tmp;
  123. struct shim_cp_map_entry * e = NULL;
  124. listp_for_each_entry(tmp, head, hlist)
  125. if (tmp->entry.addr == addr)
  126. e = &tmp->entry;
  127. if (create && !e) {
  128. struct cp_map_buffer * buffer = m->buffers;
  129. if (buffer->cnt == buffer->num)
  130. buffer = extend_cp_map(m);
  131. struct cp_map_entry *new = &buffer->entries[buffer->cnt++];
  132. INIT_LIST_HEAD(new, hlist);
  133. listp_add(new, head, hlist);
  134. new->entry.addr = addr;
  135. new->entry.off = 0;
  136. e = &new->entry;
  137. }
  138. return e;
  139. }
  140. BEGIN_CP_FUNC(memory)
  141. {
  142. struct shim_mem_entry * entry =
  143. (void *) (base + ADD_CP_OFFSET(sizeof(struct shim_mem_entry)));
  144. entry->addr = obj;
  145. entry->size = size;
  146. entry->paddr = NULL;
  147. entry->prot = PAL_PROT_READ|PAL_PROT_WRITE;
  148. entry->data = NULL;
  149. entry->prev = store->last_mem_entry;
  150. store->last_mem_entry = entry;
  151. store->mem_nentries++;
  152. store->mem_size += size;
  153. if (objp)
  154. *objp = entry;
  155. }
  156. END_CP_FUNC_NO_RS(memory)
  157. BEGIN_CP_FUNC(palhdl)
  158. {
  159. ptr_t off = ADD_CP_OFFSET(sizeof(struct shim_palhdl_entry));
  160. struct shim_palhdl_entry * entry = (void *) (base + off);
  161. entry->handle = (PAL_HANDLE) obj;
  162. entry->uri = NULL;
  163. entry->phandle = NULL;
  164. entry->prev = store->last_palhdl_entry;
  165. store->last_palhdl_entry = entry;
  166. store->palhdl_nentries++;
  167. ADD_CP_FUNC_ENTRY(off);
  168. if (objp)
  169. *objp = entry;
  170. }
  171. END_CP_FUNC(palhdl)
  172. BEGIN_RS_FUNC(palhdl)
  173. {
  174. struct shim_palhdl_entry * ent = (void *) (base + GET_CP_FUNC_ENTRY());
  175. if (ent->phandle && !ent->phandle && ent->uri) {
  176. /* XXX: reopen the stream */
  177. }
  178. }
  179. END_RS_FUNC(palhdl)
  180. BEGIN_CP_FUNC(migratable)
  181. {
  182. struct shim_mem_entry * mem_entry;
  183. DO_CP_SIZE(memory, &__migratable, &__migratable_end - &__migratable,
  184. &mem_entry);
  185. struct shim_cp_entry * entry = ADD_CP_FUNC_ENTRY(0);
  186. mem_entry->paddr = (void **) &entry->cp_un.cp_val;
  187. }
  188. END_CP_FUNC(migratable)
  189. BEGIN_RS_FUNC(migratable)
  190. {
  191. void * data = (void *) GET_CP_FUNC_ENTRY();
  192. CP_REBASE(data);
  193. memcpy(&__migratable, data, &__migratable_end - &__migratable);
  194. }
  195. END_RS_FUNC(migratable)
  196. BEGIN_CP_FUNC(environ)
  197. {
  198. const char ** e, ** envp = (void *) obj;
  199. int nenvp = 0;
  200. int envp_bytes = 0;
  201. for (e = envp ; *e ; e++) {
  202. nenvp++;
  203. envp_bytes += strlen(*e) + 1;
  204. }
  205. ptr_t off = ADD_CP_OFFSET(sizeof(char *) * (nenvp + 1) + envp_bytes);
  206. const char ** new_envp = (void *) base + off;
  207. char * ptr = (void *) base + off + sizeof(char *) * (nenvp + 1);
  208. for (int i = 0 ; i < nenvp ; i++) {
  209. int len = strlen(envp[i]);
  210. new_envp[i] = ptr;
  211. memcpy(ptr, envp[i], len + 1);
  212. ptr += len + 1;
  213. }
  214. new_envp[nenvp] = NULL;
  215. ADD_CP_FUNC_ENTRY(off);
  216. }
  217. END_CP_FUNC(environ)
  218. BEGIN_RS_FUNC(environ)
  219. {
  220. const char ** envp = (void *) base + GET_CP_FUNC_ENTRY();
  221. const char ** e;
  222. for (e = envp ; *e ; e++) {
  223. CP_REBASE(*e);
  224. DEBUG_RS("%s", *e);
  225. }
  226. initial_envp = envp;
  227. }
  228. END_RS_FUNC(environ)
  229. BEGIN_CP_FUNC(qstr)
  230. {
  231. struct shim_qstr * qstr = (struct shim_qstr *) obj;
  232. if (qstr->len < QSTR_SIZE) {
  233. if (qstr->oflow) {
  234. memcpy(qstr->name, qstr->oflow, qstr->len + 1);
  235. qstr->oflow = NULL;
  236. }
  237. } else {
  238. struct shim_str * str =
  239. (void *) (base + ADD_CP_OFFSET(qstr->len + 1));
  240. memcpy(str, qstr->oflow, qstr->len + 1);
  241. qstr->oflow = str;
  242. ADD_CP_FUNC_ENTRY((ptr_t) qstr - base);
  243. }
  244. }
  245. END_CP_FUNC(qstr)
  246. BEGIN_RS_FUNC(qstr)
  247. {
  248. struct shim_qstr * qstr = (void *) (base + GET_CP_FUNC_ENTRY());
  249. CP_REBASE(qstr->oflow);
  250. }
  251. END_RS_FUNC(qstr)
  252. BEGIN_CP_FUNC(gipc)
  253. {
  254. ptr_t off = ADD_CP_OFFSET(sizeof(struct shim_gipc_entry));
  255. void * send_addr = (void *) ALIGN_DOWN(obj);
  256. size_t send_size = (void *) ALIGN_UP(obj + size) - send_addr;
  257. struct shim_gipc_entry * entry = (void *) (base + off);
  258. entry->mem.addr = send_addr;
  259. entry->mem.size = send_size;
  260. entry->mem.prot = PAL_PROT_READ|PAL_PROT_WRITE;
  261. entry->mem.prev = (void *) store->last_gipc_entry;
  262. store->last_gipc_entry = entry;
  263. store->gipc_nentries++;
  264. #if HASH_GIPC == 1
  265. struct md5_ctx ctx;
  266. md5_init(&ctx);
  267. md5_update(&ctx, send_addr, allocsize);
  268. md5_final(&ctx);
  269. entry->first_hash = *(unsigned long *) ctx.digest;
  270. #endif /* HASH_GIPC == 1 */
  271. ADD_CP_FUNC_ENTRY(off);
  272. if (objp)
  273. *objp = entry;
  274. }
  275. END_CP_FUNC(gipc)
  276. BEGIN_RS_FUNC(gipc)
  277. {
  278. #if HASH_GIPC == 1
  279. struct shim_gipc_entry * entry = (void *) (base + GET_CP_FUNC_ENTRY());
  280. PAL_FLG pal_prot = PAL_PROT(entry->prot, 0);
  281. if (!(pal_prot & PROT_READ))
  282. DkVirtualMemoryProtect(entry->addr, entry->npages * allocsize,
  283. pal_prot|PAL_PROT_READ);
  284. struct md5_ctx ctx;
  285. md5_init(&ctx);
  286. md5_update(&ctx, entry->addr, allocsize);
  287. md5_final(&ctx);
  288. assert(*(unsigned long *) ctx.digest == entry->first_hash);
  289. if (!(pal_prot & PAL_PROT_READ))
  290. DkVirtualMemoryProtect(entry->addr, entry->npages * allocsize,
  291. pal_prot);
  292. #endif /* HASH_GIPC == 1 */
  293. }
  294. END_RS_FUNC(gipc)
  295. static int send_checkpoint_by_gipc (PAL_HANDLE gipc_store,
  296. struct shim_cp_store * store)
  297. {
  298. PAL_PTR hdr_addr = (PAL_PTR) store->base;
  299. PAL_NUM hdr_size = (PAL_NUM) store->offset + store->mem_size;
  300. assert(ALIGNED(hdr_addr));
  301. int mem_nentries = store->mem_nentries;
  302. if (mem_nentries) {
  303. struct shim_mem_entry ** mem_entries =
  304. __alloca(sizeof(struct shim_mem_entry *) * mem_nentries);
  305. int mem_cnt = mem_nentries;
  306. struct shim_mem_entry * mem_ent = store->last_mem_entry;
  307. for (; mem_ent ; mem_ent = mem_ent->prev) {
  308. if (!mem_cnt)
  309. return -EINVAL;
  310. mem_entries[--mem_cnt] = mem_ent;
  311. }
  312. mem_entries += mem_cnt;
  313. mem_nentries -= mem_cnt;
  314. for (int i = 0 ; i < mem_nentries ; i++) {
  315. void * mem_addr = (void *) store->base +
  316. __ADD_CP_OFFSET(mem_entries[i]->size);
  317. assert(store->offset <= hdr_size);
  318. memcpy(mem_addr, mem_entries[i]->addr, mem_entries[i]->size);
  319. mem_entries[i]->data = mem_addr;
  320. }
  321. }
  322. hdr_size = ALIGN_UP(hdr_size);
  323. int npages = DkPhysicalMemoryCommit(gipc_store, 1, &hdr_addr, &hdr_size, 0);
  324. if (!npages)
  325. return -EPERM;
  326. int nentries = store->gipc_nentries;
  327. PAL_PTR * gipc_addrs = __alloca(sizeof(PAL_PTR) * nentries);
  328. PAL_NUM * gipc_sizes = __alloca(sizeof(PAL_NUM) * nentries);
  329. int total_pages = 0;
  330. int cnt = nentries;
  331. struct shim_gipc_entry * ent = store->last_gipc_entry;
  332. for (; ent ; ent = (void *) ent->mem.prev) {
  333. if (!cnt)
  334. return -EINVAL;
  335. cnt--;
  336. gipc_addrs[cnt] = ent->mem.addr;
  337. gipc_sizes[cnt] = ent->mem.size;
  338. total_pages += ent->mem.size / allocsize;
  339. }
  340. gipc_addrs += cnt;
  341. gipc_sizes += cnt;
  342. nentries -= cnt;
  343. /* Chia-Che: sending an empty page can't ever be a smart idea.
  344. we might rather fail here */
  345. npages = DkPhysicalMemoryCommit(gipc_store, nentries, gipc_addrs,
  346. gipc_sizes, 0);
  347. if (npages < total_pages) {
  348. debug("gipc supposed to send %d pages, but only %d pages sent\n",
  349. total_pages, npages);
  350. return -ENOMEM;
  351. }
  352. ADD_PROFILE_OCCURENCE(migrate_send_gipc_pages, npages);
  353. return 0;
  354. }
  355. static int send_checkpoint_on_stream (PAL_HANDLE stream,
  356. struct shim_cp_store * store)
  357. {
  358. int mem_nentries = store->mem_nentries;
  359. struct shim_mem_entry ** mem_entries;
  360. if (mem_nentries) {
  361. mem_entries = __alloca(sizeof(struct shim_mem_entry *) * mem_nentries);
  362. int mem_cnt = mem_nentries;
  363. struct shim_mem_entry * mem_ent = store->last_mem_entry;
  364. for (; mem_ent ; mem_ent = mem_ent->prev) {
  365. if (!mem_cnt)
  366. return -EINVAL;
  367. mem_entries[--mem_cnt] = mem_ent;
  368. }
  369. void * mem_addr = (void *) store->base + store->offset;
  370. mem_entries += mem_cnt;
  371. mem_nentries -= mem_cnt;
  372. for (int i = 0 ; i < mem_nentries ; i++) {
  373. int mem_size = mem_entries[i]->size;
  374. mem_entries[i]->data = mem_addr;
  375. mem_addr += mem_size;
  376. }
  377. }
  378. int total_bytes = store->offset;
  379. int bytes = 0;
  380. do {
  381. int ret = DkStreamWrite(stream, 0, total_bytes - bytes,
  382. (void *) store->base + bytes, NULL);
  383. if (!ret)
  384. return -PAL_ERRNO;
  385. bytes += ret;
  386. } while (bytes < total_bytes);
  387. ADD_PROFILE_OCCURENCE(migrate_send_on_stream, total_bytes);
  388. for (int i = 0 ; i < mem_nentries ; i++) {
  389. int mem_size = mem_entries[i]->size;
  390. void * mem_addr = mem_entries[i]->addr;
  391. bytes = 0;
  392. do {
  393. int ret = DkStreamWrite(stream, 0, mem_size - bytes,
  394. mem_addr + bytes, NULL);
  395. if (!ret)
  396. return -PAL_ERRNO;
  397. bytes += ret;
  398. } while (bytes < mem_entries[i]->size);
  399. mem_entries[i]->size = mem_size;
  400. ADD_PROFILE_OCCURENCE(migrate_send_on_stream, mem_size);
  401. }
  402. return 0;
  403. }
  404. static int restore_gipc (PAL_HANDLE gipc, struct gipc_header * hdr, ptr_t base,
  405. long rebase)
  406. {
  407. struct shim_gipc_entry * gipc_entries = (void *) (base + hdr->entoffset);
  408. int nentries = hdr->nentries;
  409. if (!nentries)
  410. return 0;
  411. debug("restore memory by gipc: %d entries\n", nentries);
  412. struct shim_gipc_entry ** entries =
  413. __alloca(sizeof(struct shim_gipc_entry *) * nentries);
  414. struct shim_gipc_entry * entry = gipc_entries;
  415. int cnt = nentries;
  416. while (entry) {
  417. CP_REBASE(entry->mem.prev);
  418. CP_REBASE(entry->mem.paddr);
  419. if (!cnt)
  420. return -EINVAL;
  421. entries[--cnt] = entry;
  422. entry = (void *) entry->mem.prev;
  423. }
  424. entries += cnt;
  425. nentries -= cnt;
  426. PAL_PTR * addrs = __alloca(sizeof(PAL_PTR) * nentries);
  427. PAL_NUM * sizes = __alloca(sizeof(PAL_NUM) * nentries);
  428. PAL_FLG * prots = __alloca(sizeof(PAL_FLG) * nentries);
  429. for (int i = 0 ; i < nentries ; i++) {
  430. addrs[i] = entries[i]->mem.paddr ? NULL : (PAL_PTR) entries[i]->mem.addr;
  431. sizes[i] = entries[i]->mem.size;
  432. prots[i] = entries[i]->mem.prot;
  433. }
  434. if (!DkPhysicalMemoryMap(gipc, nentries, addrs, sizes, prots))
  435. return -PAL_ERRNO;
  436. for (int i = 0 ; i < nentries ; i++)
  437. if (entries[i]->mem.paddr)
  438. *(void **) entries[i]->mem.paddr = (void *) addrs[i];
  439. return 0;
  440. }
  441. int restore_checkpoint (struct cp_header * cphdr, struct mem_header * memhdr,
  442. ptr_t base, int type)
  443. {
  444. ptr_t cpoffset = cphdr->offset;
  445. ptr_t * offset = &cpoffset;
  446. long rebase = base - (ptr_t) cphdr->addr;
  447. int ret = 0;
  448. if (type)
  449. debug("restore checkpoint at %p rebased from %p (%s only)\n",
  450. base, cphdr->addr, CP_FUNC_NAME(type));
  451. else
  452. debug("restore checkpoint at %p rebased from %p\n",
  453. base, cphdr->addr);
  454. if (memhdr && memhdr->nentries) {
  455. struct shim_mem_entry * entry =
  456. (void *) (base + memhdr->entoffset);
  457. for (; entry ; entry = entry->prev) {
  458. CP_REBASE(entry->prev);
  459. CP_REBASE(entry->paddr);
  460. if (entry->paddr) {
  461. *entry->paddr = entry->data;
  462. } else {
  463. debug("memory entry [%p]: %p-%p\n", entry, entry->addr,
  464. entry->addr + entry->size);
  465. PAL_PTR addr = ALIGN_DOWN(entry->addr);
  466. PAL_NUM size = ALIGN_UP(entry->addr + entry->size) -
  467. (void *) addr;
  468. PAL_FLG prot = entry->prot;
  469. if (!DkVirtualMemoryAlloc(addr, size, 0, prot|PAL_PROT_WRITE)) {
  470. debug("failed allocating %p-%p\n", addr, addr + size);
  471. return -PAL_ERRNO;
  472. }
  473. CP_REBASE(entry->data);
  474. memcpy(entry->addr, entry->data, entry->size);
  475. if (!(entry->prot & PAL_PROT_WRITE) &&
  476. !DkVirtualMemoryProtect(addr, size, prot)) {
  477. debug("failed protecting %p-%p (ignored)\n", addr, addr + size);
  478. }
  479. }
  480. }
  481. }
  482. struct shim_cp_entry * cpent = NEXT_CP_ENTRY();
  483. while (cpent) {
  484. if (cpent->cp_type < CP_FUNC_BASE)
  485. goto next;
  486. if (type && cpent->cp_type != type)
  487. goto next;
  488. rs_func rs = (&__rs_func) [cpent->cp_type - CP_FUNC_BASE];
  489. ret = (*rs) (cpent, base, offset, rebase);
  490. if (ret < 0) {
  491. debug("restoring %s failed at %p (err=%d)\n", CP_FUNC_NAME(cpent->cp_type),
  492. base + offset, -ret);
  493. return ret;
  494. }
  495. next:
  496. cpent = NEXT_CP_ENTRY();
  497. }
  498. debug("successfully restore checkpoint loaded at %p - %p\n",
  499. base, base + cphdr->size);
  500. return 0;
  501. }
  502. int init_from_checkpoint_file (const char * filename,
  503. struct newproc_cp_header * hdr,
  504. void ** cpptr)
  505. {
  506. struct shim_dentry * dir = NULL;
  507. int ret;
  508. /* XXX: Not sure what to do here yet */
  509. assert(0);
  510. ret = path_lookupat(NULL, filename, LOOKUP_ACCESS|LOOKUP_DIRECTORY, &dir, NULL);
  511. if (ret < 0)
  512. return ret;
  513. struct shim_mount * fs = dir->fs;
  514. struct shim_dirent * dirent;
  515. if (!fs->d_ops || !fs->d_ops->readdir) {
  516. ret = -EACCES;
  517. goto out;
  518. }
  519. if ((ret = fs->d_ops->readdir(dir, &dirent)) < 0)
  520. goto out;
  521. struct shim_dentry * first = NULL;
  522. struct shim_dirent * d = dirent;
  523. for ( ; d ; d = d->next) {
  524. struct shim_dentry * file;
  525. if ((ret = lookup_dentry(dir, d->name, strlen(d->name), false,
  526. &file, dir->fs)) < 0)
  527. continue;
  528. if (file->state & DENTRY_NEGATIVE)
  529. continue;
  530. if (!first) {
  531. first = file;
  532. continue;
  533. }
  534. const char * argv[3];
  535. argv[0] = "-resume-file";
  536. argv[1] = dentry_get_path(file, true, NULL);
  537. argv[2] = 0;
  538. PAL_HANDLE proc = DkProcessCreate(NULL, 0, argv);
  539. if (!proc) {
  540. ret = -PAL_ERRNO;
  541. goto out;
  542. }
  543. put_dentry(file);
  544. }
  545. if (first) {
  546. ret = restore_from_file(dentry_get_path(first, true, NULL), hdr, cpptr);
  547. put_dentry(first);
  548. }
  549. free(dirent);
  550. out:
  551. put_dentry(dir);
  552. return ret;
  553. }
  554. int restore_from_file (const char * filename, struct newproc_cp_header * hdr,
  555. void ** cpptr)
  556. {
  557. struct shim_handle * file = get_new_handle();
  558. if (!file)
  559. return -ENOMEM;
  560. int ret = open_namei(file, NULL, filename, O_RDWR, 0, NULL);
  561. if (ret < 0) {
  562. put_handle(file);
  563. return ret;
  564. }
  565. struct shim_mount * fs = file->fs;
  566. open_handle(file);
  567. debug("restore %s\n", filename);
  568. struct cp_header cphdr;
  569. ret = fs->fs_ops->read(file, &cphdr, sizeof(struct cp_header));
  570. if (ret < 0)
  571. goto out;
  572. void * cpaddr = cphdr.addr;
  573. ret = fs->fs_ops->mmap(file, &cpaddr, ALIGN_UP(cphdr.size),
  574. PROT_READ|PROT_WRITE,
  575. MAP_PRIVATE|MAP_FILE, 0);
  576. if (ret < 0)
  577. goto out;
  578. hdr->hdr = cphdr;
  579. *cpptr = cpaddr;
  580. migrated_memory_start = cpaddr;
  581. migrated_memory_end = cpaddr + hdr->hdr.size;
  582. out:
  583. close_handle(file);
  584. return ret;
  585. }
  586. int send_handles_on_stream (PAL_HANDLE stream, struct shim_cp_store * store)
  587. {
  588. int nentries = store->palhdl_nentries;
  589. if (!nentries)
  590. return 0;
  591. struct shim_palhdl_entry ** entries =
  592. __alloca(sizeof(struct shim_palhdl_entry *) * nentries);
  593. struct shim_palhdl_entry * entry = store->last_palhdl_entry;
  594. int cnt = nentries;
  595. for ( ; entry ; entry = entry->prev)
  596. if (entry->handle) {
  597. if (!cnt)
  598. return -EINVAL;
  599. entries[--cnt] = entry;
  600. }
  601. entries += cnt;
  602. nentries -= cnt;
  603. for (int i = 0 ; i < nentries ; i++)
  604. if (!DkSendHandle(stream, entries[i]->handle))
  605. entries[i]->handle = NULL;
  606. return 0;
  607. }
  608. int receive_handles_on_stream (struct palhdl_header * hdr, ptr_t base,
  609. long rebase)
  610. {
  611. struct shim_palhdl_entry * palhdl_entries =
  612. (void *) (base + hdr->entoffset);
  613. int nentries = hdr->nentries;
  614. if (!nentries)
  615. return 0;
  616. debug("receive handles: %d entries\n", nentries);
  617. struct shim_palhdl_entry ** entries =
  618. __alloca(sizeof(struct shim_palhdl_entry *) * nentries);
  619. struct shim_palhdl_entry * entry = palhdl_entries;
  620. int cnt = nentries;
  621. for ( ; entry ; entry = entry->prev) {
  622. CP_REBASE(entry->prev);
  623. CP_REBASE(entry->phandle);
  624. if (!cnt)
  625. return -EINVAL;
  626. entries[--cnt] = entry;
  627. }
  628. entries += cnt;
  629. nentries -= cnt;
  630. for (int i = 0 ; i < nentries ; i++) {
  631. entry = entries[i];
  632. if (entry->handle) {
  633. PAL_HANDLE hdl = DkReceiveHandle(PAL_CB(parent_process));
  634. if (hdl) {
  635. *entry->phandle = hdl;
  636. continue;
  637. }
  638. }
  639. }
  640. return 0;
  641. }
  642. #define NTRIES 4
  643. static void * cp_alloc (struct shim_cp_store * store, void * addr, int size)
  644. {
  645. void * requested = addr;
  646. struct shim_vma * vma;
  647. int ret, n = 0;
  648. if (!requested) {
  649. again:
  650. if (n == NTRIES)
  651. return NULL;
  652. if (!(addr = get_unmapped_vma_for_cp(size)))
  653. return NULL;
  654. } else {
  655. ret = lookup_overlap_vma(addr, size, &vma);
  656. if (!ret) {
  657. if (vma->addr != addr || vma->length != size ||
  658. !(vma->flags & VMA_UNMAPPED)) {
  659. put_vma(vma);
  660. return NULL;
  661. }
  662. }
  663. }
  664. addr = (void *) DkVirtualMemoryAlloc(addr, size, 0,
  665. PAL_PROT_READ|PAL_PROT_WRITE);
  666. if (!addr) {
  667. if (!requested)
  668. goto again;
  669. return NULL;
  670. }
  671. if (requested && addr != requested) {
  672. DkVirtualMemoryFree(addr, size);
  673. return NULL;
  674. }
  675. return addr;
  676. }
  677. DEFINE_PROFILE_CATAGORY(migrate_proc, migrate);
  678. DEFINE_PROFILE_INTERVAL(migrate_create_process, migrate_proc);
  679. DEFINE_PROFILE_INTERVAL(migrate_create_gipc, migrate_proc);
  680. DEFINE_PROFILE_INTERVAL(migrate_connect_ipc, migrate_proc);
  681. DEFINE_PROFILE_INTERVAL(migrate_init_checkpoint, migrate_proc);
  682. DEFINE_PROFILE_INTERVAL(migrate_save_checkpoint, migrate_proc);
  683. DEFINE_PROFILE_INTERVAL(migrate_send_header, migrate_proc);
  684. DEFINE_PROFILE_INTERVAL(migrate_send_checkpoint, migrate_proc);
  685. DEFINE_PROFILE_OCCURENCE(migrate_send_on_stream, migrate_proc);
  686. DEFINE_PROFILE_OCCURENCE(migrate_send_gipc_pages, migrate_proc);
  687. DEFINE_PROFILE_INTERVAL(migrate_send_pal_handles, migrate_proc);
  688. DEFINE_PROFILE_INTERVAL(migrate_free_checkpoint, migrate_proc);
  689. DEFINE_PROFILE_INTERVAL(migrate_wait_response, migrate_proc);
  690. int do_migrate_process (int (*migrate) (struct shim_cp_store *,
  691. struct shim_thread *,
  692. struct shim_process *, va_list),
  693. struct shim_handle * exec,
  694. const char ** argv,
  695. struct shim_thread * thread, ...)
  696. {
  697. int ret = 0;
  698. struct shim_process * new_process = NULL;
  699. struct newproc_header hdr;
  700. struct shim_cp_store * cpstore = NULL;
  701. int bytes;
  702. memset(&hdr, 0, sizeof(hdr));
  703. #ifdef PROFILE
  704. unsigned long begin_create_time = GET_PROFILE_INTERVAL();
  705. unsigned long create_time = begin_create_time;
  706. #endif
  707. BEGIN_PROFILE_INTERVAL();
  708. PAL_HANDLE proc = DkProcessCreate(exec ? qstrgetstr(&exec->uri) :
  709. pal_control.executable,
  710. 0, argv);
  711. if (!proc) {
  712. ret = -PAL_ERRNO;
  713. goto err;
  714. }
  715. SAVE_PROFILE_INTERVAL(migrate_create_process);
  716. bool use_gipc = false;
  717. PAL_NUM gipc_key;
  718. PAL_HANDLE gipc_hdl = DkCreatePhysicalMemoryChannel(&gipc_key);
  719. if (gipc_hdl) {
  720. debug("created gipc store: gipc:%lu\n", gipc_key);
  721. use_gipc = true;
  722. SAVE_PROFILE_INTERVAL(migrate_create_gipc);
  723. } else {
  724. sys_printf("WARNING: no physical memory support, process creation "
  725. "will be slow.\n");
  726. }
  727. if (!(new_process = create_new_process(true))) {
  728. ret = -ENOMEM;
  729. goto err;
  730. }
  731. if (!(new_process->self = create_ipc_port(0, false))) {
  732. ret = -EACCES;
  733. goto err;
  734. }
  735. SAVE_PROFILE_INTERVAL(migrate_connect_ipc);
  736. cpstore = __alloca(sizeof(struct shim_cp_store));
  737. memset(cpstore, 0, sizeof(struct shim_cp_store));
  738. cpstore->alloc = cp_alloc;
  739. cpstore->use_gipc = use_gipc;
  740. cpstore->bound = CP_INIT_VMA_SIZE;
  741. while (1) {
  742. debug("try allocate checkpoint store (size = %d)\n", cpstore->bound);
  743. cpstore->base = (ptr_t) cp_alloc(cpstore, 0, cpstore->bound);
  744. if (cpstore->base)
  745. break;
  746. cpstore->bound >>= 1;
  747. if (cpstore->bound < allocsize)
  748. break;
  749. }
  750. if (!cpstore->base) {
  751. ret = -ENOMEM;
  752. debug("failed creating checkpoint store\n");
  753. goto err;
  754. }
  755. SAVE_PROFILE_INTERVAL(migrate_init_checkpoint);
  756. va_list ap;
  757. va_start(ap, thread);
  758. ret = (*migrate) (cpstore, thread, new_process, ap);
  759. va_end(ap);
  760. if (ret < 0) {
  761. debug("failed creating checkpoint (ret = %d)\n", ret);
  762. goto err;
  763. }
  764. SAVE_PROFILE_INTERVAL(migrate_save_checkpoint);
  765. unsigned long checkpoint_time = GET_PROFILE_INTERVAL();
  766. unsigned long checkpoint_size = cpstore->offset + cpstore->mem_size;
  767. debug("checkpoint of %u bytes created, %lu microsecond is spent.\n",
  768. checkpoint_size, checkpoint_time);
  769. hdr.checkpoint.hdr.addr = (void *) cpstore->base;
  770. hdr.checkpoint.hdr.size = checkpoint_size;
  771. if (cpstore->mem_nentries) {
  772. hdr.checkpoint.mem.entoffset =
  773. (ptr_t) cpstore->last_mem_entry - cpstore->base;
  774. hdr.checkpoint.mem.nentries = cpstore->mem_nentries;
  775. }
  776. if (cpstore->use_gipc) {
  777. snprintf(hdr.checkpoint.gipc.uri, sizeof(hdr.checkpoint.gipc.uri),
  778. "gipc:%lld", gipc_key);
  779. if (cpstore->gipc_nentries) {
  780. hdr.checkpoint.gipc.entoffset =
  781. (ptr_t) cpstore->last_gipc_entry - cpstore->base;
  782. hdr.checkpoint.gipc.nentries = cpstore->gipc_nentries;
  783. }
  784. }
  785. if (cpstore->palhdl_nentries) {
  786. hdr.checkpoint.palhdl.entoffset =
  787. (ptr_t) cpstore->last_palhdl_entry - cpstore->base;
  788. hdr.checkpoint.palhdl.nentries = cpstore->palhdl_nentries;
  789. }
  790. #ifdef PROFILE
  791. hdr.begin_create_time = begin_create_time;
  792. hdr.create_time = create_time;
  793. hdr.write_proc_time = GET_PROFILE_INTERVAL();
  794. #endif
  795. bytes = DkStreamWrite(proc, 0, sizeof(struct newproc_header), &hdr, NULL);
  796. if (!bytes) {
  797. ret = -PAL_ERRNO;
  798. debug("failed writing to process stream (ret = %d)\n", ret);
  799. goto err;
  800. } else if (bytes < sizeof(struct newproc_header)) {
  801. ret = -EACCES;
  802. goto err;
  803. }
  804. ADD_PROFILE_OCCURENCE(migrate_send_on_stream, bytes);
  805. SAVE_PROFILE_INTERVAL(migrate_send_header);
  806. ret = cpstore->use_gipc ? send_checkpoint_by_gipc(gipc_hdl, cpstore) :
  807. send_checkpoint_on_stream(proc, cpstore);
  808. if (ret < 0) {
  809. debug("failed sending checkpoint (ret = %d)\n", ret);
  810. goto err;
  811. }
  812. SAVE_PROFILE_INTERVAL(migrate_send_checkpoint);
  813. if ((ret = send_handles_on_stream(proc, cpstore)) < 0)
  814. goto err;
  815. SAVE_PROFILE_INTERVAL(migrate_send_pal_handles);
  816. system_free((void *) cpstore->base, cpstore->bound);
  817. SAVE_PROFILE_INTERVAL(migrate_free_checkpoint);
  818. struct newproc_response res;
  819. bytes = DkStreamRead(proc, 0, sizeof(struct newproc_response), &res,
  820. NULL, 0);
  821. if (bytes == 0) {
  822. ret = -PAL_ERRNO;
  823. goto err;
  824. }
  825. SAVE_PROFILE_INTERVAL(migrate_wait_response);
  826. if (gipc_hdl)
  827. DkObjectClose(gipc_hdl);
  828. ipc_pid_sublease_send(res.child_vmid, thread->tid,
  829. qstrgetstr(&new_process->self->uri),
  830. NULL);
  831. add_ipc_port_by_id(res.child_vmid, proc,
  832. IPC_PORT_DIRCLD|IPC_PORT_LISTEN|IPC_PORT_KEEPALIVE,
  833. &ipc_child_exit,
  834. NULL);
  835. destroy_process(new_process);
  836. return 0;
  837. err:
  838. if (gipc_hdl)
  839. DkObjectClose(gipc_hdl);
  840. if (proc)
  841. DkObjectClose(proc);
  842. if (new_process)
  843. destroy_process(new_process);
  844. sys_printf("process creation failed\n");
  845. return ret;
  846. }
  847. int do_migration (struct newproc_cp_header * hdr, void ** cpptr)
  848. {
  849. ptr_t base = (ptr_t) hdr->hdr.addr;
  850. int size = hdr->hdr.size;
  851. PAL_PTR mapaddr;
  852. PAL_NUM mapsize;
  853. unsigned long mapoff;
  854. long rebase;
  855. bool use_gipc = !!hdr->gipc.uri[0];
  856. PAL_HANDLE gipc_store;
  857. int ret = 0;
  858. debug("checkpoint detected (%d bytes, expected at %p)\n",
  859. size, base);
  860. if (base && lookup_overlap_vma((void *) base, size, NULL) == -ENOENT) {
  861. mapaddr = (PAL_PTR) ALIGN_DOWN(base);
  862. mapsize = (PAL_PTR) ALIGN_UP(base + size) - mapaddr;
  863. mapoff = base - (ptr_t) mapaddr;
  864. } else {
  865. mapaddr = (PAL_PTR) 0;
  866. mapsize = ALIGN_UP(size);
  867. mapoff = 0;
  868. }
  869. BEGIN_PROFILE_INTERVAL();
  870. if (use_gipc) {
  871. debug("open gipc store: %s\n", hdr->gipc.uri);
  872. PAL_FLG mapprot = PAL_PROT_READ|PAL_PROT_WRITE;
  873. gipc_store = DkStreamOpen(hdr->gipc.uri, 0, 0, 0, 0);
  874. if (!gipc_store ||
  875. !DkPhysicalMemoryMap(gipc_store, 1, &mapaddr, &mapsize, &mapprot))
  876. return -PAL_ERRNO;
  877. SAVE_PROFILE_INTERVAL(child_load_checkpoint_by_gipc);
  878. } else {
  879. void * mapped = NULL;
  880. for (int tries = 3 ; tries ; tries--) {
  881. if ((mapped = DkVirtualMemoryAlloc(mapaddr, mapsize, 0,
  882. PAL_PROT_READ|PAL_PROT_WRITE)))
  883. break;
  884. debug("cannot map address %p-%p\n", mapaddr, mapaddr + mapsize);
  885. ret =-PAL_ERRNO;
  886. mapaddr = NULL;
  887. }
  888. if (!mapped)
  889. return ret;
  890. mapaddr = mapped;
  891. }
  892. bkeep_mmap((void *) mapaddr, mapsize,
  893. PROT_READ|PROT_WRITE,
  894. MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL,
  895. NULL, 0, NULL);
  896. base = (ptr_t) mapaddr + mapoff;
  897. rebase = (long) base - (long) hdr->hdr.addr;
  898. debug("checkpoint loaded at %p\n", base);
  899. if (use_gipc) {
  900. if ((ret = restore_gipc(gipc_store, &hdr->gipc, base, rebase)) < 0)
  901. return ret;
  902. SAVE_PROFILE_INTERVAL(child_load_memory_by_gipc);
  903. DkStreamDelete(gipc_store, 0);
  904. } else {
  905. int total_bytes = 0;
  906. while (total_bytes < size) {
  907. int bytes = DkStreamRead(PAL_CB(parent_process), 0,
  908. size - total_bytes,
  909. (void *) base + total_bytes, NULL, 0);
  910. if (!bytes)
  911. return -PAL_ERRNO;
  912. total_bytes += bytes;
  913. }
  914. SAVE_PROFILE_INTERVAL(child_load_checkpoint_on_pipe);
  915. debug("%d bytes read on stream\n", total_bytes);
  916. }
  917. struct newproc_response res;
  918. res.child_vmid = cur_process.vmid;
  919. res.failure = 0;
  920. int bytes = DkStreamWrite(PAL_CB(parent_process), 0,
  921. sizeof(struct newproc_response),
  922. &res, NULL);
  923. if (!bytes)
  924. return -PAL_ERRNO;
  925. if ((ret = receive_handles_on_stream(&hdr->palhdl, base, rebase)) < 0)
  926. return ret;
  927. SAVE_PROFILE_INTERVAL(child_receive_handles);
  928. migrated_memory_start = (void *) mapaddr;
  929. migrated_memory_end = (void *) mapaddr + mapsize;
  930. *cpptr = (void *) base;
  931. return 0;
  932. }
  933. void restore_context (struct shim_context * context)
  934. {
  935. int nregs = sizeof(struct shim_regs) / sizeof(void *);
  936. void * regs[nregs + 1];
  937. if (context->regs)
  938. memcpy(regs, context->regs, sizeof(struct shim_regs));
  939. else
  940. memset(regs, 0, sizeof(struct shim_regs));
  941. debug("restore context: SP = %p, IP = %p\n", context->sp, context->ret_ip);
  942. regs[nregs] = (void *) context->sp - 8;
  943. *(void **) (context->sp - 8) = context->ret_ip;
  944. memset(context, 0, sizeof(struct shim_context));
  945. asm volatile("movq %0, %%rsp\r\n"
  946. "popq %%r15\r\n"
  947. "popq %%r14\r\n"
  948. "popq %%r13\r\n"
  949. "popq %%r12\r\n"
  950. "popq %%r11\r\n"
  951. "popq %%r10\r\n"
  952. "popq %%r9\r\n"
  953. "popq %%r8\r\n"
  954. "popq %%rcx\r\n"
  955. "popq %%rdx\r\n"
  956. "popq %%rsi\r\n"
  957. "popq %%rdi\r\n"
  958. "popq %%rbx\r\n"
  959. "popq %%rbp\r\n"
  960. "popq %%rsp\r\n"
  961. "movq $0, %%rax\r\n"
  962. "retq\r\n"
  963. :: "g"(&regs) : "memory");
  964. }