shim_vma.c 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278
  1. /* Copyright (C) 2014 Stony Brook University
  2. This file is part of Graphene Library OS.
  3. Graphene Library OS is free software: you can redistribute it and/or
  4. modify it under the terms of the GNU Lesser General Public License
  5. as published by the Free Software Foundation, either version 3 of the
  6. License, or (at your option) any later version.
  7. Graphene Library OS is distributed in the hope that it will be useful,
  8. but WITHOUT ANY WARRANTY; without even the implied warranty of
  9. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  10. GNU Lesser General Public License for more details.
  11. You should have received a copy of the GNU Lesser General Public License
  12. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  13. /*
  14. * shim_vma.c
  15. *
  16. * This file contains code to maintain bookkeeping of VMAs in library OS.
  17. */
  18. #include <shim_internal.h>
  19. #include <shim_thread.h>
  20. #include <shim_handle.h>
  21. #include <shim_vma.h>
  22. #include <shim_checkpoint.h>
  23. #include <shim_fs.h>
  24. #include <pal.h>
  25. #include <list.h>
  26. #include <asm/mman.h>
  27. #include <errno.h>
  28. #include <stdbool.h>
  29. /*
  30. * Internal bookkeeping for VMAs (virtual memory areas). This data
  31. * structure can only be accessed in this source file, with vma_list_lock
  32. * held. No reference counting needed in this data structure.
  33. */
  34. DEFINE_LIST(shim_vma);
  35. /* struct shim_vma tracks the area of [start, end) */
  36. struct shim_vma {
  37. LIST_TYPE(shim_vma) list;
  38. void * start;
  39. void * end;
  40. int prot;
  41. int flags;
  42. off_t offset;
  43. struct shim_handle * file;
  44. char comment[VMA_COMMENT_LEN];
  45. };
  46. #define VMA_MGR_ALLOC DEFAULT_VMA_COUNT
  47. #define RESERVED_VMAS 6
  48. static struct shim_vma * reserved_vmas[RESERVED_VMAS];
  49. static struct shim_vma early_vmas[RESERVED_VMAS];
  50. static void * __bkeep_unmapped (void * top_addr, void * bottom_addr,
  51. size_t length, int prot, int flags,
  52. struct shim_handle * file,
  53. off_t offset, const char * comment);
  54. /*
  55. * Because the default system_malloc() must create VMA(s), we need
  56. * a new system_malloc() to avoid cicular dependency. This __malloc()
  57. * stores the VMA address and size in the current thread to delay the
  58. * bookkeeping until the allocator finishes extension.
  59. */
  60. static inline void * __malloc (size_t size)
  61. {
  62. void * addr;
  63. size = ALLOC_ALIGN_UP(size);
  64. /*
  65. * Chia-Che 3/3/18: We must enforce the policy that all VMAs have to
  66. * be created before issuing the PAL calls.
  67. */
  68. addr = __bkeep_unmapped(PAL_CB(user_address.end),
  69. PAL_CB(user_address.start), size,
  70. PROT_READ|PROT_WRITE,
  71. MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL,
  72. NULL, 0, "vma");
  73. debug("allocate %p-%p for vmas\n", addr, addr + size);
  74. return (void *) DkVirtualMemoryAlloc(addr, size, 0,
  75. PAL_PROT_WRITE|PAL_PROT_READ);
  76. }
  77. #undef system_malloc
  78. #define system_malloc __malloc
  79. #define OBJ_TYPE struct shim_vma
  80. #include <memmgr.h>
  81. /*
  82. * "vma_mgr" has no specific lock. "vma_list_lock" must be held when
  83. * allocating or freeing any VMAs.
  84. */
  85. static MEM_MGR vma_mgr = NULL;
  86. /*
  87. * "vma_list" contains a sorted list of non-overlapping VMAs.
  88. * "vma_list_lock" must be held when accessing either the vma_list or any
  89. * field of a VMA.
  90. */
  91. DEFINE_LISTP(shim_vma);
  92. static LISTP_TYPE(shim_vma) vma_list = LISTP_INIT;
  93. static struct shim_lock vma_list_lock;
  94. /*
  95. * Return true if [s, e) is exactly the area represented by vma.
  96. */
  97. static inline bool test_vma_equal (struct shim_vma * vma,
  98. void * s, void * e)
  99. {
  100. assert(s < e);
  101. return vma->start == s && vma->end == e;
  102. }
  103. /*
  104. * Return true if [s, e) is part of the area represented by vma.
  105. */
  106. static inline bool test_vma_contain (struct shim_vma * vma,
  107. void * s, void * e)
  108. {
  109. assert(s < e);
  110. return vma->start <= s && vma->end >= e;
  111. }
  112. /*
  113. * Return true if [s, e) contains the starting address of vma.
  114. */
  115. static inline bool test_vma_startin (struct shim_vma * vma,
  116. void * s, void * e)
  117. {
  118. assert(s < e);
  119. return vma->start >= s && vma->start < e;
  120. }
  121. /*
  122. * Return true if [s, e) contains the ending address of vma.
  123. */
  124. static inline bool test_vma_endin (struct shim_vma * vma,
  125. void * s, void * e)
  126. {
  127. assert(s < e);
  128. return vma->end > s && vma->end <= e;
  129. }
  130. /*
  131. * Return true if [s, e) overlaps with the area represented by vma.
  132. */
  133. static inline bool test_vma_overlap (struct shim_vma * vma,
  134. void * s, void * e)
  135. {
  136. assert(s < e);
  137. return test_vma_contain(vma, s, s + 1) ||
  138. test_vma_contain(vma, e - 1, e) ||
  139. test_vma_startin(vma, s, e);
  140. }
  141. static inline void __assert_vma_list (void)
  142. {
  143. struct shim_vma * tmp;
  144. struct shim_vma * prev __attribute__((unused)) = NULL;
  145. LISTP_FOR_EACH_ENTRY(tmp, &vma_list, list) {
  146. /* Assert we are really sorted */
  147. assert(tmp->end > tmp->start);
  148. assert(!prev || prev->end <= tmp->start);
  149. prev = tmp;
  150. }
  151. }
  152. // In a debug build only, assert that the VMA list is
  153. // sorted. This should be called with the vma_list_lock held.
  154. static inline void assert_vma_list (void)
  155. {
  156. #ifdef DEBUG
  157. __assert_vma_list();
  158. #endif
  159. }
  160. /*
  161. * __lookup_vma() returns the VMA that contains the address; otherwise,
  162. * returns NULL. "pprev" returns the highest VMA below the address.
  163. * __lookup_vma() fills "pprev" even when the function cannot find a
  164. * matching vma for "addr".
  165. *
  166. * vma_list_lock must be held when calling this function.
  167. */
  168. static inline struct shim_vma *
  169. __lookup_vma (void * addr, struct shim_vma ** pprev)
  170. {
  171. struct shim_vma * vma, * prev = NULL;
  172. struct shim_vma * found = NULL;
  173. LISTP_FOR_EACH_ENTRY(vma, &vma_list, list) {
  174. if (addr < vma->start)
  175. break;
  176. if (test_vma_contain(vma, addr, addr + 1)) {
  177. found = vma;
  178. break;
  179. }
  180. assert(vma->end > vma->start);
  181. assert(!prev || prev->end <= vma->start);
  182. prev = vma;
  183. }
  184. if (pprev) *pprev = prev;
  185. return found;
  186. }
  187. /*
  188. * __insert_vma() places "vma" after "prev", or at the beginning of
  189. * vma_list if "prev" is NULL. vma_list_lock must be held when calling
  190. * this function.
  191. */
  192. static inline void
  193. __insert_vma (struct shim_vma * vma, struct shim_vma * prev)
  194. {
  195. assert(!prev || prev->end <= vma->start);
  196. assert(vma != prev);
  197. /* check the next entry */
  198. struct shim_vma * next = prev ?
  199. LISTP_NEXT_ENTRY(prev, &vma_list, list) :
  200. LISTP_FIRST_ENTRY(&vma_list, struct shim_vma, list);
  201. __UNUSED(next);
  202. assert(!next || vma->end <= next->start);
  203. if (prev)
  204. LISTP_ADD_AFTER(vma, prev, &vma_list, list);
  205. else
  206. LISTP_ADD(vma, &vma_list, list);
  207. }
  208. /*
  209. * __remove_vma() removes "vma" after "prev", or at the beginnning of
  210. * vma_list if "prev" is NULL. vma_list_lock must be held when calling
  211. * this function.
  212. */
  213. static inline void
  214. __remove_vma (struct shim_vma * vma, struct shim_vma * prev)
  215. {
  216. __UNUSED(prev);
  217. assert(vma != prev);
  218. LISTP_DEL(vma, &vma_list, list);
  219. }
  220. /*
  221. * Storing a cursor pointing to the current heap top. With ASLR, the cursor
  222. * is randomized at initialization. The cursor is monotonically decremented
  223. * when allocating user VMAs. Updating this cursor needs holding vma_list_lock.
  224. */
  225. static void * current_heap_top;
  226. static int __bkeep_mmap (struct shim_vma * prev,
  227. void * start, void * end, int prot, int flags,
  228. struct shim_handle * file, off_t offset,
  229. const char * comment);
  230. static int __bkeep_munmap (struct shim_vma ** prev,
  231. void * start, void * end, int flags);
  232. static int __bkeep_mprotect (struct shim_vma * prev,
  233. void * start, void * end, int prot, int flags);
  234. static int
  235. __bkeep_preloaded (void * start, void * end, int prot, int flags,
  236. const char * comment)
  237. {
  238. if (!start || !end || start == end)
  239. return 0;
  240. struct shim_vma * prev = NULL;
  241. __lookup_vma(start, &prev);
  242. return __bkeep_mmap(prev, start, end, prot, flags, NULL, 0, comment);
  243. }
  244. int init_vma (void)
  245. {
  246. int ret;
  247. for (int i = 0 ; i < RESERVED_VMAS ; i++)
  248. reserved_vmas[i] = &early_vmas[i];
  249. /* Bookkeeping for preloaded areas */
  250. if (PAL_CB(user_address_hole.end) - PAL_CB(user_address_hole.start) > 0) {
  251. ret = __bkeep_preloaded(PAL_CB(user_address_hole.start),
  252. PAL_CB(user_address_hole.end),
  253. PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|VMA_UNMAPPED,
  254. "reserved");
  255. if (ret < 0)
  256. return ret;
  257. }
  258. ret = __bkeep_preloaded(PAL_CB(executable_range.start),
  259. PAL_CB(executable_range.end),
  260. PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|VMA_UNMAPPED,
  261. "exec");
  262. if (ret < 0)
  263. return ret;
  264. ret = __bkeep_preloaded(PAL_CB(manifest_preload.start),
  265. PAL_CB(manifest_preload.end),
  266. PROT_READ, MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL,
  267. "manifest");
  268. if (ret < 0)
  269. return ret;
  270. /* Keep track of LibOS code itself so nothing overwrites it */
  271. ret = __bkeep_preloaded(&__load_address,
  272. ALLOC_ALIGN_UP_PTR(&__load_address_end),
  273. PROT_READ, MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL,
  274. "LibOS");
  275. if (ret < 0)
  276. return ret;
  277. /* Initialize the allocator */
  278. if (!(vma_mgr = create_mem_mgr(init_align_up(VMA_MGR_ALLOC)))) {
  279. debug("failed creating the VMA allocator\n");
  280. return -ENOMEM;
  281. }
  282. for (int i = 0 ; i < RESERVED_VMAS ; i++) {
  283. if (!reserved_vmas[i]) {
  284. struct shim_vma * new = get_mem_obj_from_mgr(vma_mgr);
  285. assert(new);
  286. struct shim_vma * e = &early_vmas[i];
  287. struct shim_vma * prev = LISTP_PREV_ENTRY(e, &vma_list, list);
  288. debug("Converting early VMA [%p] %p-%p\n", e, e->start, e->end);
  289. memcpy(new, e, sizeof(*e));
  290. INIT_LIST_HEAD(new, list);
  291. __remove_vma(e, prev);
  292. __insert_vma(new, prev);
  293. }
  294. /* replace all reserved VMAs */
  295. reserved_vmas[i] = get_mem_obj_from_mgr(vma_mgr);
  296. assert(reserved_vmas[i]);
  297. }
  298. create_lock(&vma_list_lock);
  299. current_heap_top = PAL_CB(user_address.end);
  300. #if ENABLE_ASLR == 1
  301. /*
  302. * Randomize the heap top in top 5/6 of the user address space.
  303. * This is a simplified version of the mmap_base() logic in the Linux
  304. * kernel: https://elixir.bootlin.com/linux/v4.8/ident/mmap_base
  305. */
  306. size_t addr_rand_size =
  307. (PAL_CB(user_address.end) - PAL_CB(user_address.start)) * 5 / 6;
  308. size_t rand;
  309. ret = DkRandomBitsRead(&rand, sizeof(rand));
  310. if (ret < 0)
  311. return -convert_pal_errno(-ret);
  312. current_heap_top -= ALLOC_ALIGN_DOWN(rand % addr_rand_size);
  313. #endif
  314. debug("heap top adjusted to %p\n", current_heap_top);
  315. return 0;
  316. }
  317. static inline struct shim_vma * __get_new_vma (void)
  318. {
  319. struct shim_vma * tmp = NULL;
  320. if (vma_mgr)
  321. tmp = get_mem_obj_from_mgr(vma_mgr);
  322. if (tmp == NULL) {
  323. for (int i = 0 ; i < RESERVED_VMAS ; i++)
  324. if (reserved_vmas[i]) {
  325. tmp = reserved_vmas[i];
  326. reserved_vmas[i] = NULL;
  327. break;
  328. }
  329. }
  330. if (tmp == NULL) {
  331. /* Should never reach here; if this happens, increase RESERVED_VMAS */
  332. debug("failed to allocate new vma\n");
  333. BUG();
  334. return NULL;
  335. }
  336. memset(tmp, 0, sizeof(*tmp));
  337. INIT_LIST_HEAD(tmp, list);
  338. return tmp;
  339. }
  340. static inline void __restore_reserved_vmas (void)
  341. {
  342. bool nothing_reserved;
  343. do {
  344. nothing_reserved = true;
  345. for (int i = 0 ; i < RESERVED_VMAS ; i++)
  346. if (!reserved_vmas[i]) {
  347. struct shim_vma * new =
  348. get_mem_obj_from_mgr_enlarge(vma_mgr,
  349. size_align_up(VMA_MGR_ALLOC));
  350. /* this allocation must succeed */
  351. assert(new);
  352. reserved_vmas[i] = new;
  353. nothing_reserved = false;
  354. }
  355. } while (!nothing_reserved);
  356. }
  357. static inline void __drop_vma (struct shim_vma * vma)
  358. {
  359. if (vma->file)
  360. put_handle(vma->file);
  361. for (int i = 0 ; i < RESERVED_VMAS ; i++)
  362. if (!reserved_vmas[i]) {
  363. reserved_vmas[i] = vma;
  364. return;
  365. }
  366. free_mem_obj_to_mgr(vma_mgr, vma);
  367. }
  368. static inline void
  369. __assert_vma_flags (const struct shim_vma * vma, int flags)
  370. {
  371. if (!(vma->flags & VMA_UNMAPPED)
  372. && VMA_TYPE(vma->flags) != VMA_TYPE(flags)) {
  373. debug("Check vma flag failure: vma flags %x, checked flags %x\n",
  374. vma->flags, flags);
  375. BUG();
  376. }
  377. }
  378. static inline void
  379. __set_vma_comment (struct shim_vma * vma, const char * comment)
  380. {
  381. if (!comment) {
  382. vma->comment[0] = 0;
  383. return;
  384. }
  385. size_t len = strlen(comment);
  386. if (len > VMA_COMMENT_LEN - 1)
  387. len = VMA_COMMENT_LEN - 1;
  388. memcpy(vma->comment, comment, len);
  389. vma->comment[len] = 0;
  390. }
  391. /*
  392. * Add bookkeeping for mmap(). "prev" must point to the the immediately
  393. * precedent vma of the address to map, or be NULL if no vma is lower than
  394. * the address. If the bookkeeping area overlaps with some existing vmas,
  395. * we must check whether the caller (from user, internal code, or checkpointing
  396. * procedure) is allowed to overwrite the existing vmas.
  397. *
  398. * Bookkeeping convention (must follow):
  399. * Create the bookkeeping BEFORE any allocation PAL calls
  400. * (DkVirtualMemoryAlloc() or DkStreamMap()).
  401. */
  402. static int __bkeep_mmap (struct shim_vma * prev,
  403. void * start, void * end, int prot, int flags,
  404. struct shim_handle * file, off_t offset,
  405. const char * comment)
  406. {
  407. int ret = 0;
  408. struct shim_vma * new = __get_new_vma();
  409. /* First, remove any overlapping VMAs */
  410. ret = __bkeep_munmap(&prev, start, end, flags);
  411. if (ret < 0) {
  412. __drop_vma(new);
  413. return ret;
  414. }
  415. /* Inserting the new VMA */
  416. new->start = start;
  417. new->end = end;
  418. new->prot = prot;
  419. new->flags = flags|((file && (prot & PROT_WRITE)) ? VMA_TAINTED : 0);
  420. new->file = file;
  421. if (new->file)
  422. get_handle(new->file);
  423. new->offset = offset;
  424. __set_vma_comment(new, comment);
  425. __insert_vma(new, prev);
  426. return 0;
  427. }
  428. int bkeep_mmap (void * addr, size_t length, int prot, int flags,
  429. struct shim_handle * file, off_t offset, const char * comment)
  430. {
  431. if (!addr || !length)
  432. return -EINVAL;
  433. if (comment && !comment[0])
  434. comment = NULL;
  435. debug("bkeep_mmap: %p-%p\n", addr, addr + length);
  436. lock(&vma_list_lock);
  437. struct shim_vma * prev = NULL;
  438. __lookup_vma(addr, &prev);
  439. int ret = __bkeep_mmap(prev, addr, addr + length, prot, flags, file, offset,
  440. comment);
  441. assert_vma_list();
  442. __restore_reserved_vmas();
  443. unlock(&vma_list_lock);
  444. return ret;
  445. }
  446. /*
  447. * __shrink_vma() removes the area in a VMA that overlaps with [start, end).
  448. * The function deals with three cases:
  449. * (1) [start, end) overlaps with the beginning of the VMA.
  450. * (2) [start, end) overlaps with the ending of the VMA.
  451. * (3) [start, end) overlaps with the middle of the VMA. In this case, the VMA
  452. * is splitted into two. The new VMA is stored in 'tailptr'.
  453. * In either of these cases, "vma" is the only one changed among vma_list.
  454. */
  455. static inline void __shrink_vma (struct shim_vma * vma, void * start, void * end,
  456. struct shim_vma ** tailptr)
  457. {
  458. if (test_vma_startin(vma, start, end)) {
  459. /*
  460. * Dealing with the head: if the starting address of "vma" is in
  461. * [start, end), move the starting address.
  462. */
  463. if (end < vma->end) {
  464. if (vma->file) /* must adjust offset */
  465. vma->offset += end - vma->start;
  466. vma->start = end;
  467. } else {
  468. if (vma->file) /* must adjust offset */
  469. vma->offset += vma->end - vma->start;
  470. vma->start = vma->end;
  471. }
  472. } else if (test_vma_endin(vma, start, end)) {
  473. /*
  474. * Dealing with the tail: if the ending address of "vma" is in
  475. * [start, end), move the ending address.
  476. */
  477. if (start > vma->start) {
  478. vma->end = start;
  479. } else {
  480. vma->end = vma->start;
  481. }
  482. /* offset is not affected */
  483. } else if (test_vma_contain(vma, start, end)) {
  484. /*
  485. * If [start, end) is inside the range of "vma", divide up
  486. * the VMA. A new VMA is created to represent the remaining tail.
  487. */
  488. void * old_end = vma->end;
  489. vma->end = start;
  490. /* Remaining space after [start, end), creating a new VMA */
  491. if (old_end > end) {
  492. struct shim_vma * tail = __get_new_vma();
  493. tail->start = end;
  494. tail->end = old_end;
  495. tail->prot = vma->prot;
  496. tail->flags = vma->flags;
  497. tail->file = vma->file;
  498. if (tail->file) {
  499. get_handle(tail->file);
  500. tail->offset = vma->offset + (tail->start - vma->start);
  501. } else {
  502. tail->offset = 0;
  503. }
  504. memcpy(tail->comment, vma->comment, VMA_COMMENT_LEN);
  505. *tailptr = tail;
  506. }
  507. } else {
  508. /* Never reach here */
  509. BUG();
  510. }
  511. assert(!test_vma_overlap(vma, start, end));
  512. assert(vma->start < vma->end);
  513. }
  514. /*
  515. * Update bookkeeping for munmap(). "*pprev" must point to the immediately
  516. * precedent vma of the address to unmap, or be NULL if no vma is lower than
  517. * the address. If the bookkeeping area overlaps with some existing vmas,
  518. * we must check whether the caller (from user, internal code, or checkpointing
  519. * procedure) is allowed to overwrite the existing vmas. "pprev" can be
  520. * updated if a new vma lower than the unmapping address is added.
  521. *
  522. * Bookkeeping convention (must follow):
  523. * Make deallocation PAL calls (DkVirtualMemoryFree() or DkStreamUnmap())
  524. * BEFORE updating the bookkeeping.
  525. */
  526. static int __bkeep_munmap (struct shim_vma ** pprev,
  527. void * start, void * end, int flags)
  528. {
  529. struct shim_vma * prev = *pprev;
  530. struct shim_vma * cur, * next;
  531. if (!prev) {
  532. cur = LISTP_FIRST_ENTRY(&vma_list, struct shim_vma, list);
  533. if (!cur)
  534. return 0;
  535. } else {
  536. cur = LISTP_NEXT_ENTRY(prev, &vma_list, list);
  537. }
  538. next = cur ? LISTP_NEXT_ENTRY(cur, &vma_list, list) : NULL;
  539. while (cur) {
  540. struct shim_vma * tail = NULL;
  541. /* Stop unmapping if "cur" no longer overlaps with [start, end) */
  542. if (!test_vma_overlap(cur, start, end))
  543. break;
  544. if (VMA_TYPE(cur->flags) != VMA_TYPE(flags))
  545. return -EACCES;
  546. /* If [start, end) contains the VMA, just drop the VMA. */
  547. if (start <= cur->start && cur->end <= end) {
  548. __remove_vma(cur, prev);
  549. __drop_vma(cur);
  550. } else {
  551. __shrink_vma(cur, start, end, &tail);
  552. if (cur->end <= start) {
  553. prev = cur;
  554. if (tail) {
  555. __insert_vma(tail, cur); /* insert "tail" after "cur" */
  556. cur = tail; /* "tail" is the new "cur" */
  557. break;
  558. }
  559. } else if (cur->start >= end) {
  560. /* __shrink_vma() only creates a new VMA when the beginning of the
  561. * original VMA is preserved. */
  562. assert(!tail);
  563. break;
  564. } else {
  565. /* __shrink_vma() should never allow this case. */
  566. BUG();
  567. }
  568. }
  569. cur = next;
  570. next = cur ? LISTP_NEXT_ENTRY(cur, &vma_list, list) : NULL;
  571. }
  572. if (prev)
  573. assert(cur == LISTP_NEXT_ENTRY(prev, &vma_list, list));
  574. else
  575. assert(cur == LISTP_FIRST_ENTRY(&vma_list, struct shim_vma, list));
  576. assert(!prev || prev->end <= start);
  577. assert(!cur || end <= cur->start);
  578. *pprev = prev;
  579. return 0;
  580. }
  581. int bkeep_munmap (void * addr, size_t length, int flags)
  582. {
  583. if (!length)
  584. return -EINVAL;
  585. debug("bkeep_munmap: %p-%p\n", addr, addr + length);
  586. lock(&vma_list_lock);
  587. struct shim_vma * prev = NULL;
  588. __lookup_vma(addr, &prev);
  589. int ret = __bkeep_munmap(&prev, addr, addr + length, flags);
  590. assert_vma_list();
  591. __restore_reserved_vmas();
  592. /* DEP 5/20/19: If this is a debugging region we are removing, take it out
  593. * of the checkpoint. Otherwise, it will be restored erroneously after a fork. */
  594. remove_r_debug(addr);
  595. unlock(&vma_list_lock);
  596. return ret;
  597. }
  598. /*
  599. * Update bookkeeping for mprotect(). "prev" must point to the immediately
  600. * precedent vma of the address to protect, or be NULL if no vma is lower than
  601. * the address. If the bookkeeping area overlaps with some existing vmas,
  602. * we must check whether the caller (from user, internal code, or checkpointing
  603. * procedure) is allowed to overwrite the existing vmas.
  604. *
  605. * Bookkeeping convention (must follow):
  606. * Update the bookkeeping BEFORE calling DkVirtualMemoryProtect().
  607. */
  608. static int __bkeep_mprotect (struct shim_vma * prev,
  609. void * start, void * end, int prot, int flags)
  610. {
  611. struct shim_vma * cur, * next;
  612. if (!prev) {
  613. cur = LISTP_FIRST_ENTRY(&vma_list, struct shim_vma, list);
  614. if (!cur)
  615. return 0;
  616. } else {
  617. cur = LISTP_NEXT_ENTRY(prev, &vma_list, list);
  618. }
  619. next = cur ? LISTP_NEXT_ENTRY(cur, &vma_list, list) : NULL;
  620. while (cur) {
  621. struct shim_vma * new, * tail = NULL;
  622. /* Stop protecting if "cur" no longer overlaps with [start, end) */
  623. if (!test_vma_overlap(cur, start, end))
  624. break;
  625. if (VMA_TYPE(cur->flags) != VMA_TYPE(flags))
  626. /* For now, just shout loudly. */
  627. return -EACCES;
  628. /* If protection doesn't change anything, move on to the next */
  629. if (cur->prot != prot) {
  630. /* If [start, end) contains the VMA, just update its protection. */
  631. if (start <= cur->start && cur->end <= end) {
  632. cur->prot = prot;
  633. if (cur->file && (prot & PROT_WRITE)) {
  634. cur->flags |= VMA_TAINTED;
  635. }
  636. } else {
  637. /* Create a new VMA for the protected area */
  638. new = __get_new_vma();
  639. new->start = cur->start > start ? cur->start : start;
  640. new->end = cur->end < end ? cur->end : end;
  641. new->prot = prot;
  642. new->flags = cur->flags | ((cur->file && (prot & PROT_WRITE)) ? VMA_TAINTED : 0);
  643. new->file = cur->file;
  644. if (new->file) {
  645. get_handle(new->file);
  646. new->offset = cur->offset + (new->start - cur->start);
  647. } else {
  648. new->offset = 0;
  649. }
  650. memcpy(new->comment, cur->comment, VMA_COMMENT_LEN);
  651. /* Like unmapping, shrink (and potentially split) the VMA first. */
  652. __shrink_vma(cur, start, end, &tail);
  653. if (cur->end <= start) {
  654. prev = cur;
  655. if (tail) {
  656. __insert_vma(tail, cur); /* insert "tail" after "cur" */
  657. cur = tail; /* "tail" is the new "cur" */
  658. /* "next" is the same */
  659. }
  660. } else if (cur->start >= end) {
  661. /* __shrink_vma() only creates a new VMA when the beginning of the
  662. * original VMA is preserved. */
  663. assert(!tail);
  664. } else {
  665. /* __shrink_vma() should never allow this case. */
  666. BUG();
  667. }
  668. /* Now insert the new protected vma between prev and cur */
  669. __insert_vma(new, prev);
  670. assert(!prev || prev->end <= new->end);
  671. assert(new->start < new->end);
  672. }
  673. }
  674. prev = cur;
  675. cur = next;
  676. next = cur ? LISTP_NEXT_ENTRY(cur, &vma_list, list) : NULL;
  677. }
  678. return 0;
  679. }
  680. int bkeep_mprotect (void * addr, size_t length, int prot, int flags)
  681. {
  682. if (!addr || !length)
  683. return -EINVAL;
  684. debug("bkeep_mprotect: %p-%p\n", addr, addr + length);
  685. lock(&vma_list_lock);
  686. struct shim_vma * prev = NULL;
  687. __lookup_vma(addr, &prev);
  688. int ret = __bkeep_mprotect(prev, addr, addr + length, prot, flags);
  689. assert_vma_list();
  690. __restore_reserved_vmas();
  691. unlock(&vma_list_lock);
  692. return ret;
  693. }
  694. /*
  695. * Search for an unmapped area within [bottom, top) that is big enough
  696. * to allocate "length" bytes. The search approach is top-down.
  697. * If this function returns a non-NULL address, the corresponding VMA is
  698. * added to the VMA list.
  699. */
  700. static void * __bkeep_unmapped (void * top_addr, void * bottom_addr,
  701. size_t length, int prot, int flags,
  702. struct shim_handle * file,
  703. off_t offset, const char * comment)
  704. {
  705. assert(top_addr > bottom_addr);
  706. if (!length || length > (uintptr_t) top_addr - (uintptr_t) bottom_addr)
  707. return NULL;
  708. struct shim_vma * prev = NULL;
  709. struct shim_vma * cur = __lookup_vma(top_addr, &prev);
  710. while (true) {
  711. /* Set the range for searching */
  712. void * end = cur ? cur->start : top_addr;
  713. void * start =
  714. (prev && prev->end > bottom_addr) ? prev->end : bottom_addr;
  715. assert(start <= end);
  716. /* Check if there is enough space between prev and cur */
  717. if (length <= (uintptr_t) end - (uintptr_t) start) {
  718. /* create a new VMA at the top of the range */
  719. __bkeep_mmap(prev, end - length, end, prot, flags,
  720. file, offset, comment);
  721. assert_vma_list();
  722. debug("bkeep_unmapped: %p-%p%s%s\n", end - length, end,
  723. comment ? " => " : "", comment ? : "");
  724. return end - length;
  725. }
  726. if (!prev || prev->start <= bottom_addr)
  727. break;
  728. cur = prev;
  729. prev = LISTP_PREV_ENTRY(cur, &vma_list, list);
  730. }
  731. return NULL;
  732. }
  733. void * bkeep_unmapped (void * top_addr, void * bottom_addr, size_t length,
  734. int prot, int flags, off_t offset, const char * comment)
  735. {
  736. lock(&vma_list_lock);
  737. void * addr = __bkeep_unmapped(top_addr, bottom_addr, length, prot, flags,
  738. NULL, offset, comment);
  739. assert_vma_list();
  740. __restore_reserved_vmas();
  741. unlock(&vma_list_lock);
  742. return addr;
  743. }
  744. void * bkeep_unmapped_heap (size_t length, int prot, int flags,
  745. struct shim_handle * file,
  746. off_t offset, const char * comment)
  747. {
  748. lock(&vma_list_lock);
  749. void * bottom_addr = PAL_CB(user_address.start);
  750. void * top_addr = current_heap_top;
  751. void * heap_max = PAL_CB(user_address.end);
  752. void * addr = NULL;
  753. #ifdef MAP_32BIT
  754. /*
  755. * If MAP_32BIT is given in the flags, force the searching range to
  756. * be lower than 1ULL << 32.
  757. */
  758. #define ADDR_32BIT ((void*)(1ULL << 32))
  759. if (flags & MAP_32BIT) {
  760. /* Try the lower 4GB memory space */
  761. if (heap_max > ADDR_32BIT)
  762. heap_max = ADDR_32BIT;
  763. if (top_addr > heap_max)
  764. top_addr = heap_max;
  765. }
  766. #endif
  767. if (top_addr > bottom_addr) {
  768. /* Try first time */
  769. addr = __bkeep_unmapped(top_addr, bottom_addr,
  770. length, prot, flags,
  771. file, offset, comment);
  772. assert_vma_list();
  773. }
  774. if (addr) {
  775. /*
  776. * we only update the current heap top when we get the
  777. * address from [bottom_addr, current_heap_top).
  778. */
  779. if (top_addr == current_heap_top) {
  780. debug("heap top adjusted to %p\n", addr);
  781. current_heap_top = addr;
  782. }
  783. } else if (top_addr < heap_max) {
  784. /* Try to allocate above the current heap top */
  785. addr = __bkeep_unmapped(heap_max, bottom_addr,
  786. length, prot, flags,
  787. file, offset, comment);
  788. assert_vma_list();
  789. }
  790. __restore_reserved_vmas();
  791. unlock(&vma_list_lock);
  792. #ifdef MAP_32BIT
  793. assert(!(flags & MAP_32BIT) || !addr || addr + length <= ADDR_32BIT);
  794. #endif
  795. return addr;
  796. }
  797. static inline void
  798. __dump_vma (struct shim_vma_val * val, const struct shim_vma * vma)
  799. {
  800. val->addr = vma->start;
  801. val->length = vma->end - vma->start;
  802. val->prot = vma->prot;
  803. val->flags = vma->flags;
  804. val->file = vma->file;
  805. if (val->file)
  806. get_handle(val->file);
  807. val->offset = vma->offset;
  808. memcpy(val->comment, vma->comment, VMA_COMMENT_LEN);
  809. }
  810. int lookup_vma (void * addr, struct shim_vma_val * res)
  811. {
  812. lock(&vma_list_lock);
  813. struct shim_vma * vma = __lookup_vma(addr, NULL);
  814. if (!vma) {
  815. unlock(&vma_list_lock);
  816. return -ENOENT;
  817. }
  818. if (res)
  819. __dump_vma(res, vma);
  820. unlock(&vma_list_lock);
  821. return 0;
  822. }
  823. int lookup_overlap_vma (void * addr, size_t length, struct shim_vma_val * res)
  824. {
  825. struct shim_vma * tmp, * vma = NULL;
  826. lock(&vma_list_lock);
  827. LISTP_FOR_EACH_ENTRY(tmp, &vma_list, list)
  828. if (test_vma_overlap (tmp, addr, addr + length)) {
  829. vma = tmp;
  830. break;
  831. }
  832. if (!vma) {
  833. unlock(&vma_list_lock);
  834. return -ENOENT;
  835. }
  836. if (res)
  837. __dump_vma(res, vma);
  838. unlock(&vma_list_lock);
  839. return 0;
  840. }
  841. bool is_in_adjacent_vmas (void * addr, size_t length)
  842. {
  843. struct shim_vma* vma;
  844. struct shim_vma* prev = NULL;
  845. lock(&vma_list_lock);
  846. /* we rely on the fact that VMAs are sorted (for adjacent VMAs) */
  847. assert_vma_list();
  848. LISTP_FOR_EACH_ENTRY(vma, &vma_list, list) {
  849. if (addr >= vma->start && addr < vma->end) {
  850. assert(prev == NULL);
  851. prev = vma;
  852. }
  853. if (prev) {
  854. if (prev != vma && prev->end != vma->start) {
  855. /* prev and current VMAs are not adjacent */
  856. break;
  857. }
  858. if ((addr + length) > vma->start && (addr + length) <= vma->end) {
  859. unlock(&vma_list_lock);
  860. return true;
  861. }
  862. prev = vma;
  863. }
  864. }
  865. unlock(&vma_list_lock);
  866. return false;
  867. }
  868. int dump_all_vmas (struct shim_vma_val * vmas, size_t max_count)
  869. {
  870. struct shim_vma_val * val = vmas;
  871. struct shim_vma * vma;
  872. size_t cnt = 0;
  873. lock(&vma_list_lock);
  874. LISTP_FOR_EACH_ENTRY(vma, &vma_list, list) {
  875. if (VMA_TYPE(vma->flags))
  876. continue;
  877. if (vma->flags & VMA_UNMAPPED)
  878. continue;
  879. if (cnt == max_count) {
  880. cnt = -EOVERFLOW;
  881. for (size_t i = 0 ; i < max_count ; i++)
  882. if (vmas[i].file)
  883. put_handle(vmas[i].file);
  884. break;
  885. }
  886. __dump_vma(val, vma);
  887. cnt++;
  888. val++;
  889. }
  890. unlock(&vma_list_lock);
  891. return cnt;
  892. }
  893. BEGIN_CP_FUNC(vma)
  894. {
  895. __UNUSED(size);
  896. assert(size == sizeof(struct shim_vma_val));
  897. struct shim_vma_val * vma = (struct shim_vma_val *) obj;
  898. struct shim_vma_val * new_vma = NULL;
  899. PAL_FLG pal_prot = PAL_PROT(vma->prot, 0);
  900. ptr_t off = GET_FROM_CP_MAP(obj);
  901. if (!off) {
  902. off = ADD_CP_OFFSET(sizeof(*vma));
  903. ADD_TO_CP_MAP(obj, off);
  904. new_vma = (struct shim_vma_val *) (base + off);
  905. memcpy(new_vma, vma, sizeof(*vma));
  906. if (vma->file)
  907. DO_CP(handle, vma->file, &new_vma->file);
  908. void * need_mapped = vma->addr;
  909. if (NEED_MIGRATE_MEMORY(vma)) {
  910. void* send_addr = vma->addr;
  911. size_t send_size = vma->length;
  912. if (vma->file) {
  913. /*
  914. * Chia-Che 8/13/2017:
  915. * A fix for cloning a private VMA which maps a file to a process.
  916. *
  917. * (1) Application can access any page backed by the file, wholly
  918. * or partially.
  919. *
  920. * (2) Access beyond the last file-backed page will cause SIGBUS.
  921. * For reducing fork latency, the following code truncates the
  922. * memory size for migrating a process. The memory size is
  923. * truncated to the file size, round up to pages.
  924. *
  925. * (3) Data in the last file-backed page is valid before or after
  926. * forking. Has to be included in process migration.
  927. */
  928. off_t file_len = get_file_size(vma->file);
  929. if (file_len >= 0 &&
  930. (off_t)(vma->offset + vma->length) > file_len) {
  931. send_size = file_len > vma->offset ?
  932. file_len - vma->offset : 0;
  933. send_size = ALLOC_ALIGN_UP(send_size);
  934. }
  935. }
  936. if (send_size > 0) {
  937. if (!(pal_prot & PAL_PROT_READ)) {
  938. /* Make the area readable */
  939. DkVirtualMemoryProtect(send_addr, send_size,
  940. pal_prot|PAL_PROT_READ);
  941. }
  942. struct shim_mem_entry * mem;
  943. DO_CP_SIZE(memory, send_addr, send_size, &mem);
  944. mem->prot = pal_prot;
  945. need_mapped = vma->addr + vma->length;
  946. }
  947. }
  948. ADD_CP_FUNC_ENTRY(off);
  949. ADD_CP_ENTRY(ADDR, need_mapped);
  950. } else {
  951. new_vma = (struct shim_vma_val *) (base + off);
  952. }
  953. if (objp)
  954. *objp = (void *) new_vma;
  955. }
  956. END_CP_FUNC(vma)
  957. DEFINE_PROFILE_CATEGORY(inside_rs_vma, resume_func);
  958. DEFINE_PROFILE_INTERVAL(vma_add_bookkeep, inside_rs_vma);
  959. DEFINE_PROFILE_INTERVAL(vma_map_file, inside_rs_vma);
  960. DEFINE_PROFILE_INTERVAL(vma_map_anonymous, inside_rs_vma);
  961. BEGIN_RS_FUNC(vma)
  962. {
  963. struct shim_vma_val * vma = (void *) (base + GET_CP_FUNC_ENTRY());
  964. void * need_mapped = (void *) GET_CP_ENTRY(ADDR);
  965. BEGIN_PROFILE_INTERVAL();
  966. CP_REBASE(vma->file);
  967. int ret = bkeep_mmap(vma->addr, vma->length, vma->prot, vma->flags,
  968. vma->file, vma->offset, vma->comment);
  969. if (ret < 0)
  970. return ret;
  971. SAVE_PROFILE_INTERVAL(vma_add_bookkeep);
  972. DEBUG_RS("vma: %p-%p flags %x prot 0x%08x\n",
  973. vma->addr, vma->addr + vma->length, vma->flags, vma->prot);
  974. if (!(vma->flags & VMA_UNMAPPED)) {
  975. if (vma->file) {
  976. struct shim_mount * fs = vma->file->fs;
  977. get_handle(vma->file);
  978. if (need_mapped < vma->addr + vma->length) {
  979. /* first try, use hstat to force it resumes pal handle */
  980. assert(vma->file->fs && vma->file->fs->fs_ops &&
  981. vma->file->fs->fs_ops->mmap);
  982. void * addr = need_mapped;
  983. int ret = fs->fs_ops->mmap(vma->file, &addr,
  984. vma->addr + vma->length -
  985. need_mapped,
  986. vma->prot,
  987. vma->flags,
  988. vma->offset +
  989. (need_mapped - vma->addr));
  990. if (ret < 0)
  991. return ret;
  992. if (!addr)
  993. return -ENOMEM;
  994. if (addr != need_mapped)
  995. return -EACCES;
  996. need_mapped += vma->length;
  997. SAVE_PROFILE_INTERVAL(vma_map_file);
  998. }
  999. }
  1000. if (need_mapped < vma->addr + vma->length) {
  1001. int pal_alloc_type = 0;
  1002. int pal_prot = vma->prot;
  1003. if (DkVirtualMemoryAlloc(need_mapped,
  1004. vma->addr + vma->length - need_mapped,
  1005. pal_alloc_type, pal_prot)) {
  1006. need_mapped += vma->length;
  1007. SAVE_PROFILE_INTERVAL(vma_map_anonymous);
  1008. }
  1009. }
  1010. if (need_mapped < vma->addr + vma->length)
  1011. SYS_PRINTF("vma %p-%p cannot be allocated!\n", need_mapped,
  1012. vma->addr + vma->length);
  1013. }
  1014. if (vma->file)
  1015. get_handle(vma->file);
  1016. if (vma->file)
  1017. DEBUG_RS("%p-%p,size=%ld,prot=%08x,flags=%08x,off=%ld,path=%s,uri=%s",
  1018. vma->addr, vma->addr + vma->length, vma->length,
  1019. vma->prot, vma->flags, vma->offset,
  1020. qstrgetstr(&vma->file->path), qstrgetstr(&vma->file->uri));
  1021. else
  1022. DEBUG_RS("%p-%p,size=%ld,prot=%08x,flags=%08x,off=%ld",
  1023. vma->addr, vma->addr + vma->length, vma->length,
  1024. vma->prot, vma->flags, vma->offset);
  1025. }
  1026. END_RS_FUNC(vma)
  1027. BEGIN_CP_FUNC(all_vmas)
  1028. {
  1029. size_t count = DEFAULT_VMA_COUNT;
  1030. struct shim_vma_val * vmas = malloc(sizeof(*vmas) * count);
  1031. int ret;
  1032. __UNUSED(obj);
  1033. __UNUSED(size);
  1034. __UNUSED(objp);
  1035. if (!vmas)
  1036. return -ENOMEM;
  1037. while (true) {
  1038. ret = dump_all_vmas(vmas, count);
  1039. if (ret != -EOVERFLOW)
  1040. break;
  1041. struct shim_vma_val * new_vmas
  1042. = malloc(sizeof(*new_vmas) * count * 2);
  1043. if (!new_vmas) {
  1044. free(vmas);
  1045. return -ENOMEM;
  1046. }
  1047. free(vmas);
  1048. vmas = new_vmas;
  1049. count *= 2;
  1050. }
  1051. if (ret < 0)
  1052. return ret;
  1053. count = ret;
  1054. for (struct shim_vma_val * vma = &vmas[count - 1] ; vma >= vmas ; vma--)
  1055. DO_CP(vma, vma, NULL);
  1056. free_vma_val_array(vmas, count);
  1057. }
  1058. END_CP_FUNC_NO_RS(all_vmas)
  1059. void debug_print_vma (struct shim_vma *vma)
  1060. {
  1061. const char * type = "", * name = "";
  1062. if (vma->file) {
  1063. if (!qstrempty(&vma->file->path)) {
  1064. type = " path=";
  1065. name = qstrgetstr(&vma->file->path);
  1066. } else if (!qstrempty(&vma->file->uri)) {
  1067. type = " uri=";
  1068. name = qstrgetstr(&vma->file->uri);
  1069. }
  1070. }
  1071. SYS_PRINTF("[%p-%p] prot=%08x flags=%08x%s%s offset=%ld%s%s%s%s\n",
  1072. vma->start, vma->end,
  1073. vma->prot,
  1074. vma->flags & ~(VMA_INTERNAL|VMA_UNMAPPED|VMA_TAINTED|VMA_CP),
  1075. type, name,
  1076. vma->offset,
  1077. vma->flags & VMA_INTERNAL ? " (internal)" : "",
  1078. vma->flags & VMA_UNMAPPED ? " (unmapped)" : "",
  1079. vma->comment[0] ? " comment=" : "",
  1080. vma->comment[0] ? vma->comment : "");
  1081. }
  1082. void debug_print_vma_list (void)
  1083. {
  1084. SYS_PRINTF("vma bookkeeping:\n");
  1085. struct shim_vma * vma;
  1086. LISTP_FOR_EACH_ENTRY(vma, &vma_list, list) {
  1087. debug_print_vma(vma);
  1088. }
  1089. }