shim_vma.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444
  1. /* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
  2. /* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
  3. /* Copyright (C) 2014 Stony Brook University
  4. This file is part of Graphene Library OS.
  5. Graphene Library OS is free software: you can redistribute it and/or
  6. modify it under the terms of the GNU Lesser General Public License
  7. as published by the Free Software Foundation, either version 3 of the
  8. License, or (at your option) any later version.
  9. Graphene Library OS is distributed in the hope that it will be useful,
  10. but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. GNU Lesser General Public License for more details.
  13. You should have received a copy of the GNU Lesser General Public License
  14. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  15. /*
  16. * shim_vma.c
  17. *
  18. * This file contains codes to maintain bookkeeping of VMAs in library OS.
  19. */
  20. #include <shim_internal.h>
  21. #include <shim_thread.h>
  22. #include <shim_handle.h>
  23. #include <shim_vma.h>
  24. #include <shim_checkpoint.h>
  25. #include <shim_fs.h>
  26. #include <pal.h>
  27. #include <list.h>
  28. #include <asm/mman.h>
  29. #include <errno.h>
  30. unsigned long mem_max_npages __attribute_migratable = DEFAULT_MEM_MAX_NPAGES;
  31. static void * heap_top, * heap_bottom;
  32. #define VMA_MGR_ALLOC 64
  33. #define PAGE_SIZE allocsize
  34. static LOCKTYPE vma_mgr_lock;
  35. #define system_lock() lock(vma_mgr_lock)
  36. #define system_unlock() unlock(vma_mgr_lock)
  37. static inline void * __vma_malloc (size_t size)
  38. {
  39. struct shim_thread * thread = get_cur_thread();
  40. if (!thread)
  41. return system_malloc(size);
  42. size = ALIGN_UP(size);
  43. void * addr = (void *) DkVirtualMemoryAlloc(NULL, size, 0,
  44. PAL_PROT_WRITE|PAL_PROT_READ);
  45. debug("allocate %p-%p for vmas\n", addr, addr + size);
  46. thread->delayed_bkeep_mmap.addr = addr;
  47. thread->delayed_bkeep_mmap.length = size;
  48. return addr;
  49. }
  50. #undef system_malloc
  51. #define system_malloc(size) __vma_malloc(size)
  52. #define OBJ_TYPE struct shim_vma
  53. #include <memmgr.h>
  54. static MEM_MGR vma_mgr = NULL;
  55. DEFINE_LISTP(shim_vma);
  56. static LISTP_TYPE(shim_vma) vma_list = LISTP_INIT;
  57. static LOCKTYPE vma_list_lock;
  58. static inline bool test_vma_equal (struct shim_vma * tmp,
  59. const void * addr, uint64_t length)
  60. {
  61. return tmp->addr == addr && tmp->length == length;
  62. }
  63. static inline bool test_vma_contain (struct shim_vma * tmp,
  64. const void * addr, uint64_t length)
  65. {
  66. return tmp->addr <= addr &&
  67. tmp->addr + tmp->length >= addr + length;
  68. }
  69. static inline bool test_vma_startin (struct shim_vma * tmp,
  70. const void * addr, uint64_t length)
  71. {
  72. return tmp->addr >= addr &&
  73. tmp->addr < addr + length;
  74. }
  75. static inline bool test_vma_endin (struct shim_vma * tmp,
  76. const void * addr, uint64_t length)
  77. {
  78. return tmp->addr + tmp->length > addr &&
  79. tmp->addr + tmp->length <= addr + length;
  80. }
  81. static inline bool test_vma_overlap (struct shim_vma * tmp,
  82. const void * addr, uint64_t length)
  83. {
  84. return test_vma_contain(tmp, addr, 1) ||
  85. test_vma_contain(tmp, addr + length - 1, 1) ||
  86. test_vma_startin(tmp, addr, length);
  87. }
  88. int bkeep_shim_heap (void);
  89. static void __set_heap_top (void * bottom, void * top);
  90. int init_vma (void)
  91. {
  92. if (!(vma_mgr = create_mem_mgr(init_align_up(VMA_MGR_ALLOC)))) {
  93. debug("failed allocating VMAs\n");
  94. return -ENOMEM;
  95. }
  96. debug("User space range given from PAL: %p-%p\n",
  97. (void *) PAL_CB(user_address.start),
  98. (void *) PAL_CB(user_address.end));
  99. heap_bottom = (void *) PAL_CB(user_address.start);
  100. if (heap_bottom + DEFAULT_HEAP_MIN_SIZE > PAL_CB(executable_range.start)
  101. && heap_bottom < PAL_CB(executable_range.end)
  102. && heap_top > PAL_CB(executable_range.start))
  103. heap_bottom = (void *) ALIGN_UP(PAL_CB(executable_range.end));
  104. debug("setting initial heap to %p-%p\n", heap_bottom,
  105. (void *) PAL_CB(user_address.end));
  106. __set_heap_top(heap_bottom, (void *) PAL_CB(user_address.end));
  107. bkeep_shim_heap();
  108. create_lock(vma_list_lock);
  109. return 0;
  110. }
  111. /* This might not give the same vma but we might need to
  112. split after we find something */
  113. static inline void assert_vma (void)
  114. {
  115. struct shim_vma * tmp;
  116. struct shim_vma * prev __attribute__((unused)) = NULL;
  117. listp_for_each_entry(tmp, &vma_list, list) {
  118. /* Assert we are really sorted */
  119. assert(tmp->length > 0);
  120. assert(!prev || prev->addr + prev->length <= tmp->addr);
  121. prev = tmp;
  122. }
  123. }
  124. static struct shim_vma * __lookup_vma (const void * addr, uint64_t len);
  125. static struct shim_vma * __lookup_supervma (const void * addr, uint64_t length,
  126. struct shim_vma ** prev);
  127. static struct shim_vma * __lookup_overlap_vma (const void * addr, uint64_t length,
  128. struct shim_vma ** prev);
  129. void get_vma (struct shim_vma * vma)
  130. {
  131. #ifdef DEBUG_REF
  132. int ref_count = REF_INC(vma->ref_count);
  133. debug("get vma %p(%p-%p) (ref_count = %d)\n", vma, vma->addr,
  134. vma->addr + vma->length, ref_count);
  135. #else
  136. REF_INC(vma->ref_count);
  137. #endif
  138. }
  139. void put_vma (struct shim_vma * vma)
  140. {
  141. int ref_count = REF_DEC(vma->ref_count);
  142. #ifdef DEBUG_REF
  143. debug("put vma %p(%p-%p) (ref_count = %d)\n", vma,
  144. vma->addr, vma->addr + vma->length, ref_count - 1);
  145. #endif
  146. if (ref_count < 1) {
  147. if (vma->file)
  148. put_handle(vma->file);
  149. if (MEMORY_MIGRATED(vma))
  150. memset(vma, 0, sizeof(struct shim_vma));
  151. else
  152. free_mem_obj_to_mgr(vma_mgr, vma);
  153. }
  154. }
  155. static void __remove_vma (struct shim_vma * vma)
  156. {
  157. /* We assume all vmas are on the vma_list.
  158. * Perhaps worth asserting? */
  159. listp_del(vma, &vma_list, list);
  160. put_vma(vma);
  161. }
  162. static int __bkeep_mmap (void * addr, uint64_t length, int prot, int flags,
  163. struct shim_handle * file, uint64_t offset,
  164. const char * comment);
  165. static int __bkeep_mprotect (void * addr, uint64_t length, int prot,
  166. const int * flags);
  167. static void __check_delayed_bkeep (void)
  168. {
  169. struct shim_thread * thread = get_cur_thread();
  170. if (!thread)
  171. return;
  172. if (!thread->delayed_bkeep_mmap.addr)
  173. return;
  174. void * bkeep_addr = thread->delayed_bkeep_mmap.addr;
  175. uint64_t bkeep_length = thread->delayed_bkeep_mmap.length;
  176. thread->delayed_bkeep_mmap.addr = NULL;
  177. thread->delayed_bkeep_mmap.length = 0;
  178. __bkeep_mmap(bkeep_addr, bkeep_length,
  179. PROT_READ|PROT_WRITE,
  180. MAP_PRIVATE|MAP_ANONYMOUS|VMA_INTERNAL,
  181. NULL, 0, NULL);
  182. }
  183. static struct shim_vma * get_new_vma (void)
  184. {
  185. struct shim_vma * tmp =
  186. get_mem_obj_from_mgr_enlarge(vma_mgr, size_align_up(VMA_MGR_ALLOC));
  187. if (!tmp)
  188. return NULL;
  189. memset(tmp, 0, sizeof(struct shim_vma));
  190. INIT_LIST_HEAD(tmp, list);
  191. REF_SET(tmp->ref_count, 1);
  192. return tmp;
  193. }
  194. static bool check_vma_flags (const struct shim_vma * vma, const int * flags)
  195. {
  196. if (!flags)
  197. return true;
  198. if (vma->flags & VMA_UNMAPPED)
  199. return true;
  200. if ((vma->flags & VMA_INTERNAL) != ((*flags) & VMA_INTERNAL)) {
  201. debug("Check vma flag failure: vma flags %x, checked flags %x\n", vma->flags, *flags);
  202. bug();
  203. return false;
  204. }
  205. return true;
  206. }
  207. static inline void __set_comment (struct shim_vma * vma, const char * comment)
  208. {
  209. if (!comment) {
  210. vma->comment[0] = 0;
  211. return;
  212. }
  213. uint64_t len = strlen(comment);
  214. if (len > VMA_COMMENT_LEN - 1)
  215. len = VMA_COMMENT_LEN - 1;
  216. memcpy(vma->comment, comment, len);
  217. vma->comment[len] = 0;
  218. }
  219. static int __bkeep_mmap (void * addr, uint64_t length,
  220. int prot, int flags,
  221. struct shim_handle * file, uint64_t offset,
  222. const char * comment)
  223. {
  224. struct shim_vma * prev = NULL;
  225. struct shim_vma * tmp = __lookup_supervma(addr, length, &prev);
  226. int ret = 0;
  227. debug("bkeep_mmap: %p-%p\n", addr, addr + length);
  228. if (file)
  229. get_handle(file);
  230. if (tmp) { /* the range is included in a vma */
  231. if (tmp->addr != addr || tmp->length != length) {
  232. /* we are inside some unmapped area, do a split case */
  233. ret = __bkeep_mprotect(addr, length, prot, &flags);
  234. if (ret < 0)
  235. goto err;
  236. /* now we get the exact vma handle */
  237. tmp = __lookup_vma(addr, length);
  238. assert(tmp);
  239. assert(check_vma_flags(tmp, &flags));
  240. }
  241. } else {
  242. struct shim_vma * cont = NULL, * n; /* cont: continue to scan vmas */
  243. struct shim_vma * pos = NULL; /* pos: position to add the vma */
  244. if (prev && prev->addr == addr &&
  245. prev->length <= length) { /* find a vma at the same addr */
  246. cont = tmp = prev;
  247. } else { /* need to add a new vma */
  248. if (!(tmp = get_new_vma()))
  249. return -ENOMEM;
  250. if (prev) { /* has a precendent vma */
  251. if (test_vma_endin(prev, addr, length)) {
  252. if (!check_vma_flags(prev, &flags)) {
  253. ret = -EACCES;
  254. goto err;
  255. }
  256. /* the previous vma ends in the range; otherwise, there is
  257. * no overlapping. Another case is handled by the supervma
  258. * case. */
  259. prev->length = addr - prev->addr;
  260. }
  261. assert(prev->addr + prev->length <= addr);
  262. cont = prev;
  263. pos = prev;
  264. } else { /* has no precendent vma */
  265. cont = tmp;
  266. listp_add(tmp, &vma_list, list);
  267. }
  268. }
  269. if (cont)
  270. listp_for_each_entry_safe_continue(cont, n, &vma_list, list) {
  271. if (!test_vma_startin(cont, addr, length))
  272. break;
  273. if (!check_vma_flags(cont, &flags)) {
  274. ret = -EACCES;
  275. goto err;
  276. }
  277. if (test_vma_endin(cont, addr, length)) {
  278. __remove_vma(cont);
  279. continue;
  280. }
  281. long offset = addr + length - cont->addr;
  282. assert(offset > 0);
  283. if (cont->file)
  284. cont->offset += offset;
  285. cont->addr += offset;
  286. cont->length -= offset;
  287. break;
  288. }
  289. if (tmp && pos)
  290. list_add(tmp, pos, list);
  291. }
  292. tmp->addr = addr;
  293. tmp->length = length;
  294. tmp->prot = prot;
  295. tmp->flags = flags|((file && (prot & PROT_WRITE)) ? VMA_TAINTED : 0);
  296. tmp->file = file;
  297. tmp->offset = offset;
  298. __set_comment(tmp, comment);
  299. assert(!prev || prev == tmp || prev->addr + prev->length <= tmp->addr);
  300. return 0;
  301. err:
  302. if (file)
  303. put_handle(file);
  304. return ret;
  305. }
  306. int bkeep_mmap (void * addr, uint64_t length, int prot, int flags,
  307. struct shim_handle * file, uint64_t offset, const char * comment)
  308. {
  309. if (!addr || !length)
  310. return -EINVAL;
  311. lock(vma_list_lock);
  312. int ret = __bkeep_mmap(addr, length, prot, flags, file, offset,
  313. comment);
  314. //assert_vma();
  315. __check_delayed_bkeep();
  316. unlock(vma_list_lock);
  317. return ret;
  318. }
  319. /*
  320. * munmap start at any address and it might be split in between so
  321. * We need to split the area aur reduce the size
  322. * Check the address falls between alread allocated area or not
  323. */
  324. static int __bkeep_munmap (void * addr, uint64_t length, const int * flags)
  325. {
  326. struct shim_vma * tmp, * n;
  327. debug("bkeep_unmmap: %p-%p\n", addr, addr + length);
  328. listp_for_each_entry_safe(tmp, n, &vma_list, list) {
  329. if (test_vma_equal (tmp, addr, length)) {
  330. if (!check_vma_flags(tmp, flags))
  331. return -EACCES;
  332. __remove_vma(tmp);
  333. } else if (test_vma_overlap (tmp, addr, length)) {
  334. uint64_t before_length;
  335. uint64_t after_length;
  336. uint64_t after_offset;
  337. if (addr > tmp->addr)
  338. before_length = addr - tmp->addr;
  339. else
  340. before_length = 0;
  341. if (tmp->addr + tmp->length > addr + length)
  342. after_length = (tmp->addr + tmp->length) - (addr + length);
  343. else
  344. after_length = 0;
  345. after_offset = tmp->file ? tmp->offset + tmp->length -
  346. after_length : 0;
  347. /* split case
  348. * it is Unlikely that a process does an partical unmap
  349. * but We take care of it by splitting the book-keep
  350. *
  351. * case 1 if the vma is entirely between a mapped area
  352. * .e.g See case:
  353. * ---unmap--
  354. * ------map-----------
  355. */
  356. if (before_length) {
  357. /* Case 1: Space in the vma before */
  358. if (!check_vma_flags(tmp, flags))
  359. return -EACCES;
  360. tmp->length = before_length;
  361. if (after_length) {
  362. /* Case 2: Space before and also space after */
  363. int ret = __bkeep_mmap((void *) addr + length, after_length,
  364. tmp->prot, tmp->flags,
  365. tmp->file, after_offset,
  366. tmp->comment);
  367. if (ret < 0)
  368. return ret;
  369. }
  370. } else if (after_length) {
  371. /* Case 3: Only after length */
  372. if (!check_vma_flags(tmp, flags))
  373. return -EACCES;
  374. tmp->addr = (void *) addr + length;
  375. tmp->length = after_length;
  376. tmp->offset = after_offset;
  377. } else {
  378. if (!check_vma_flags(tmp, flags))
  379. return -EACCES;
  380. __remove_vma(tmp);
  381. }
  382. } else if (tmp->addr > (addr + length))
  383. break;
  384. }
  385. return 0;
  386. }
  387. int bkeep_munmap (void * addr, uint64_t length, const int * flags)
  388. {
  389. if (!addr || !length)
  390. return -EINVAL;
  391. lock(vma_list_lock);
  392. int ret = __bkeep_munmap(addr, length, flags);
  393. //assert_vma();
  394. __check_delayed_bkeep();
  395. unlock(vma_list_lock);
  396. return ret;
  397. }
  398. static int __bkeep_mprotect (void * addr, uint64_t length, int prot,
  399. const int * flags)
  400. {
  401. struct shim_vma * tmp = __lookup_vma(addr, length);
  402. int ret;
  403. debug("bkeep_mprotect: %p-%p\n", addr, addr + length);
  404. if (tmp) {
  405. /* exact match */
  406. if (!check_vma_flags(tmp, flags))
  407. return -EACCES;
  408. tmp->prot = prot;
  409. if (tmp->file && (prot & PROT_WRITE))
  410. tmp->flags |= VMA_TAINTED;
  411. return 0;
  412. }
  413. /* split case
  414. * it is Unlikely that a process does an partical unmap
  415. * but We take care of it by splitting the book-keep
  416. *
  417. * case 1 if the vma is entirely between a mapped area .e.g See case:
  418. * ---unmap--
  419. * ------map-----------
  420. */
  421. tmp = __lookup_supervma(addr, length, NULL);
  422. if (tmp) {
  423. if (!check_vma_flags(tmp, flags))
  424. return -EACCES;
  425. uint64_t before_length = addr - tmp->addr;
  426. uint64_t after_length = tmp->addr + tmp->length - addr - length;
  427. uint64_t after_offset = tmp->file ? tmp->offset + tmp->length -
  428. after_length : 0;
  429. uint64_t inside_offset = tmp->file ? tmp->offset + before_length : 0;
  430. /* split the handler first, because we might call bkeep_mmap */
  431. tmp->addr = (void *) addr;
  432. tmp->length = length;
  433. if (before_length) {
  434. ret = __bkeep_mmap((void *) addr - before_length, before_length,
  435. tmp->prot, tmp->flags,
  436. tmp->file, tmp->offset,
  437. tmp->comment);
  438. if (ret < 0)
  439. return ret;
  440. }
  441. if (after_length) {
  442. ret = __bkeep_mmap((void *)addr + length, after_length,
  443. tmp->prot, tmp->flags,
  444. tmp->file, after_offset,
  445. tmp->comment);
  446. if (ret < 0)
  447. return ret;
  448. }
  449. tmp->prot = prot;
  450. tmp->offset = inside_offset;
  451. if (tmp->file && (prot & PROT_WRITE))
  452. tmp->flags |= VMA_TAINTED;
  453. return 0;
  454. }
  455. /* split case
  456. * if the unmap are in between to mapped
  457. * area then we need to split two VMA here
  458. * This is the most unlikely case
  459. *
  460. * case 2
  461. * ------unmap------
  462. * ----map1-----;-----map2-------
  463. *
  464. * TODO: this algorithm is very inefficient, and may change
  465. * the mapping if it fails
  466. */
  467. uint64_t o_length = length;
  468. while (length) {
  469. struct shim_vma * candidate = NULL;
  470. listp_for_each_entry(tmp, &vma_list, list) {
  471. if (test_vma_contain (tmp, addr, 1)) {
  472. if (!check_vma_flags(tmp, flags))
  473. return -EACCES;
  474. uint64_t before_length = addr - tmp->addr;
  475. uint64_t after_length = tmp->addr + tmp->length > addr + length ?
  476. tmp->addr + tmp->length - addr - length : 0;
  477. uint64_t after_offset = tmp->file ? tmp->offset + tmp->length -
  478. after_length : 0;
  479. uint64_t inside_length = tmp->addr + tmp->length > addr + length ?
  480. length :
  481. addr + length - tmp->addr - tmp->length;
  482. uint64_t inside_offset = tmp->file ? tmp->offset + before_length : 0;
  483. /* split the handler first, because we might call bkeep_mmap */
  484. tmp->addr = (void *) addr;
  485. tmp->length = inside_length;
  486. if (before_length) {
  487. ret = __bkeep_mmap((void *) addr - before_length, before_length,
  488. tmp->prot, tmp->flags,
  489. tmp->file, tmp->offset,
  490. tmp->comment);
  491. if (ret < 0)
  492. return ret;
  493. }
  494. if (after_length) {
  495. ret = __bkeep_mmap((void *) addr + length, after_length,
  496. tmp->prot, tmp->flags,
  497. tmp->file, after_offset,
  498. tmp->comment);
  499. if (ret < 0)
  500. return ret;
  501. }
  502. tmp->prot = prot;
  503. tmp->offset = inside_offset;
  504. if (tmp->file && (prot & PROT_WRITE))
  505. tmp->flags |= VMA_TAINTED;
  506. addr += inside_length;
  507. length -= inside_length;
  508. break;
  509. }
  510. if (test_vma_startin(tmp, addr, length))
  511. if (!candidate || candidate->addr > tmp->addr)
  512. candidate = tmp;
  513. }
  514. if (o_length == length) {
  515. if (!candidate) {
  516. /* no more vmas, protect the whole area */
  517. ret = __bkeep_mmap((void *) addr, length, prot,
  518. VMA_UNMAPPED|(flags ? *flags : 0),
  519. NULL, 0, NULL);
  520. if (ret < 0)
  521. return ret;
  522. candidate = __lookup_vma((void *) addr, length);
  523. assert(candidate);
  524. /* DEP 10/19/16: If we make a vma that perfectly matches this
  525. * region, we want to break the loop and stop. */
  526. length = 0;
  527. }
  528. length -= candidate->addr - addr;
  529. }
  530. o_length = length;
  531. }
  532. return 0;
  533. }
  534. int bkeep_mprotect (void * addr, uint64_t length, int prot, const int * flags)
  535. {
  536. if (!addr || !length)
  537. return -EINVAL;
  538. lock(vma_list_lock);
  539. int ret = __bkeep_mprotect(addr, length, prot, flags);
  540. //assert_vma();
  541. unlock(vma_list_lock);
  542. return ret;
  543. }
  544. static void __set_heap_top (void * bottom, void * top)
  545. {
  546. bottom += DEFAULT_HEAP_MIN_SIZE;
  547. if (bottom >= top) {
  548. heap_top = top;
  549. return;
  550. }
  551. uint64_t rand;
  552. while (getrand(&rand, sizeof(uint64_t)) < sizeof(uint64_t));
  553. rand %= (uint64_t) (top - bottom) / allocsize;
  554. heap_top = bottom + rand * allocsize;
  555. debug("heap top adjusted to %p\n", heap_top);
  556. }
  557. void * get_unmapped_vma (uint64_t length, int flags)
  558. {
  559. struct shim_vma * new = get_new_vma(), * prev = NULL;
  560. if (!new)
  561. return NULL;
  562. lock(vma_list_lock);
  563. __check_delayed_bkeep();
  564. if (heap_top - heap_bottom < length) {
  565. debug("current heap %p-%p is not enough for allocating %lld bytes\n",
  566. heap_bottom, heap_top, length);
  567. unlock(vma_list_lock);
  568. put_vma(new);
  569. return NULL;
  570. }
  571. debug("find unmapped vma between %p-%p\n", heap_bottom, heap_top);
  572. do {
  573. int found = 0;
  574. new->addr = heap_top - length;
  575. new->length = length;
  576. new->flags = flags|VMA_UNMAPPED;
  577. new->prot = PROT_NONE;
  578. listp_for_each_entry_reverse(prev, &vma_list, list) {
  579. if (new->addr >= prev->addr + prev->length) {
  580. found = 1;
  581. break;
  582. }
  583. if (new->addr < heap_bottom) {
  584. found = 1;
  585. break;
  586. }
  587. if (prev->addr - heap_bottom < length) {
  588. unlock(vma_list_lock);
  589. put_vma(new);
  590. return NULL;
  591. }
  592. if (new->addr > prev->addr - length)
  593. new->addr = prev->addr - length;
  594. }
  595. /* DEP 6/4/17: This case appears to be detecting whether you wrapped around the
  596. * list wtihout finding anything. Let's add an explicit variable for
  597. * this case, but keep the check for now to be safe. */
  598. if (listp_empty(&vma_list)
  599. || (!found && (prev == listp_last_entry(&vma_list, shim_vma, list)))) {
  600. prev = NULL;
  601. break;
  602. }
  603. if (new->addr < heap_bottom) {
  604. if (heap_top == PAL_CB(user_address.end)) {
  605. unlock(vma_list_lock);
  606. put_vma(new);
  607. return NULL;
  608. } else {
  609. __set_heap_top(heap_top, (void *) PAL_CB(user_address.end));
  610. new->addr = NULL;
  611. }
  612. }
  613. } while (!new->addr);
  614. assert(!prev || prev->addr + prev->length <= new->addr);
  615. get_vma(new);
  616. listp_add_after(new, prev, &vma_list, list);
  617. debug("get unmapped: %p-%p\n", new->addr, new->addr + new->length);
  618. unlock(vma_list_lock);
  619. return new->addr;
  620. }
  621. #define NTRIES 4
  622. void * get_unmapped_vma_for_cp (uint64_t length)
  623. {
  624. struct shim_vma * new = get_new_vma(), * prev = NULL;
  625. if (!new)
  626. return NULL;
  627. if (length > PAL_CB(user_address.end) - PAL_CB(user_address.start)) {
  628. debug("user space is not enough for allocating %lld bytes\n", length);
  629. return NULL;
  630. }
  631. lock(vma_list_lock);
  632. __check_delayed_bkeep();
  633. uint64_t top = (uint64_t) PAL_CB(user_address.end) - length;
  634. uint64_t bottom = (uint64_t) heap_top;
  635. int flags = MAP_ANONYMOUS|VMA_UNMAPPED|VMA_INTERNAL;
  636. void * addr;
  637. if (bottom >= top) {
  638. unlock(vma_list_lock);
  639. return get_unmapped_vma(length, flags);
  640. }
  641. debug("find unmapped vma between %p-%p\n", bottom, top);
  642. for (int i = 0 ; i < NTRIES ; i++) {
  643. uint64_t rand;
  644. while (getrand(&rand, sizeof(uint64_t)) < sizeof(uint64_t));
  645. rand %= (uint64_t) (top - bottom) / allocsize;
  646. addr = (void *) bottom + rand * allocsize;
  647. if (!__lookup_overlap_vma(addr, length, &prev))
  648. break;
  649. addr = NULL;
  650. }
  651. if (!addr) {
  652. unlock(vma_list_lock);
  653. debug("cannot find unmapped vma for checkpoint\n");
  654. return NULL;
  655. }
  656. new->addr = addr;
  657. new->length = length;
  658. new->flags = flags;
  659. new->prot = PROT_NONE;
  660. listp_add_after(new, prev, &vma_list, list);
  661. unlock(vma_list_lock);
  662. return addr;
  663. }
  664. /* This might not give the same vma but we might need to
  665. split after we find something */
  666. static struct shim_vma * __lookup_overlap_vma (const void * addr, uint64_t length,
  667. struct shim_vma ** pprev)
  668. {
  669. struct shim_vma * tmp, * prev = NULL;
  670. listp_for_each_entry(tmp, &vma_list, list) {
  671. if (test_vma_overlap (tmp, addr, length)) {
  672. if (pprev)
  673. *pprev = prev;
  674. return tmp;
  675. }
  676. /* Assert we are really sorted */
  677. assert(!prev || prev->addr < tmp->addr);
  678. /* Insert in order; break once we are past the appropriate point */
  679. if (tmp->addr > addr)
  680. break;
  681. prev = tmp;
  682. }
  683. if (pprev)
  684. *pprev = prev;
  685. return NULL;
  686. }
  687. int lookup_overlap_vma (const void * addr, uint64_t length,
  688. struct shim_vma ** vma)
  689. {
  690. struct shim_vma * tmp = NULL;
  691. void * tmp_addr = NULL;
  692. uint64_t tmp_length;
  693. lock(vma_list_lock);
  694. if ((tmp = __lookup_overlap_vma(addr, length, NULL)) && vma)
  695. get_vma((tmp));
  696. if (tmp) {
  697. tmp_addr = tmp->addr;
  698. tmp_length = tmp->length;
  699. }
  700. unlock(vma_list_lock);
  701. if (tmp)
  702. debug("vma overlapped at %p-%p\n", tmp_addr, tmp_addr + tmp_length);
  703. if (vma)
  704. *vma = tmp;
  705. return tmp ? 0: -ENOENT;
  706. }
  707. static struct shim_vma * __lookup_vma (const void * addr, uint64_t length)
  708. {
  709. struct shim_vma * tmp;
  710. struct shim_vma * prev __attribute__((unused)) = NULL;
  711. listp_for_each_entry(tmp, &vma_list, list) {
  712. if (test_vma_equal(tmp, addr, length))
  713. return tmp;
  714. /* Assert we are really sorted */
  715. assert(!prev || prev->addr + prev->length <= tmp->addr);
  716. prev = tmp;
  717. }
  718. return NULL;
  719. }
  720. static struct shim_vma * __lookup_supervma (const void * addr, uint64_t length,
  721. struct shim_vma ** pprev)
  722. {
  723. struct shim_vma * tmp, * prev = NULL;
  724. listp_for_each_entry(tmp, &vma_list, list) {
  725. if (test_vma_contain(tmp, addr, length)) {
  726. if (pprev)
  727. *pprev = prev;
  728. return tmp;
  729. }
  730. /* Assert we are really sorted */
  731. if (!(!prev || prev->addr + prev->length <= tmp->addr)) {
  732. struct shim_vma * tmp2;
  733. warn("Failure\n");
  734. uint64_t last_addr = 0;
  735. listp_for_each_entry(tmp2, &vma_list, list) {
  736. warn ("Entry: %llx..%llx (%llx)\n", tmp2->addr, tmp2->addr + tmp2->length, tmp2->length);
  737. // Don't do an infinite dump if the list gets corrupted
  738. if (tmp2->addr < last_addr) {
  739. warn("VMA list corruption detected. Stopping debug print.\n");
  740. break;
  741. }
  742. last_addr = tmp2->addr;
  743. }
  744. warn("Prev is %p, tmp->addr = %llx, len is %llx\n", prev, tmp->addr, tmp->length);
  745. if (prev)
  746. warn("prev addr is %llx, len is %llx\n", prev->addr, prev->length);
  747. }
  748. assert(!prev || prev->addr + prev->length <= tmp->addr);
  749. /* Insert in order; break once we are past the appropriate point */
  750. if (tmp->addr > addr)
  751. break;
  752. prev = tmp;
  753. }
  754. if (pprev)
  755. *pprev = prev;
  756. return NULL;
  757. }
  758. int lookup_supervma (const void * addr, uint64_t length, struct shim_vma ** vma)
  759. {
  760. struct shim_vma * tmp = NULL;
  761. lock(vma_list_lock);
  762. if ((tmp = __lookup_supervma(addr, length, NULL)) && vma)
  763. get_vma((tmp));
  764. unlock(vma_list_lock);
  765. if (vma)
  766. *vma = tmp;
  767. return tmp ? 0 : -ENOENT;
  768. }
  769. struct shim_vma * next_vma (struct shim_vma * vma)
  770. {
  771. struct shim_vma * tmp = vma;
  772. lock(vma_list_lock);
  773. if (!tmp) {
  774. if (!listp_empty(&vma_list) &&
  775. (tmp = listp_first_entry(&vma_list, struct shim_vma, list)))
  776. get_vma(tmp);
  777. unlock(vma_list_lock);
  778. return tmp;
  779. }
  780. if (tmp->list.next == listp_first_entry(&vma_list, NULL, NULL)) {
  781. tmp = NULL;
  782. } else if (tmp->list.next == tmp) {
  783. struct shim_vma * tmp2;
  784. tmp = NULL;
  785. listp_for_each_entry(tmp2, &vma_list, list)
  786. if (tmp2->addr >= vma->addr) {
  787. tmp = tmp2;
  788. get_vma(tmp);
  789. break;
  790. }
  791. } else {
  792. tmp = list_entry(tmp->list.next, struct shim_vma, list);
  793. get_vma(tmp);
  794. }
  795. put_vma(vma);
  796. unlock(vma_list_lock);
  797. return tmp;
  798. }
  799. /* to speed up the checkpointing, go organize the VMAs */
  800. void __shrink_vmas (void)
  801. {
  802. struct shim_vma * vma, * n, * last;
  803. listp_for_each_entry_safe(vma, n, &vma_list, list) {
  804. if (!last)
  805. goto unmap;
  806. if (last->addr + last->length != vma->addr ||
  807. last->prot != vma->prot ||
  808. last->flags != vma->flags ||
  809. last->file != vma->file)
  810. goto unmap;
  811. if (last->file && last->offset + last->length != vma->offset)
  812. goto unmap;
  813. debug("shrink vma %p-%p and %p-%p\n", last->addr,
  814. last->addr + last->length, vma->addr, vma->addr + vma->length);
  815. last->length += vma->length;
  816. __remove_vma(vma);
  817. continue;
  818. next:
  819. last = vma;
  820. continue;
  821. unmap:
  822. if (vma->prot == PROT_NONE && !(vma->flags & VMA_TAINTED))
  823. vma->flags |= VMA_UNMAPPED;
  824. goto next;
  825. }
  826. }
  827. int dump_all_vmas (struct shim_thread * thread, char * buf, uint64_t size)
  828. {
  829. struct shim_vma * vma;
  830. int cnt = 0;
  831. lock(vma_list_lock);
  832. listp_for_each_entry(vma, &vma_list, list) {
  833. void * start = vma->addr, * end = vma->addr + vma->length;
  834. if ((vma->flags & (VMA_INTERNAL|VMA_UNMAPPED)) && !vma->comment[0])
  835. continue;
  836. char prot[3] = {'-', '-', '-'};
  837. if (vma->prot & PROT_READ)
  838. prot[0] = 'r';
  839. if (vma->prot & PROT_WRITE)
  840. prot[1] = 'w';
  841. if (vma->prot & PROT_EXEC)
  842. prot[2] = 'x';
  843. if (vma->file) {
  844. int dev_major = 0, dev_minor = 0;
  845. unsigned long ino = vma->file->dentry ? vma->file->dentry->ino : 0;
  846. const char * name = "[unknown]";
  847. if (!qstrempty(&vma->file->path))
  848. name = qstrgetstr(&vma->file->path);
  849. cnt += snprintf(buf + cnt, size - cnt,
  850. start > (void *) 0xffffffff ? "%lx" : "%08x",
  851. start);
  852. cnt += snprintf(buf + cnt, size - cnt,
  853. end > (void *) 0xffffffff ? "-%lx" : "-%08x", end);
  854. cnt += snprintf(buf + cnt, size - cnt,
  855. " %c%c%cp %08x %02d:%02d %u %s\n",
  856. prot[0], prot[1], prot[2],
  857. vma->offset, dev_major, dev_minor, ino, name);
  858. } else {
  859. cnt += snprintf(buf + cnt, size - cnt,
  860. start > (void *) 0xffffffff ? "%lx" : "%08x",
  861. start);
  862. cnt += snprintf(buf + cnt, size - cnt,
  863. end > (void *) 0xffffffff ? "-%lx" : "-%08x", end);
  864. if (vma->comment[0])
  865. cnt += snprintf(buf + cnt, size - cnt,
  866. " %c%c%cp 00000000 00:00 0 [%s]\n",
  867. prot[0], prot[1], prot[2], vma->comment);
  868. else
  869. cnt += snprintf(buf + cnt, size - cnt,
  870. " %c%c%cp 00000000 00:00 0\n",
  871. prot[0], prot[1], prot[2]);
  872. }
  873. if (cnt >= size) {
  874. cnt = -EOVERFLOW;
  875. break;
  876. }
  877. }
  878. unlock(vma_list_lock);
  879. return cnt;
  880. }
  881. void unmap_all_vmas (void)
  882. {
  883. struct shim_thread * cur_thread = get_cur_thread();
  884. struct shim_vma * tmp, * n;
  885. void * start = NULL, * end = NULL;
  886. lock(vma_list_lock);
  887. listp_for_each_entry_safe(tmp, n, &vma_list, list) {
  888. /* a adhoc vma can never be removed */
  889. if (tmp->flags & VMA_INTERNAL)
  890. continue;
  891. if (tmp->flags & VMA_UNMAPPED) {
  892. __remove_vma(tmp);
  893. continue;
  894. }
  895. if (cur_thread->stack &&
  896. test_vma_overlap(tmp, cur_thread->stack,
  897. cur_thread->stack_top - cur_thread->stack))
  898. continue;
  899. if (start == NULL)
  900. start = end = tmp->addr;
  901. if (end == tmp->addr) {
  902. end += tmp->length;
  903. __remove_vma(tmp);
  904. continue;
  905. }
  906. debug("removing vma %p - %p\n", start, end);
  907. DkVirtualMemoryFree(start, end - start);
  908. start = end = tmp->addr;
  909. end += tmp->length;
  910. __remove_vma(tmp);
  911. }
  912. if (start != NULL && start < end) {
  913. debug("removing vma %p - %p\n", start, end);
  914. DkVirtualMemoryFree(start, end - start);
  915. }
  916. unlock(vma_list_lock);
  917. }
  918. BEGIN_CP_FUNC(vma)
  919. {
  920. assert(size == sizeof(struct shim_vma));
  921. struct shim_vma * vma = (struct shim_vma *) obj;
  922. struct shim_vma * new_vma = NULL;
  923. PAL_FLG pal_prot = PAL_PROT(vma->prot, 0);
  924. ptr_t off = GET_FROM_CP_MAP(obj);
  925. if (!off) {
  926. off = ADD_CP_OFFSET(sizeof(struct shim_vma));
  927. ADD_TO_CP_MAP(obj, off);
  928. new_vma = (struct shim_vma *) (base + off);
  929. memcpy(new_vma, vma, sizeof(struct shim_vma));
  930. if (vma->file)
  931. DO_CP(handle, vma->file, &new_vma->file);
  932. REF_SET(new_vma->ref_count, 0);
  933. INIT_LIST_HEAD(new_vma, list);
  934. void * need_mapped = vma->addr;
  935. #if MIGRATE_MORE_GIPC == 1
  936. if (store->use_gipc) {
  937. if (!NEED_MIGRATE_MEMORY_IF_GIPC(vma))
  938. goto no_mem;
  939. } else {
  940. if (!NEED_MIGRATE_MEMORY(vma))
  941. goto no_mem;
  942. }
  943. #else
  944. if (!NEED_MIGRATE_MEMORY(vma))
  945. goto no_mem;
  946. #endif
  947. void * send_addr = vma->addr;
  948. uint64_t send_size = vma->length;
  949. bool protected = false;
  950. if (vma->file) {
  951. /*
  952. * Chia-Che 8/13/2017:
  953. * A fix for cloning a private VMA which maps a file to a process.
  954. *
  955. * (1) Application can access any page backed by the file, wholly
  956. * or partially.
  957. *
  958. * (2) Access beyond the last file-backed page will cause SIGBUS.
  959. * For reducing fork latency, the following code truncates the
  960. * memory size for migrating a process. The memory size is
  961. * truncated to the file size, round up to pages.
  962. *
  963. * (3) Data in the last file-backed page is valid before or after
  964. * forking. Has to be included in process migration.
  965. */
  966. uint64_t file_len = get_file_size(vma->file);
  967. if (file_len >= 0 &&
  968. vma->offset + vma->length > file_len) {
  969. send_size = file_len > vma->offset ?
  970. file_len - vma->offset : 0;
  971. send_size = ALIGN_UP(send_size);
  972. }
  973. }
  974. if (!send_size)
  975. goto no_mem;
  976. if (store->use_gipc) {
  977. #if HASH_GIPC == 1
  978. if (!(pal_prot & PAL_PROT_READ)) {
  979. protected = true;
  980. DkVirtualMemoryProtect(send_addr,
  981. send_size,
  982. pal_prot|PAL_PROT_READ);
  983. }
  984. #endif /* HASH_GIPC == 1 */
  985. struct shim_gipc_entry * gipc;
  986. DO_CP_SIZE(gipc, send_addr, send_size, &gipc);
  987. gipc->mem.prot = pal_prot;
  988. } else {
  989. if (!(pal_prot & PROT_READ)) {
  990. protected = true;
  991. DkVirtualMemoryProtect(send_addr,
  992. send_size,
  993. pal_prot|PAL_PROT_READ);
  994. }
  995. struct shim_mem_entry * mem;
  996. DO_CP_SIZE(memory, send_addr, send_size, &mem);
  997. mem->prot = pal_prot;
  998. }
  999. need_mapped = vma->addr + vma->length;
  1000. if (protected)
  1001. DkVirtualMemoryProtect(send_addr, send_size, pal_prot);
  1002. no_mem:
  1003. ADD_CP_FUNC_ENTRY(off);
  1004. ADD_CP_ENTRY(ADDR, need_mapped);
  1005. } else {
  1006. new_vma = (struct shim_vma *) (base + off);
  1007. }
  1008. if (objp)
  1009. *objp = (void *) new_vma;
  1010. }
  1011. END_CP_FUNC(vma)
  1012. DEFINE_PROFILE_CATAGORY(inside_rs_vma, resume_func);
  1013. DEFINE_PROFILE_INTERVAL(vma_lookup_overlap, inside_rs_vma);
  1014. DEFINE_PROFILE_INTERVAL(vma_add_bookkeep, inside_rs_vma);
  1015. DEFINE_PROFILE_INTERVAL(vma_map_file, inside_rs_vma);
  1016. DEFINE_PROFILE_INTERVAL(vma_map_anonymous, inside_rs_vma);
  1017. BEGIN_RS_FUNC(vma)
  1018. {
  1019. struct shim_vma * vma = (void *) (base + GET_CP_FUNC_ENTRY());
  1020. struct shim_vma * tmp, * prev = NULL;
  1021. void * need_mapped = (void *) GET_CP_ENTRY(ADDR);
  1022. int ret = 0;
  1023. CP_REBASE(vma->file);
  1024. CP_REBASE(vma->list);
  1025. lock(vma_list_lock);
  1026. BEGIN_PROFILE_INTERVAL();
  1027. tmp = __lookup_overlap_vma(vma->addr, vma->length, &prev);
  1028. SAVE_PROFILE_INTERVAL(vma_lookup_overlap);
  1029. if (tmp) {
  1030. if ((ret = __bkeep_munmap(vma->addr, vma->length, &vma->flags)) < 0)
  1031. return ret;
  1032. if (prev->list.next == tmp &&
  1033. tmp->addr < vma->addr)
  1034. prev = tmp;
  1035. }
  1036. get_vma(vma);
  1037. listp_add_after(vma, prev, &vma_list, list);
  1038. assert_vma();
  1039. SAVE_PROFILE_INTERVAL(vma_add_bookkeep);
  1040. __check_delayed_bkeep();
  1041. unlock(vma_list_lock);
  1042. debug("vma: %p-%p flags %x prot %p\n", vma->addr, vma->addr + vma->length,
  1043. vma->flags, vma->prot);
  1044. if (!(vma->flags & VMA_UNMAPPED)) {
  1045. if (vma->file) {
  1046. struct shim_mount * fs = vma->file->fs;
  1047. get_handle(vma->file);
  1048. if (need_mapped < vma->addr + vma->length) {
  1049. /* first try, use hstat to force it resumes pal handle */
  1050. assert(vma->file->fs && vma->file->fs->fs_ops &&
  1051. vma->file->fs->fs_ops->mmap);
  1052. void * addr = need_mapped;
  1053. int ret = fs->fs_ops->mmap(vma->file, &addr,
  1054. vma->addr + vma->length -
  1055. need_mapped,
  1056. vma->prot,
  1057. vma->flags,
  1058. vma->offset +
  1059. (need_mapped - vma->addr));
  1060. if (ret < 0)
  1061. return ret;
  1062. if (!addr)
  1063. return -ENOMEM;
  1064. if (addr != need_mapped)
  1065. return -EACCES;
  1066. need_mapped += vma->length;
  1067. SAVE_PROFILE_INTERVAL(vma_map_file);
  1068. }
  1069. }
  1070. if (need_mapped < vma->addr + vma->length) {
  1071. int pal_alloc_type = 0;
  1072. int pal_prot = vma->prot;
  1073. if (DkVirtualMemoryAlloc(need_mapped,
  1074. vma->addr + vma->length - need_mapped,
  1075. pal_alloc_type, pal_prot)) {
  1076. need_mapped += vma->length;
  1077. SAVE_PROFILE_INTERVAL(vma_map_anonymous);
  1078. }
  1079. }
  1080. if (need_mapped < vma->addr + vma->length)
  1081. sys_printf("vma %p-%p cannot be allocated!\n", need_mapped,
  1082. vma->addr + vma->length);
  1083. }
  1084. if (vma->file)
  1085. get_handle(vma->file);
  1086. if (vma->file)
  1087. DEBUG_RS("%p-%p,size=%d,prot=%08x,flags=%08x,off=%d,path=%s,uri=%s",
  1088. vma->addr, vma->addr + vma->length, vma->length,
  1089. vma->prot, vma->flags, vma->offset,
  1090. qstrgetstr(&vma->file->path), qstrgetstr(&vma->file->uri));
  1091. else
  1092. DEBUG_RS("%p-%p,size=%d,prot=%08x,flags=%08x,off=%d",
  1093. vma->addr, vma->addr + vma->length, vma->length,
  1094. vma->prot, vma->flags, vma->offset);
  1095. }
  1096. END_RS_FUNC(vma)
  1097. BEGIN_CP_FUNC(all_vmas)
  1098. {
  1099. struct shim_vma * tmp, ** vmas;
  1100. int nvmas = 0, cnt = 0;
  1101. lock(vma_list_lock);
  1102. __shrink_vmas();
  1103. listp_for_each_entry(tmp, &vma_list, list)
  1104. if (!(tmp->flags & VMA_INTERNAL))
  1105. nvmas++;
  1106. if (!nvmas) {
  1107. unlock(vma_list_lock);
  1108. return 0;
  1109. }
  1110. vmas = __alloca(sizeof(struct shim_vam *) * nvmas);
  1111. listp_for_each_entry(tmp, &vma_list, list)
  1112. if (!(tmp->flags & VMA_INTERNAL)) {
  1113. get_vma(tmp);
  1114. vmas[cnt++] = tmp;
  1115. }
  1116. unlock(vma_list_lock);
  1117. for (cnt = 0 ; cnt < nvmas ; cnt++) {
  1118. DO_CP(vma, vmas[cnt], NULL);
  1119. put_vma(vmas[cnt]);
  1120. }
  1121. }
  1122. END_CP_FUNC_NO_RS(all_vmas)
  1123. void debug_print_vma_list (void)
  1124. {
  1125. sys_printf("vma bookkeeping:\n");
  1126. struct shim_vma * vma;
  1127. listp_for_each_entry(vma, &vma_list, list) {
  1128. const char * type = "", * name = "";
  1129. if (vma->file) {
  1130. if (!qstrempty(&vma->file->path)) {
  1131. type = " path=";
  1132. name = qstrgetstr(&vma->file->path);
  1133. } else if (!qstrempty(&vma->file->uri)) {
  1134. type = " uri=";
  1135. name = qstrgetstr(&vma->file->uri);
  1136. }
  1137. }
  1138. sys_printf("[%p-%p] prot=%08x flags=%08x%s%s offset=%d%s%s%s%s\n",
  1139. vma->addr, vma->addr + vma->length,
  1140. vma->prot,
  1141. vma->flags & ~(VMA_INTERNAL|VMA_UNMAPPED|VMA_TAINTED),
  1142. type, name,
  1143. vma->offset,
  1144. vma->flags & VMA_INTERNAL ? " (internal)" : "",
  1145. vma->flags & VMA_UNMAPPED ? " (unmapped)" : "",
  1146. vma->comment[0] ? " comment=" : "",
  1147. vma->comment[0] ? vma->comment : "");
  1148. }
  1149. }
  1150. void print_vma_hash (struct shim_vma * vma, void * addr, uint64_t len,
  1151. bool force_protect)
  1152. {
  1153. if (!addr)
  1154. addr = vma->addr;
  1155. if (!len)
  1156. len = vma->length - (addr - vma->addr);
  1157. if (addr < vma->addr || addr + len > vma->addr + vma->length)
  1158. return;
  1159. if (!(vma->prot & PROT_READ)) {
  1160. if (!force_protect)
  1161. return;
  1162. DkVirtualMemoryProtect(vma->addr, vma->length, PAL_PROT_READ);
  1163. }
  1164. for (uint64_t p = (uint64_t) addr ;
  1165. p < (uint64_t) addr + len ; p += allocsize) {
  1166. uint64_t hash = 0;
  1167. struct shim_md5_ctx ctx;
  1168. md5_init(&ctx);
  1169. md5_update(&ctx, (void *) p, allocsize);
  1170. md5_final(&ctx);
  1171. memcpy(&hash, ctx.digest, sizeof(uint64_t));
  1172. }
  1173. if (!(vma->prot & PROT_READ))
  1174. DkVirtualMemoryProtect(vma->addr, vma->length, vma->prot);
  1175. }