shim_rtld.c 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734
  1. /* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
  2. /* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
  3. /* Copyright (C) 2014 Stony Brook University
  4. This file is part of Graphene Library OS.
  5. Graphene Library OS is free software: you can redistribute it and/or
  6. modify it under the terms of the GNU Lesser General Public License
  7. as published by the Free Software Foundation, either version 3 of the
  8. License, or (at your option) any later version.
  9. Graphene Library OS is distributed in the hope that it will be useful,
  10. but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. GNU Lesser General Public License for more details.
  13. You should have received a copy of the GNU Lesser General Public License
  14. along with this program. If not, see <http://www.gnu.org/licenses/>. */
  15. /*
  16. * shim_rtld.c
  17. *
  18. * This file contains codes for dynamic loading of ELF binaries in library OS.
  19. * It's espeically used for loading interpreter (ld.so, in general) and
  20. * optimization of execve.
  21. * Most of the source codes are imported from GNU C library.
  22. */
  23. #include <shim_internal.h>
  24. #include <shim_table.h>
  25. #include <shim_utils.h>
  26. #include <shim_handle.h>
  27. #include <shim_thread.h>
  28. #include <shim_fs.h>
  29. #include <shim_vma.h>
  30. #include <shim_checkpoint.h>
  31. #include <shim_profile.h>
  32. #include <errno.h>
  33. #include <asm/prctl.h>
  34. #include <asm/mman.h>
  35. #include "ldsodefs.h"
  36. #include "elf.h"
  37. #ifndef DT_THISPROCNUM
  38. # define DT_THISPROCNUM 0
  39. #endif
  40. typedef ElfW(Word) Elf_Symndx;
  41. #define BOOKKEEP_INTERNAL_OBJ 0
  42. enum object_type {
  43. OBJECT_INTERNAL = 0,
  44. OBJECT_LOAD = 1,
  45. OBJECT_MAPPED = 2,
  46. OBJECT_REMAP = 3,
  47. OBJECT_USER = 4,
  48. };
  49. /* Structure describing a loaded shared object. The `l_next' and `l_prev'
  50. members form a chain of all the shared objects loaded at startup.
  51. These data structures exist in space used by the run-time dynamic linker;
  52. modifying them may have disastrous results.
  53. This data structure might change in future, if necessary. User-level
  54. programs must avoid defining objects of this type. */
  55. /* This is a simplified link_map structure */
  56. struct link_map {
  57. /* These first few members are part of the protocol with the debugger.
  58. This is the same format used in SVR4. */
  59. ElfW(Addr) l_addr; /* Base address shared object is loaded at. */
  60. const char * l_name; /* Absolute file name object was found in. */
  61. ElfW(Dyn) * l_real_ld; /* Dynamic section of the shared object. */
  62. struct link_map * l_next, * l_prev; /* Chain of loaded objects. */
  63. /* All following members are internal to the dynamic linker.
  64. They may change without notice. */
  65. ElfW(Dyn) * l_ld;
  66. char * l_soname;
  67. ElfW(Dyn) *l_info[DT_NUM + DT_THISPROCNUM + DT_VERSIONTAGNUM
  68. + DT_EXTRANUM + DT_VALNUM + DT_ADDRNUM];
  69. const ElfW(Phdr) *l_phdr; /* Pointer to program header table in core. */
  70. ElfW(Addr) l_entry; /* Entry point location. */
  71. ElfW(Half) l_phnum; /* Number of program header entries. */
  72. ElfW(Half) l_ldnum; /* Number of dynamic segment entries. */
  73. /* Start and finish of memory map for this object. l_map_start
  74. need not be the same as l_addr. */
  75. ElfW(Addr) l_map_start, l_map_end;
  76. bool l_resolved;
  77. ElfW(Addr) l_resolved_map;
  78. const char * l_interp_libname;
  79. ElfW(Addr) l_main_entry;
  80. /* Information used to change permission after the relocations are
  81. done. */
  82. ElfW(Addr) l_relro_addr;
  83. size_t l_relro_size;
  84. /* For DT_HASH */
  85. Elf_Symndx l_nbuckets;
  86. const Elf_Symndx * l_buckets;
  87. const Elf_Symndx * l_chain;
  88. /* For DT_GNU_HASH */
  89. Elf32_Word l_gnu_bitmask_idxbits;
  90. Elf32_Word l_gnu_shift;
  91. const ElfW(Addr) * l_gnu_bitmask;
  92. const Elf32_Word * l_gnu_buckets;
  93. const Elf32_Word * l_gnu_chain_zero;
  94. /* pointer to related file */
  95. struct shim_handle * l_file;
  96. enum object_type l_type;
  97. #define MAX_LOADCMDS 4
  98. struct loadcmd {
  99. ElfW(Addr) mapstart, mapend, dataend, allocend;
  100. off_t mapoff;
  101. int prot, flags;
  102. struct shim_vma * vma;
  103. } loadcmds[MAX_LOADCMDS];
  104. int nloadcmds;
  105. struct textrel {
  106. ElfW(Addr) start, end;
  107. int prot;
  108. struct textrel * next;
  109. } * textrels;
  110. #define MAX_LINKSYMS 32
  111. struct linksym {
  112. void * rel;
  113. ElfW(Sym) * sym;
  114. void * reloc;
  115. } linksyms[MAX_LINKSYMS];
  116. int nlinksyms;
  117. };
  118. struct link_map * lookup_symbol (const char * undef_name, ElfW(Sym) ** ref);
  119. static struct link_map * loaded_libraries = NULL;
  120. static struct link_map * internal_map = NULL, * interp_map = NULL;
  121. /* This macro is used as a callback from the ELF_DYNAMIC_RELOCATE code. */
  122. static ElfW(Addr) resolve_map (const char ** strtab, ElfW(Sym) ** ref)
  123. {
  124. if (ELFW(ST_BIND) ((*ref)->st_info) != STB_LOCAL) {
  125. struct link_map *l = lookup_symbol ((*strtab) + (*ref)->st_name, ref);
  126. if (l) {
  127. *strtab = (const void *) D_PTR (l->l_info[DT_STRTAB]);
  128. return l->l_addr;
  129. }
  130. }
  131. return 0;
  132. }
  133. static int protect_page (struct link_map * l, void * addr, size_t size)
  134. {
  135. struct loadcmd * c = l->loadcmds;
  136. int prot = 0;
  137. for ( ; c < &l->loadcmds[l->nloadcmds] ; c++)
  138. if ((void *) l->l_addr + c->mapstart <= addr &&
  139. addr + size <= (void *) l->l_addr + c->mapend)
  140. break;
  141. if (c < &l->loadcmds[l->nloadcmds])
  142. prot = c->prot;
  143. struct textrel * t = l->textrels, ** loc = &l->textrels;
  144. for ( ; t ; t = t->next) {
  145. if ((void *) t->start <= addr && addr + size <= (void *) t->end)
  146. return 0;
  147. loc = &t->next;
  148. }
  149. if ((prot & (PROT_READ|PROT_WRITE)) == (PROT_READ|PROT_WRITE)) {
  150. struct shim_vma * vma = NULL;
  151. /* the actual protection of the vma might be changed */
  152. if (lookup_supervma(addr, size, &vma) < 0)
  153. return 0;
  154. prot = vma->prot;
  155. put_vma(vma);
  156. if ((prot & (PROT_READ|PROT_WRITE)) == (PROT_READ|PROT_WRITE))
  157. return 0;
  158. }
  159. void * start = ALIGN_DOWN(addr);
  160. void * end = ALIGN_UP(addr + size);
  161. if (!DkVirtualMemoryProtect(start, end - start,
  162. PAL_PROT_READ|PAL_PROT_WRITE|prot))
  163. return -PAL_ERRNO;
  164. if (!c)
  165. return 0;
  166. t = malloc(sizeof(struct textrel));
  167. if (!t)
  168. return -ENOMEM;
  169. t->start = (ElfW(Addr)) start;
  170. t->end = (ElfW(Addr)) end;
  171. t->prot = prot;
  172. t->next = NULL;
  173. *loc = t;
  174. return 0;
  175. }
  176. static int reprotect_map (struct link_map * l)
  177. {
  178. struct textrel * t = l->textrels, * next;
  179. int ret = 0;
  180. while (t) {
  181. struct loadcmd * c = l->loadcmds;
  182. for ( ; c < &l->loadcmds[l->nloadcmds] ; c++)
  183. if (l->l_addr + c->mapstart <= t->start &&
  184. t->end <= l->l_addr + c->mapend)
  185. break;
  186. ElfW(Addr) start = t->start, end = t->end;
  187. int prot = t->prot;
  188. next = t->next;
  189. free(t);
  190. t = next;
  191. l->textrels = t;
  192. if (c && !DkVirtualMemoryProtect((void *) start, end - start, prot)) {
  193. ret = -PAL_ERRNO;
  194. break;
  195. }
  196. }
  197. return ret;
  198. }
  199. #define RESOLVE_MAP(strtab, ref) resolve_map(strtab, ref)
  200. #define PROTECT_PAGE(map, addr, size) protect_page(map, addr, size)
  201. #define USE__THREAD 0 /* disable TLS support */
  202. #include "rel.h"
  203. struct link_map * new_elf_object (const char * realname, int type)
  204. {
  205. struct link_map *new;
  206. new = (struct link_map *) malloc (sizeof (struct link_map));
  207. if (new == NULL)
  208. return NULL;
  209. /* We apparently expect this to be zeroed. */
  210. memset(new, 0, sizeof(struct link_map));
  211. new->l_name = realname;
  212. new->l_type = type;
  213. return new;
  214. }
  215. #include <endian.h>
  216. #if BYTE_ORDER == BIG_ENDIAN
  217. # define byteorder ELFDATA2MSB
  218. #elif BYTE_ORDER == LITTLE_ENDIAN
  219. # define byteorder ELFDATA2LSB
  220. #else
  221. # error "Unknown BYTE_ORDER " BYTE_ORDER
  222. # define byteorder ELFDATANONE
  223. #endif
  224. #if __WORDSIZE == 32
  225. # define FILEBUF_SIZE 512
  226. #else
  227. # define FILEBUF_SIZE 832
  228. #endif
  229. /* Cache the location of MAP's hash table. */
  230. void setup_elf_hash (struct link_map * map)
  231. {
  232. Elf_Symndx * hash;
  233. if (__builtin_expect (map->l_info[DT_ADDRTAGIDX (DT_GNU_HASH) + DT_NUM
  234. + DT_THISPROCNUM + DT_VERSIONTAGNUM
  235. + DT_EXTRANUM + DT_VALNUM] != NULL, 1)) {
  236. Elf32_Word * hash32
  237. = (void *) D_PTR (map->l_info[DT_ADDRTAGIDX (DT_GNU_HASH) + DT_NUM
  238. + DT_THISPROCNUM + DT_VERSIONTAGNUM
  239. + DT_EXTRANUM + DT_VALNUM]);
  240. map->l_nbuckets = *hash32++;
  241. Elf32_Word symbias = *hash32++;
  242. Elf32_Word bitmask_nwords = *hash32++;
  243. /* Must be a power of two. */
  244. assert ((bitmask_nwords & (bitmask_nwords - 1)) == 0);
  245. map->l_gnu_bitmask_idxbits = bitmask_nwords - 1;
  246. map->l_gnu_shift = *hash32++;
  247. map->l_gnu_bitmask = (ElfW(Addr) *) hash32;
  248. hash32 += __ELF_NATIVE_CLASS / 32 * bitmask_nwords;
  249. map->l_gnu_buckets = hash32;
  250. hash32 += map->l_nbuckets;
  251. map->l_gnu_chain_zero = hash32 - symbias;
  252. return;
  253. }
  254. if (!map->l_info[DT_HASH])
  255. return;
  256. hash = (void *) D_PTR (map->l_info[DT_HASH]);
  257. /* Structure of DT_HASH:
  258. The bucket array forms the hast table itself. The entries in the
  259. chain array parallel the symbol table.
  260. [ nbucket ]
  261. [ nchain ]
  262. [ bucket[0] ]
  263. [ ... ]
  264. [ bucket[nbucket-1] ]
  265. [ chain[0] ]
  266. [ ... ]
  267. [ chain[nchain-1] ] */
  268. map->l_nbuckets = *hash++;
  269. hash++;
  270. map->l_buckets = hash;
  271. hash += map->l_nbuckets;
  272. map->l_chain = hash;
  273. }
  274. /* Map in the shared object NAME, actually located in REALNAME, and already
  275. opened on FD */
  276. static struct link_map *
  277. __map_elf_object (struct shim_handle * file,
  278. const void * fbp, size_t fbp_len, void * addr, int type,
  279. struct link_map * remap)
  280. {
  281. if (file && (!file->fs || !file->fs->fs_ops))
  282. return NULL;
  283. int (*read) (struct shim_handle *, void *, size_t) =
  284. file ? file->fs->fs_ops->read : NULL;
  285. int (*mmap) (struct shim_handle *, void **, size_t, int, int, off_t) =
  286. file ? file->fs->fs_ops->mmap : NULL;
  287. int (*seek) (struct shim_handle *, off_t, int) =
  288. file ? file->fs->fs_ops->seek : NULL;
  289. if (file && (!read || !mmap || !seek))
  290. return NULL;
  291. struct link_map * l = remap ? :
  292. new_elf_object(file ? (!qstrempty(&file->path) ?
  293. qstrgetstr(&file->path) :
  294. qstrgetstr(&file->uri)) : "", type);
  295. const char * errstring __attribute__((unused)) = NULL;
  296. int errval = 0;
  297. int ret;
  298. if (type != OBJECT_INTERNAL && !file) {
  299. errstring = "shared object has to be backed by file";
  300. errval = -EINVAL;
  301. call_lose:
  302. debug("loading %s: %s\n", l->l_name, errstring);
  303. return NULL;
  304. }
  305. /* Scan the program header table, collecting its load commands. */
  306. struct loadcmd * c = l->loadcmds;
  307. /* This is the ELF header. We read it in `open_verify'. */
  308. const ElfW(Ehdr) * header = fbp;
  309. if (type == OBJECT_REMAP)
  310. goto do_remap;
  311. /* Extract the remaining details we need from the ELF header
  312. and then read in the program header table. */
  313. l->l_addr = (ElfW(Addr)) addr;
  314. l->l_entry = header->e_entry;
  315. int e_type = header->e_type;
  316. l->l_phnum = header->e_phnum;
  317. size_t maplength = header->e_phnum * sizeof (ElfW(Phdr));
  318. const ElfW(Phdr) * phdr = (fbp + header->e_phoff);
  319. if (type == OBJECT_LOAD &&
  320. header->e_phoff + maplength <= (size_t) fbp_len) {
  321. ElfW(Phdr) * new_phdr = (ElfW(Phdr) *) malloc (maplength);
  322. if ((ret = (*seek) (file, header->e_phoff, SEEK_SET)) < 0 ||
  323. (ret = (*read) (file, new_phdr, maplength)) < 0) {
  324. errstring = "cannot read file data";
  325. errval = ret;
  326. goto call_lose;
  327. }
  328. phdr = new_phdr;
  329. }
  330. l->nloadcmds = 0;
  331. bool has_holes = false;
  332. const ElfW(Phdr) * ph;
  333. for (ph = phdr; ph < &phdr[l->l_phnum]; ++ph)
  334. switch (ph->p_type) {
  335. /* These entries tell us where to find things once the file's
  336. segments are mapped in. We record the addresses it says
  337. verbatim, and later correct for the run-time load address. */
  338. case PT_DYNAMIC:
  339. l->l_ld = (void *) ph->p_vaddr;
  340. l->l_ldnum = ph->p_memsz / sizeof (ElfW(Dyn));
  341. break;
  342. case PT_INTERP:
  343. l->l_interp_libname = (const char *) ph->p_vaddr;
  344. break;
  345. case PT_PHDR:
  346. l->l_phdr = (void *) ph->p_vaddr;
  347. break;
  348. case PT_LOAD:
  349. /* A load command tells us to map in part of the file.
  350. We record the load commands and process them all later. */
  351. if (__builtin_expect (!ALIGNED(ph->p_align), 0)) {
  352. errstring = "ELF load command alignment not page-aligned";
  353. errval = ENOMEM;
  354. goto call_lose;
  355. }
  356. if (__builtin_expect (((ph->p_vaddr - ph->p_offset)
  357. & (ph->p_align - 1)) != 0, 0)) {
  358. errstring = "\
  359. ELF load command address/offset not properly aligned";
  360. errval = ENOMEM;
  361. goto call_lose;
  362. }
  363. if (l->nloadcmds >= MAX_LOADCMDS) {
  364. errstring = "too many load commamds";
  365. errval = -EINVAL;
  366. goto call_lose;
  367. }
  368. c = &l->loadcmds[l->nloadcmds++];
  369. c->mapstart = ALIGN_DOWN(ph->p_vaddr);
  370. c->mapend = ALIGN_UP(ph->p_vaddr + ph->p_filesz);
  371. c->dataend = ph->p_vaddr + ph->p_filesz;
  372. c->allocend = ph->p_vaddr + ph->p_memsz;
  373. c->mapoff = ALIGN_DOWN(ph->p_offset);
  374. /* Determine whether there is a gap between the last segment
  375. and this one. */
  376. if (l->nloadcmds > 1 && c[-1].mapend != c->mapstart)
  377. has_holes = true;
  378. /* Optimize a common case. */
  379. #if (PF_R | PF_W | PF_X) == 7 && (PROT_READ | PROT_WRITE | PROT_EXEC) == 7
  380. c->prot = (PF_TO_PROT
  381. >> ((ph->p_flags & (PF_R | PF_W | PF_X)) * 4)) & 0xf;
  382. #else
  383. c->prot = 0;
  384. if (ph->p_flags & PF_R)
  385. c->prot |= PROT_READ;
  386. if (ph->p_flags & PF_W)
  387. c->prot |= PROT_WRITE;
  388. if (ph->p_flags & PF_X)
  389. c->prot |= PROT_EXEC;
  390. #endif
  391. c->flags = MAP_PRIVATE|MAP_FILE;
  392. break;
  393. case PT_GNU_RELRO:
  394. l->l_relro_addr = ph->p_vaddr;
  395. l->l_relro_size = ph->p_memsz;
  396. break;
  397. }
  398. if (__builtin_expect (l->nloadcmds == 0, 0)) {
  399. /* This only happens for a bogus object that will be caught with
  400. another error below. But we don't want to go through the
  401. calculations below using NLOADCMDS - 1. */
  402. errstring = "object file has no loadable segments";
  403. goto call_lose;
  404. }
  405. c = &l->loadcmds[0];
  406. /* Length of the sections to be loaded. */
  407. maplength = l->loadcmds[l->nloadcmds - 1].allocend - c->mapstart;
  408. if (__builtin_expect (e_type, ET_DYN) == ET_DYN) {
  409. /* This is a position-independent shared object. We can let the
  410. kernel map it anywhere it likes, but we must have space for all
  411. the segments in their specified positions relative to the first.
  412. So we map the first segment without MAP_FIXED, but with its
  413. extent increased to cover all the segments. Then we remove
  414. access from excess portion, and there is known sufficient space
  415. there to remap from the later segments.
  416. As a refinement, sometimes we have an address that we would
  417. prefer to map such objects at; but this is only a preference,
  418. the OS can do whatever it likes. */
  419. ElfW(Addr) mappref = 0;
  420. if (type == OBJECT_LOAD) {
  421. if (addr)
  422. mappref = (ElfW(Addr)) c->mapstart + (ElfW(Addr)) addr;
  423. else
  424. mappref = (ElfW(Addr)) get_unmapped_vma(ALIGN_UP(maplength),
  425. MAP_PRIVATE|MAP_ANONYMOUS);
  426. /* Remember which part of the address space this object uses. */
  427. errval = (*mmap) (file, (void **) &mappref, ALIGN_UP(maplength),
  428. c->prot, c->flags|MAP_PRIVATE, c->mapoff);
  429. if (__builtin_expect (errval < 0, 0)) {
  430. map_error:
  431. errstring = "failed to map segment from shared object";
  432. goto call_lose;
  433. }
  434. } else {
  435. mappref = (ElfW(Addr)) addr;
  436. }
  437. l->l_map_start = mappref;
  438. l->l_map_end = l->l_map_start + maplength;
  439. #if BOOKKEEP_INTERNAL_OBJ == 0
  440. if (type != OBJECT_INTERNAL && type != OBJECT_USER)
  441. #else
  442. if (type != OBJECT_USER)
  443. #endif
  444. bkeep_mmap((void *) mappref, ALIGN_UP(maplength), c->prot,
  445. c->flags|MAP_PRIVATE|
  446. (type == OBJECT_INTERNAL ? VMA_INTERNAL : 0),
  447. file, c->mapoff, NULL);
  448. l->l_addr = l->l_map_start - c->mapstart;
  449. if (has_holes) {
  450. /* Change protection on the excess portion to disallow all access;
  451. the portions we do not remap later will be inaccessible as if
  452. unallocated. Then jump into the normal segment-mapping loop to
  453. handle the portion of the segment past the end of the file
  454. mapping. */
  455. if (type == OBJECT_LOAD)
  456. DkVirtualMemoryProtect((void *) RELOCATE(l, c->mapend),
  457. l->loadcmds[l->nloadcmds - 1].mapstart -
  458. c->mapend, PAL_PROT_NONE);
  459. if (type == OBJECT_MAPPED ||
  460. #if BOOKKEEP_INTERNAL_OBJ == 1
  461. type == OBJECT_INTERNAL ||
  462. #endif
  463. type == OBJECT_LOAD) {
  464. #if BOOKKEEP_INTERNAL_OBJ == 1
  465. int flags = (type == OBJECT_INTERNVAL) ? VMA_INTERVAL : 0;
  466. #else
  467. int flags = 0;
  468. #endif
  469. bkeep_mprotect((void *) RELOCATE(l, c->mapend),
  470. l->loadcmds[l->nloadcmds - 1].mapstart -
  471. c->mapend, PROT_NONE, &flags);
  472. }
  473. }
  474. goto postmap;
  475. }
  476. /* Remember which part of the address space this object uses. */
  477. l->l_addr = 0;
  478. l->l_map_start = c->mapstart;
  479. l->l_map_end = l->l_map_start + maplength;
  480. do_remap:
  481. while (c < &l->loadcmds[l->nloadcmds]) {
  482. if (c->mapend > c->mapstart) {
  483. /* Map the segment contents from the file. */
  484. void * mapaddr = (void *) RELOCATE(l, c->mapstart);
  485. if (type == OBJECT_LOAD || type == OBJECT_REMAP) {
  486. if ((*mmap) (file, &mapaddr, c->mapend - c->mapstart, c->prot,
  487. c->flags|MAP_FIXED|MAP_PRIVATE, c->mapoff) < 0)
  488. goto map_error;
  489. }
  490. #if BOOKKEEP_INTERNAL_OBJ == 0
  491. if (type != OBJECT_INTERNAL && type != OBJECT_USER)
  492. #else
  493. if (type != OBJECT_USER)
  494. #endif
  495. bkeep_mmap(mapaddr, c->mapend - c->mapstart, c->prot,
  496. c->flags|MAP_FIXED|MAP_PRIVATE|
  497. (type == OBJECT_INTERNAL ? VMA_INTERNAL : 0),
  498. file, c->mapoff, NULL);
  499. }
  500. postmap:
  501. if (l->l_phdr == 0
  502. && (ElfW(Off)) c->mapoff <= header->e_phoff
  503. && ((size_t) (c->mapend - c->mapstart + c->mapoff)
  504. >= header->e_phoff + header->e_phnum * sizeof (ElfW(Phdr))))
  505. /* Found the program header in this segment. */
  506. l->l_phdr = (void *) (c->mapstart + header->e_phoff - c->mapoff);
  507. if (c->allocend > c->dataend) {
  508. /* Extra zero pages should appear at the end of this segment,
  509. after the data mapped from the file. */
  510. ElfW(Addr) zero, zeroend, zeropage;
  511. zero = (ElfW(Addr)) RELOCATE(l, c->dataend);
  512. zeroend = ALIGN_UP((ElfW(Addr)) RELOCATE(l, c->allocend));
  513. zeropage = ALIGN_UP(zero);
  514. if (zeroend < zeropage)
  515. /* All the extra data is in the last page of the segment.
  516. We can just zero it. */
  517. zeropage = zeroend;
  518. if (type != OBJECT_MAPPED &&
  519. type != OBJECT_INTERNAL &&
  520. type != OBJECT_USER && zeropage > zero) {
  521. /* Zero the final part of the last page of the segment. */
  522. if (__builtin_expect ((c->prot & PROT_WRITE) == 0, 0)) {
  523. /* Dag nab it. */
  524. if (!DkVirtualMemoryProtect((caddr_t) ALIGN_DOWN(zero),
  525. allocsize,
  526. c->prot|PAL_PROT_WRITE)) {
  527. errstring = "cannot change memory protections";
  528. goto call_lose;
  529. }
  530. memset ((void *) zero, '\0', zeropage - zero);
  531. if (!DkVirtualMemoryProtect((caddr_t) ALIGN_DOWN(zero),
  532. allocsize, c->prot)) {
  533. errstring = "cannot change memory protections";
  534. goto call_lose;
  535. }
  536. } else {
  537. memset ((void *) zero, '\0', zeropage - zero);
  538. }
  539. }
  540. if (zeroend > zeropage) {
  541. if (type != OBJECT_MAPPED &&
  542. type != OBJECT_INTERNAL &&
  543. type != OBJECT_USER) {
  544. caddr_t mapat = (caddr_t)
  545. DkVirtualMemoryAlloc((caddr_t) zeropage,
  546. zeroend - zeropage,
  547. 0, c->prot);
  548. if (__builtin_expect (mapat < 0, 0)) {
  549. errstring = "cannot map zero-fill pages";
  550. goto call_lose;
  551. }
  552. }
  553. #if BOOKKEEP_INTERNAL_OBJ == 0
  554. if (type != OBJECT_INTERNAL && type != OBJECT_USER)
  555. #else
  556. if (type != OBJECT_USER)
  557. #endif
  558. bkeep_mmap((void *) zeropage, zeroend - zeropage, c->prot,
  559. MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED|
  560. (type == OBJECT_INTERNAL ? VMA_INTERNAL : 0),
  561. NULL, 0, 0);
  562. }
  563. }
  564. ++c;
  565. }
  566. if (type == OBJECT_REMAP)
  567. return l;
  568. if (l->l_ld == 0) {
  569. if (__builtin_expect(e_type == ET_DYN, 0)) {
  570. errstring = "object file has no dynamic section";
  571. goto call_lose;
  572. }
  573. } else {
  574. l->l_real_ld = (ElfW(Dyn) *) RELOCATE(l, l->l_ld);
  575. l->l_ld = remalloc(l->l_real_ld, sizeof(ElfW(Dyn)) * l->l_ldnum);
  576. }
  577. elf_get_dynamic_info(l);
  578. /* When we profile the SONAME might be needed for something else but
  579. loading. Add it right away. */
  580. if (l->l_info[DT_STRTAB] && l->l_info[DT_SONAME])
  581. l->l_soname = (char *) (D_PTR (l->l_info[DT_STRTAB])
  582. + D_PTR (l->l_info[DT_SONAME]));
  583. if (l->l_phdr == NULL) {
  584. /* The program header is not contained in any of the segments.
  585. We have to allocate memory ourself and copy it over from out
  586. temporary place. */
  587. ElfW(Phdr) * newp = (ElfW(Phdr) *) malloc(header->e_phnum
  588. * sizeof (ElfW(Phdr)));
  589. if (newp == NULL) {
  590. errstring = "cannot allocate memory for program header";
  591. goto call_lose;
  592. }
  593. l->l_phdr = memcpy (newp, phdr,
  594. (header->e_phnum * sizeof (ElfW(Phdr))));
  595. } else {
  596. /* Adjust the PT_PHDR value by the runtime load address. */
  597. l->l_phdr = (ElfW(Phdr) *) RELOCATE(l, l->l_phdr);
  598. }
  599. l->l_entry = RELOCATE(l, l->l_entry);
  600. /* Set up the symbol hash table. */
  601. setup_elf_hash(l);
  602. return l;
  603. }
  604. static inline
  605. struct link_map * __search_map_by_name (const char * name)
  606. {
  607. struct link_map * l = loaded_libraries;
  608. int len = strlen(name);
  609. while (l) {
  610. if (l->l_name && !memcmp(l->l_name, name, len + 1))
  611. break;
  612. l = l->l_next;
  613. }
  614. return l;
  615. }
  616. static inline
  617. struct link_map * __search_map_by_handle (struct shim_handle * file)
  618. {
  619. struct link_map * l = loaded_libraries;
  620. while (l) {
  621. if (l->l_file == file)
  622. break;
  623. l = l->l_next;
  624. }
  625. return l;
  626. }
  627. static inline
  628. struct link_map * __search_map_by_addr (void * addr)
  629. {
  630. struct link_map * l = loaded_libraries;
  631. while (l) {
  632. if ((void *) l->l_map_start == addr)
  633. break;
  634. l = l->l_next;
  635. }
  636. return l;
  637. }
  638. static int __remove_elf_object (struct link_map * l)
  639. {
  640. if (l->l_prev)
  641. l->l_prev->l_next = l->l_next;
  642. if (l->l_next)
  643. l->l_next->l_prev = l->l_prev;
  644. remove_r_debug((void *) l->l_addr);
  645. if (loaded_libraries == l)
  646. loaded_libraries = l->l_next;
  647. if (interp_map == l)
  648. interp_map = NULL;
  649. free(l);
  650. return 0;
  651. }
  652. static int __free_elf_object (struct link_map * l)
  653. {
  654. debug("removing %s as runtime object loaded at %p\n", l->l_name,
  655. l->l_map_start);
  656. struct loadcmd *c = l->loadcmds;
  657. while (c < &l->loadcmds[l->nloadcmds]) {
  658. if (c->mapend > c->mapstart)
  659. /* Unmap the segment contents from the file. */
  660. shim_do_munmap ((void *) l->l_addr + c->mapstart,
  661. c->mapend - c->mapstart);
  662. if (c->allocend > c->dataend) {
  663. /* Extra zero pages should appear at the end of this segment,
  664. after the data mapped from the file. */
  665. ElfW(Addr) zero, zeroend, zeropage;
  666. zero = l->l_addr + c->dataend;
  667. zeroend = l->l_addr + c->allocend;
  668. zeropage = ALIGN_UP(zero);
  669. if (zeroend < zeropage)
  670. /* All the extra data is in the last page of the segment.
  671. We can just zero it. */
  672. zeropage = zeroend;
  673. if (zeroend > zeropage)
  674. shim_do_munmap((void *) zeropage, zeroend - zeropage);
  675. }
  676. ++c;
  677. }
  678. __remove_elf_object(l);
  679. return 0;
  680. }
  681. int free_elf_object (struct shim_handle * file)
  682. {
  683. struct link_map * l = __search_map_by_handle(file);
  684. if (!l)
  685. return -ENOENT;
  686. __free_elf_object(l);
  687. put_handle(file);
  688. return 0;
  689. }
  690. static int __check_elf_header (void * fbp, int len)
  691. {
  692. const char * errstring __attribute__((unused));
  693. /* Now we will start verify the file as a ELF header. This part of code
  694. is borrow from open_verify() */
  695. ElfW(Ehdr) * ehdr = (ElfW(Ehdr) *) fbp;
  696. if (__builtin_expect (len < sizeof(ElfW(Ehdr)), 0)) {
  697. errstring = "ELF file with a strange size";
  698. goto verify_failed;
  699. }
  700. #define ELF32_CLASS ELFCLASS32
  701. #define ELF64_CLASS ELFCLASS64
  702. static const unsigned char expected[EI_NIDENT] = {
  703. [EI_MAG0] = ELFMAG0,
  704. [EI_MAG1] = ELFMAG1,
  705. [EI_MAG2] = ELFMAG2,
  706. [EI_MAG3] = ELFMAG3,
  707. [EI_CLASS] = ELFW(CLASS),
  708. [EI_DATA] = byteorder,
  709. [EI_VERSION] = EV_CURRENT,
  710. [EI_OSABI] = ELFOSABI_SYSV,
  711. [EI_ABIVERSION] = 0
  712. };
  713. /* See whether the ELF header is what we expect. */
  714. if (__builtin_expect (memcmp (ehdr->e_ident, expected, EI_ABIVERSION) !=
  715. 0, 0)) {
  716. errstring = "ELF file with invalid header";
  717. goto verify_failed;
  718. }
  719. /* Check whether the ELF header use the right endian */
  720. if (ehdr->e_ident[EI_DATA] != byteorder) {
  721. if (BYTE_ORDER == BIG_ENDIAN) {
  722. errstring = "ELF file data encoding not big-endian";
  723. goto verify_failed;
  724. } else {
  725. errstring = "ELF file data encoding not little-endian";
  726. goto verify_failed;
  727. }
  728. }
  729. /* checking the header is of the right version */
  730. if (ehdr->e_ident[EI_VERSION] != EV_CURRENT) {
  731. errstring = "ELF file version ident does not match current one";
  732. goto verify_failed;
  733. }
  734. if (memcmp(&ehdr->e_ident[EI_PAD], &expected[EI_PAD],
  735. EI_NIDENT - EI_PAD) != 0) {
  736. errstring = "nonzero padding in e_ident";
  737. goto verify_failed;
  738. }
  739. if (__builtin_expect (ehdr->e_version, EV_CURRENT) != EV_CURRENT) {
  740. errstring = "ELF file version does not match current one";
  741. goto verify_failed;
  742. }
  743. /* Now we check if the host match the elf machine profile */
  744. if (! __builtin_expect (elf_machine_matches_host (ehdr), 1)) {
  745. errstring = "ELF file does not match with the host";
  746. goto verify_failed;
  747. }
  748. /* check if the type of ELF header is either DYN or EXEC */
  749. if (__builtin_expect (ehdr->e_type, ET_DYN) != ET_DYN
  750. && __builtin_expect (ehdr->e_type, ET_EXEC) != ET_EXEC) {
  751. errstring = "only ET_DYN and ET_EXEC can be loaded\n";
  752. goto verify_failed;
  753. }
  754. /* check if phentsize match the size of ElfW(Phdr) */
  755. if (__builtin_expect (ehdr->e_phentsize, sizeof (ElfW(Phdr)))
  756. != sizeof (ElfW(Phdr))) {
  757. errstring = "ELF file's phentsize not the expected size";
  758. goto verify_failed;
  759. }
  760. return 0;
  761. verify_failed:
  762. debug("load runtime object: %s\n", errstring);
  763. return -EINVAL;
  764. }
  765. static int __read_elf_header (struct shim_handle * file, void * fbp)
  766. {
  767. if (!file)
  768. return -EINVAL;
  769. if (!file->fs || !file->fs->fs_ops)
  770. return -EACCES;
  771. int (*read) (struct shim_handle *, void *, size_t) =
  772. file->fs->fs_ops->read;
  773. int (*seek) (struct shim_handle *, off_t, int) =
  774. file->fs->fs_ops->seek;
  775. if (!read || !seek)
  776. return -EACCES;
  777. (*seek) (file, 0, SEEK_SET);
  778. int ret = (*read) (file, fbp, FILEBUF_SIZE);
  779. (*seek) (file, 0, SEEK_SET);
  780. return ret;
  781. }
  782. static int __load_elf_header (struct shim_handle * file, void * fbp,
  783. int * plen)
  784. {
  785. int len = __read_elf_header(file, fbp);
  786. if (len < 0)
  787. return len;
  788. int ret = __check_elf_header(fbp, len);
  789. if (ret < 0)
  790. return ret;
  791. if (plen)
  792. *plen = len;
  793. return 0;
  794. }
  795. int check_elf_object (struct shim_handle * file)
  796. {
  797. char fb[FILEBUF_SIZE];
  798. int l = __read_elf_header(file, &fb);
  799. if (l < 0)
  800. return l;
  801. return __check_elf_header(&fb, l);
  802. }
  803. static int __load_elf_object (struct shim_handle * file, void * addr,
  804. int type, struct link_map * remap);
  805. int load_elf_object (struct shim_handle * file, void * addr, size_t mapped)
  806. {
  807. if (!file)
  808. return -EINVAL;
  809. if (mapped)
  810. debug("adding %s as runtime object loaded at %p-%p\n",
  811. file ? qstrgetstr(&file->uri) : "(unknown)", addr, addr + mapped);
  812. else
  813. debug("loading %s as runtime object at %p\n",
  814. file ? qstrgetstr(&file->uri) : "(unknown)", addr);
  815. return __load_elf_object(file, addr,
  816. mapped ? OBJECT_MAPPED : OBJECT_LOAD,
  817. NULL);
  818. }
  819. static void add_link_map (struct link_map * map)
  820. {
  821. struct link_map *prev = NULL;
  822. struct link_map **pprev = &loaded_libraries;
  823. struct link_map *next = loaded_libraries;
  824. while (next) {
  825. prev = next;
  826. pprev = &next->l_next;
  827. next = next->l_next;
  828. }
  829. *pprev = map;
  830. map->l_prev = prev;
  831. map->l_next = NULL;
  832. }
  833. static void replace_link_map (struct link_map * new, struct link_map * old)
  834. {
  835. new->l_next = old->l_next;
  836. new->l_prev = old->l_prev;
  837. if (old->l_next)
  838. old->l_next->l_prev = new;
  839. if (old->l_prev)
  840. old->l_prev->l_next = new;
  841. if (loaded_libraries == old)
  842. loaded_libraries = new;
  843. }
  844. static int do_relocate_object (struct link_map * l);
  845. static int __load_elf_object (struct shim_handle * file, void * addr,
  846. int type, struct link_map * remap)
  847. {
  848. char * hdr = addr;
  849. int len, ret = 0;
  850. if (type == OBJECT_LOAD || type == OBJECT_REMAP) {
  851. hdr = __alloca(FILEBUF_SIZE);
  852. if ((ret = __load_elf_header(file, hdr, &len)) < 0)
  853. goto out;
  854. }
  855. struct link_map * map = __map_elf_object(file, hdr, len, addr, type, remap);
  856. if (!map) {
  857. ret = -EINVAL;
  858. goto out;
  859. }
  860. if (type != OBJECT_INTERNAL)
  861. do_relocate_object(map);
  862. if (internal_map) {
  863. map->l_resolved = true;
  864. map->l_resolved_map = internal_map->l_addr;
  865. }
  866. if (type == OBJECT_INTERNAL)
  867. internal_map = map;
  868. if (type != OBJECT_REMAP) {
  869. if (file) {
  870. get_handle(file);
  871. map->l_file = file;
  872. }
  873. add_link_map(map);
  874. }
  875. if ((type == OBJECT_LOAD || type == OBJECT_REMAP || type == OBJECT_USER) &&
  876. map->l_file && !qstrempty(&map->l_file->uri)) {
  877. if (type == OBJECT_REMAP)
  878. remove_r_debug((void *) map->l_addr);
  879. append_r_debug(qstrgetstr(&map->l_file->uri), (void *) map->l_map_start,
  880. (void *) map->l_real_ld);
  881. }
  882. out:
  883. return ret;
  884. }
  885. int reload_elf_object (struct shim_handle * file)
  886. {
  887. struct link_map * map = loaded_libraries;
  888. while (map) {
  889. if (map->l_file == file)
  890. break;
  891. map = map->l_next;
  892. }
  893. if (!map)
  894. return -ENOENT;
  895. debug("reloading %s as runtime object loaded at %p-%p\n",
  896. qstrgetstr(&file->uri), map->l_map_start, map->l_map_end);
  897. return __load_elf_object(file, NULL, OBJECT_REMAP, map);
  898. }
  899. struct sym_val {
  900. ElfW(Sym) * s;
  901. struct link_map * m;
  902. };
  903. static uint_fast32_t elf_fast_hash (const char *s)
  904. {
  905. uint_fast32_t h = 5381;
  906. for (unsigned char c = *s; c != '\0'; c = *++s)
  907. h = h * 33 + c;
  908. return h & 0xffffffff;
  909. }
  910. /* This is the hashing function specified by the ELF ABI. In the
  911. first five operations no overflow is possible so we optimized it a
  912. bit. */
  913. static unsigned long int
  914. elf_hash (const char * name_arg)
  915. {
  916. const unsigned char * name = (const unsigned char *) name_arg;
  917. unsigned long int hash = 0;
  918. if (*name == '\0')
  919. return hash;
  920. hash = *name++;
  921. if (*name == '\0')
  922. return hash;
  923. hash = (hash << 4) + *name++;
  924. if (*name == '\0')
  925. return hash;
  926. hash = (hash << 4) + *name++;
  927. if (*name == '\0')
  928. return hash;
  929. hash = (hash << 4) + *name++;
  930. if (*name == '\0')
  931. return hash;
  932. hash = (hash << 4) + *name++;
  933. while (*name != '\0') {
  934. unsigned long int hi;
  935. hash = (hash << 4) + *name++;
  936. hi = hash & 0xf0000000;
  937. /* The algorithm specified in the ELF ABI is as follows:
  938. if (hi != 0)
  939. hash ^= hi >> 24;
  940. hash &= ~hi;
  941. But the following is equivalent and a lot faster, especially on
  942. modern processors. */
  943. hash ^= hi;
  944. hash ^= hi >> 24;
  945. }
  946. return hash;
  947. }
  948. static ElfW(Sym) *
  949. do_lookup_map (ElfW(Sym) * ref, const char * undef_name,
  950. const uint_fast32_t hash, unsigned long int elf_hash,
  951. const struct link_map * map)
  952. {
  953. /* These variables are used in the nested function. */
  954. Elf_Symndx symidx;
  955. ElfW(Sym) * sym;
  956. /* The tables for this map. */
  957. ElfW(Sym) * symtab = (void *) D_PTR (map->l_info[DT_SYMTAB]);
  958. const char * strtab = (const void *) D_PTR (map->l_info[DT_STRTAB]);
  959. int len = strlen(undef_name);
  960. /* Nested routine to check whether the symbol matches. */
  961. ElfW(Sym) * check_match (ElfW(Sym) * sym)
  962. {
  963. unsigned int stt = ELFW(ST_TYPE) (sym->st_info);
  964. if (__builtin_expect ((sym->st_value == 0 /* No value. */
  965. && stt != STT_TLS)
  966. || sym->st_shndx == SHN_UNDEF, 0))
  967. return NULL;
  968. /* Ignore all but STT_NOTYPE, STT_OBJECT, STT_FUNC,
  969. STT_COMMON, STT_TLS, and STT_GNU_IFUNC since these are no
  970. code/data definitions. */
  971. #define ALLOWED_STT \
  972. ((1 << STT_NOTYPE) | (1 << STT_OBJECT) | (1 << STT_FUNC) \
  973. | (1 << STT_COMMON) | (1 << STT_TLS) | (1 << STT_GNU_IFUNC))
  974. if (__builtin_expect (((1 << stt) & ALLOWED_STT) == 0, 0))
  975. return NULL;
  976. if (sym != ref && memcmp(strtab + sym->st_name, undef_name, len + 1))
  977. /* Not the symbol we are looking for. */
  978. return NULL;
  979. /* There cannot be another entry for this symbol so stop here. */
  980. return sym;
  981. }
  982. const ElfW(Addr) * bitmask = map->l_gnu_bitmask;
  983. if (__builtin_expect (bitmask != NULL, 1)) {
  984. ElfW(Addr) bitmask_word = bitmask[(hash / __ELF_NATIVE_CLASS)
  985. & map->l_gnu_bitmask_idxbits];
  986. unsigned int hashbit1 = hash & (__ELF_NATIVE_CLASS - 1);
  987. unsigned int hashbit2 = (hash >> map->l_gnu_shift)
  988. & (__ELF_NATIVE_CLASS - 1);
  989. if (__builtin_expect ((bitmask_word >> hashbit1)
  990. & (bitmask_word >> hashbit2) & 1, 0)) {
  991. Elf32_Word bucket = map->l_gnu_buckets
  992. [hash % map->l_nbuckets];
  993. if (bucket != 0) {
  994. const Elf32_Word *hasharr = &map->l_gnu_chain_zero[bucket];
  995. do {
  996. if (((*hasharr ^ hash) >> 1) == 0) {
  997. symidx = hasharr - map->l_gnu_chain_zero;
  998. sym = check_match (&symtab[symidx]);
  999. if (sym != NULL)
  1000. return sym;
  1001. }
  1002. } while ((*hasharr++ & 1u) == 0);
  1003. }
  1004. }
  1005. /* No symbol found. */
  1006. symidx = SHN_UNDEF;
  1007. } else {
  1008. /* Use the old SysV-style hash table. Search the appropriate
  1009. hash bucket in this object's symbol table for a definition
  1010. for the same symbol name. */
  1011. for (symidx = map->l_buckets[elf_hash % map->l_nbuckets];
  1012. symidx != STN_UNDEF;
  1013. symidx = map->l_chain[symidx]) {
  1014. sym = check_match (&symtab[symidx]);
  1015. if (sym != NULL)
  1016. return sym;
  1017. }
  1018. }
  1019. return NULL;
  1020. }
  1021. /* Inner part of the lookup functions. We return a value > 0 if we
  1022. found the symbol, the value 0 if nothing is found and < 0 if
  1023. something bad happened. */
  1024. static int do_lookup (const char * undef_name, ElfW(Sym) * ref,
  1025. struct sym_val * result)
  1026. {
  1027. const uint_fast32_t fast_hash = elf_fast_hash(undef_name);
  1028. const long int hash = elf_hash(undef_name);
  1029. ElfW(Sym) *sym = NULL;
  1030. sym = do_lookup_map(ref, undef_name, fast_hash, hash, internal_map);
  1031. if (!sym)
  1032. return 0;;
  1033. switch (__builtin_expect (ELFW(ST_BIND) (sym->st_info), STB_GLOBAL)) {
  1034. case STB_WEAK:
  1035. /* Weak definition. Use this value if we don't find another. */
  1036. if (! result->s) {
  1037. result->s = sym;
  1038. result->m = (struct link_map *) internal_map;
  1039. }
  1040. break;
  1041. /* FALLTHROUGH */
  1042. case STB_GLOBAL:
  1043. case STB_GNU_UNIQUE:
  1044. /* success: */
  1045. /* Global definition. Just what we need. */
  1046. result->s = sym;
  1047. result->m = (struct link_map *) internal_map;
  1048. return 1;
  1049. default:
  1050. /* Local symbols are ignored. */
  1051. break;
  1052. }
  1053. /* We have not found anything until now. */
  1054. return 0;
  1055. }
  1056. /* Search loaded objects' symbol tables for a definition of the symbol
  1057. UNDEF_NAME, perhaps with a requested version for the symbol.
  1058. We must never have calls to the audit functions inside this function
  1059. or in any function which gets called. If this would happen the audit
  1060. code might create a thread which can throw off all the scope locking. */
  1061. struct link_map *
  1062. lookup_symbol (const char * undef_name, ElfW(Sym) ** ref)
  1063. {
  1064. struct sym_val current_value = { NULL, NULL };
  1065. do_lookup(undef_name, *ref, &current_value);
  1066. if (__builtin_expect (current_value.s == NULL, 0)) {
  1067. *ref = NULL;
  1068. return NULL;
  1069. }
  1070. *ref = current_value.s;
  1071. return current_value.m;
  1072. }
  1073. static int do_relocate_object (struct link_map * l)
  1074. {
  1075. int ret = 0;
  1076. if (l->l_resolved)
  1077. ELF_REDO_DYNAMIC_RELOCATE(l);
  1078. else
  1079. ELF_DYNAMIC_RELOCATE(l);
  1080. if ((ret = reprotect_map(l)) < 0)
  1081. return ret;
  1082. return 0;
  1083. }
  1084. static bool __need_interp (struct link_map * exec_map)
  1085. {
  1086. if (!exec_map->l_interp_libname)
  1087. return false;
  1088. const char * strtab = (const void *) D_PTR (exec_map->l_info[DT_STRTAB]);
  1089. const ElfW(Dyn) * d;
  1090. for (d = exec_map->l_ld ; d->d_tag != DT_NULL ; d++)
  1091. if (__builtin_expect (d->d_tag, DT_NEEDED) == DT_NEEDED) {
  1092. const char * name = strtab + d->d_un.d_val;
  1093. int len = strlen(name);
  1094. const char * filename = name + len - 1;
  1095. while (filename > name && *filename != '/')
  1096. filename--;
  1097. if (*filename == '/')
  1098. filename++;
  1099. /* if we find a dependency besides libsysdb.so, the
  1100. interpreter is necessary */
  1101. if (memcmp(filename, "libsysdb", 8))
  1102. return true;
  1103. }
  1104. return false;
  1105. }
  1106. extern const char ** library_paths;
  1107. int free_elf_interp (void)
  1108. {
  1109. if (interp_map)
  1110. __free_elf_object(interp_map);
  1111. return 0;
  1112. }
  1113. static int __load_interp_object (struct link_map * exec_map)
  1114. {
  1115. const char * interp_name = (const char *) exec_map->l_interp_libname +
  1116. (long) exec_map->l_addr;
  1117. int len = strlen(interp_name);
  1118. const char * filename = interp_name + len - 1;
  1119. while (filename > interp_name && *filename != '/')
  1120. filename--;
  1121. if (*filename == '/')
  1122. filename++;
  1123. len -= filename - interp_name;
  1124. const char * default_paths[] = { "/lib", "/lib64", NULL };
  1125. const char ** paths = library_paths ? : default_paths;
  1126. char interp_path[STR_SIZE];
  1127. for (const char ** p = paths ; *p ; p++) {
  1128. int plen = strlen(*p);
  1129. memcpy(interp_path, *p, plen);
  1130. interp_path[plen] = '/';
  1131. memcpy(interp_path + plen + 1, filename, len + 1);
  1132. debug("search interpreter: %s\n", interp_path);
  1133. struct shim_dentry * dent = NULL;
  1134. int ret = 0;
  1135. if ((ret = path_lookupat(NULL, interp_path, LOOKUP_OPEN, &dent, NULL)) < 0 ||
  1136. dent->state & DENTRY_NEGATIVE)
  1137. continue;
  1138. struct shim_mount * fs = dent->fs;
  1139. get_dentry(dent);
  1140. if (!fs->d_ops->open) {
  1141. ret = -EACCES;
  1142. err:
  1143. put_dentry(dent);
  1144. return ret;
  1145. }
  1146. if (fs->d_ops->mode) {
  1147. mode_t mode;
  1148. if ((ret = fs->d_ops->mode(dent, &mode, 1)) < 0)
  1149. goto err;
  1150. }
  1151. struct shim_handle * interp = NULL;
  1152. if (!(interp = get_new_handle())) {
  1153. ret = -ENOMEM;
  1154. goto err;
  1155. }
  1156. set_handle_fs(interp, fs);
  1157. interp->flags = O_RDONLY;
  1158. interp->acc_mode = MAY_READ;
  1159. if ((ret = fs->d_ops->open(interp, dent, O_RDONLY)) < 0) {
  1160. put_handle(interp);
  1161. goto err;
  1162. }
  1163. if (!(ret = __load_elf_object(interp, NULL, OBJECT_LOAD, NULL)))
  1164. interp_map = __search_map_by_handle(interp);
  1165. put_handle(interp);
  1166. return ret;
  1167. }
  1168. return -ENOENT;
  1169. }
  1170. int load_elf_interp (struct shim_handle * exec)
  1171. {
  1172. struct link_map * exec_map = __search_map_by_handle(exec);
  1173. if (exec_map && !interp_map &&
  1174. __need_interp(exec_map))
  1175. __load_interp_object(exec_map);
  1176. return 0;
  1177. }
  1178. int remove_loaded_libraries (void)
  1179. {
  1180. struct link_map * map = loaded_libraries, * next_map = map->l_next;
  1181. while (map) {
  1182. if (map->l_type != OBJECT_INTERNAL)
  1183. __remove_elf_object(map);
  1184. map = next_map;
  1185. next_map = map ? map->l_next : NULL;
  1186. }
  1187. return 0;
  1188. }
  1189. void * __load_address;
  1190. void * migrated_shim_addr __attribute_migratable = &__load_address;
  1191. int init_internal_map (void)
  1192. {
  1193. __load_elf_object(NULL, &__load_address, OBJECT_INTERNAL, NULL);
  1194. internal_map->l_name = "libsysdb.so";
  1195. return 0;
  1196. }
  1197. int init_brk_from_executable (struct shim_handle * exec);
  1198. int init_loader (void)
  1199. {
  1200. struct shim_thread * cur_thread = get_cur_thread();
  1201. int ret = 0;
  1202. lock(cur_thread->lock);
  1203. struct shim_handle * exec = cur_thread->exec;
  1204. if (exec)
  1205. get_handle(exec);
  1206. unlock(cur_thread->lock);
  1207. if (!exec)
  1208. return 0;
  1209. struct link_map * exec_map = __search_map_by_handle(exec);
  1210. if (!exec_map) {
  1211. if ((ret = load_elf_object(exec,
  1212. (void *) PAL_CB(executable_range.start),
  1213. PAL_CB(executable_range.end) -
  1214. PAL_CB(executable_range.start))) < 0)
  1215. goto out;
  1216. exec_map = __search_map_by_handle(exec);
  1217. }
  1218. init_brk_from_executable(exec);
  1219. if (!interp_map
  1220. && __need_interp(exec_map)
  1221. && (ret = __load_interp_object(exec_map)) < 0)
  1222. goto out;
  1223. ret = 0;
  1224. out:
  1225. put_handle(exec);
  1226. return ret;
  1227. }
  1228. int init_brk_from_executable (struct shim_handle * exec)
  1229. {
  1230. struct link_map * exec_map = __search_map_by_handle(exec);
  1231. if (exec_map) {
  1232. /*
  1233. * Chia-Che 8/24/2017:
  1234. * initialize brk region at the end of the executable data segment.
  1235. */
  1236. init_brk_region((void *) ALIGN_UP(exec_map->l_map_end));
  1237. }
  1238. return 0;
  1239. }
  1240. int register_library (const char * name, unsigned long load_address)
  1241. {
  1242. debug("glibc register library %s loaded at %p\n",
  1243. name, load_address);
  1244. struct shim_handle * hdl = get_new_handle();
  1245. if (!hdl)
  1246. return -ENOMEM;
  1247. int err = open_namei(hdl, NULL, name, O_RDONLY, 0, NULL);
  1248. if (err < 0) {
  1249. put_handle(hdl);
  1250. return err;
  1251. }
  1252. __load_elf_object(hdl, (void *) load_address, OBJECT_USER, NULL);
  1253. put_handle(hdl);
  1254. return 0;
  1255. }
  1256. int execute_elf_object (struct shim_handle * exec, int argc, const char ** argp,
  1257. int nauxv, ElfW(auxv_t) * auxp)
  1258. {
  1259. struct link_map * exec_map = __search_map_by_handle(exec);
  1260. assert(exec_map);
  1261. auxp[0].a_type = AT_PHDR;
  1262. auxp[0].a_un.a_val = (__typeof(auxp[0].a_un.a_val)) exec_map->l_phdr;
  1263. auxp[1].a_type = AT_PHNUM;
  1264. auxp[1].a_un.a_val = exec_map->l_phnum;
  1265. auxp[2].a_type = AT_PAGESZ;
  1266. auxp[2].a_un.a_val = allocsize;
  1267. auxp[3].a_type = AT_ENTRY;
  1268. auxp[3].a_un.a_val = exec_map->l_entry;
  1269. auxp[4].a_type = AT_BASE;
  1270. auxp[4].a_un.a_val = interp_map ? interp_map->l_addr : 0;
  1271. auxp[5].a_type = AT_NULL;
  1272. ElfW(Addr) entry = interp_map ? interp_map->l_entry : exec_map->l_entry;
  1273. #if defined(__x86_64__)
  1274. asm volatile (
  1275. "movq %%rbx, %%rsp\r\n"
  1276. "pushq %%rdi\r\n"
  1277. "jmp *%%rax\r\n"
  1278. :
  1279. : "a"(entry),
  1280. "b"(argp),
  1281. "D"(argc)
  1282. : "memory");
  1283. #else
  1284. # error "architecture not supported"
  1285. #endif
  1286. shim_do_exit(0);
  1287. return 0;
  1288. }
  1289. BEGIN_CP_FUNC(library)
  1290. {
  1291. assert(size == sizeof(struct link_map));
  1292. struct link_map * map = (struct link_map *) obj;
  1293. struct link_map * new_map;
  1294. ptr_t off = GET_FROM_CP_MAP(obj);
  1295. if (!off) {
  1296. off = ADD_CP_OFFSET(sizeof(struct link_map));
  1297. ADD_TO_CP_MAP(obj, off);
  1298. new_map = (struct link_map *) (base + off);
  1299. memcpy(new_map, map, sizeof(struct link_map));
  1300. new_map->l_prev = NULL;
  1301. new_map->l_next = NULL;
  1302. new_map->textrels = NULL;
  1303. if (map->l_file)
  1304. DO_CP_MEMBER(handle, map, new_map, l_file);
  1305. if (map->l_ld) {
  1306. int size = sizeof(ElfW(Dyn)) * map->l_ldnum;
  1307. ElfW(Dyn) * ld = (void *) (base + ADD_CP_OFFSET(size));
  1308. memcpy(ld, map->l_ld, size);
  1309. new_map->l_ld = ld;
  1310. ElfW(Dyn) ** start = new_map->l_info;
  1311. ElfW(Dyn) ** end = (void *) start + sizeof(new_map->l_info);
  1312. ElfW(Dyn) ** dyn;
  1313. for (dyn = start ; dyn < end ; dyn++)
  1314. if (*dyn)
  1315. *dyn = (void *) *dyn + ((void *) ld - (void *) map->l_ld);
  1316. }
  1317. if (map->l_name) {
  1318. int namelen = strlen(map->l_name);
  1319. char * name = (char *) (base + ADD_CP_OFFSET(namelen + 1));
  1320. memcpy(name, map->l_name, namelen + 1);
  1321. new_map->l_name = name;
  1322. }
  1323. if (map->l_soname) {
  1324. int sonamelen = strlen(map->l_soname);
  1325. char * soname = (char *) (base + ADD_CP_OFFSET(sonamelen + 1));
  1326. memcpy(soname, map->l_soname, sonamelen + 1);
  1327. new_map->l_soname = soname;
  1328. }
  1329. ADD_CP_FUNC_ENTRY(off);
  1330. } else {
  1331. new_map = (struct link_map *) (base + off);
  1332. }
  1333. if (objp)
  1334. *objp = (void *) new_map;
  1335. }
  1336. END_CP_FUNC(library)
  1337. DEFINE_PROFILE_CATAGORY(inside_rs_library, resume_func);
  1338. DEFINE_PROFILE_INTERVAL(clean_up_library, inside_rs_library);
  1339. DEFINE_PROFILE_INTERVAL(search_library_vma, inside_rs_library);
  1340. DEFINE_PROFILE_INTERVAL(relocate_library, inside_rs_library);
  1341. DEFINE_PROFILE_INTERVAL(add_or_replace_library, inside_rs_library);
  1342. BEGIN_RS_FUNC(library)
  1343. {
  1344. struct link_map * map = (void *) (base + GET_CP_FUNC_ENTRY());
  1345. CP_REBASE(map->l_name);
  1346. CP_REBASE(map->l_soname);
  1347. CP_REBASE(map->l_file);
  1348. if (map->l_ld && map->l_ld != map->l_real_ld) {
  1349. CP_REBASE(map->l_ld);
  1350. CP_REBASE(map->l_info);
  1351. }
  1352. BEGIN_PROFILE_INTERVAL();
  1353. struct link_map * old_map = __search_map_by_name(map->l_name);
  1354. if (old_map)
  1355. remove_r_debug((void *) old_map->l_addr);
  1356. SAVE_PROFILE_INTERVAL(clean_up_library);
  1357. if (internal_map && (!map->l_resolved ||
  1358. map->l_resolved_map != internal_map->l_addr)) {
  1359. do_relocate_object(map);
  1360. SAVE_PROFILE_INTERVAL(relocate_library);
  1361. }
  1362. if (old_map)
  1363. replace_link_map(map, old_map);
  1364. else
  1365. add_link_map(map);
  1366. SAVE_PROFILE_INTERVAL(add_or_replace_library);
  1367. DEBUG_RS("base=%p,name=%s", map->l_addr, map->l_name);
  1368. }
  1369. END_RS_FUNC(library)
  1370. BEGIN_CP_FUNC(loaded_libraries)
  1371. {
  1372. struct link_map * map = loaded_libraries, * new_interp_map = NULL;
  1373. while (map) {
  1374. struct link_map * new_map = NULL;
  1375. if (map != internal_map)
  1376. DO_CP(library, map, &new_map);
  1377. if (map == interp_map)
  1378. new_interp_map = new_map;
  1379. map = map->l_next;
  1380. }
  1381. ADD_CP_FUNC_ENTRY(new_interp_map);
  1382. }
  1383. END_CP_FUNC(loaded_libraries)
  1384. BEGIN_RS_FUNC(loaded_libraries)
  1385. {
  1386. interp_map = (void *) GET_CP_FUNC_ENTRY();
  1387. if (interp_map) {
  1388. CP_REBASE(interp_map);
  1389. DEBUG_RS("%s as interp", interp_map->l_name);
  1390. }
  1391. }
  1392. END_RS_FUNC(loaded_libraries)