graphene-ipc.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047
  1. #include <linux/module.h>
  2. #include <linux/kallsyms.h>
  3. #include <linux/version.h>
  4. #include <linux/init.h>
  5. #include <linux/fs.h>
  6. #include <linux/mm_types.h>
  7. #include <linux/mm.h>
  8. #include <linux/mmu_notifier.h>
  9. #include <linux/slab.h>
  10. #include <linux/swap.h>
  11. #include <linux/swapops.h>
  12. #include <linux/miscdevice.h>
  13. #include <linux/sched.h>
  14. #include <linux/pagemap.h>
  15. #include <linux/bitmap.h>
  16. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  17. # include <linux/sched/signal.h>
  18. #endif
  19. #include <asm/mman.h>
  20. #include <asm/tlb.h>
  21. #include "graphene-ipc.h"
  22. #include "ksyms.h"
  23. MODULE_LICENSE("Dual BSD/GPL");
  24. #define FILE_POISON LIST_POISON1
  25. struct kmem_cache *gipc_queue_cachep;
  26. struct kmem_cache *gipc_send_buffer_cachep;
  27. #define GIPC_DEBUG 0
  28. #if defined(GIPC_DEBUG) && GIPC_DEBUG == 1
  29. # define DEBUG(...) printk(KERN_INFO __VA_ARGS__)
  30. # define GIPC_BUG_ON(cond) BUG_ON(cond)
  31. #else
  32. # define DEBUG(...)
  33. # define GIPC_BUG_ON(cond)
  34. #endif
  35. #if defined(CONFIG_GRAPHENE_BULK_IPC) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
  36. # if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
  37. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  38. ({ \
  39. unsigned long populate; \
  40. unsigned long rv = do_mmap_pgoff(file, addr, len, \
  41. prot, flags, \
  42. pgoff, &populate); \
  43. rv; })
  44. # else
  45. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  46. do_mmap_pgoff(file, addr, len, prot, flags, pgoff)
  47. # endif /* kernel_version < 3.9.0 */
  48. #else
  49. # if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  50. # define MY_DO_MMAP
  51. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  52. ({ \
  53. unsigned long populate; \
  54. unsigned long rv; \
  55. rv = KSYM(do_mmap)(file, addr, len, \
  56. prot, flags, 0, pgoff, \
  57. &populate, NULL); \
  58. rv; })
  59. # elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
  60. # define MY_DO_MMAP
  61. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  62. ({ \
  63. unsigned long populate; \
  64. unsigned long rv; \
  65. rv = KSYM(do_mmap)(file, addr, len, \
  66. prot, flags, 0, pgoff, \
  67. &populate); \
  68. rv; })
  69. # elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
  70. # define MY_DO_MMAP_PGOFF
  71. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  72. ({ \
  73. unsigned long populate; \
  74. unsigned long rv; \
  75. rv = KSYM(do_mmap_pgoff)(file, addr, len, \
  76. prot, flags, pgoff, \
  77. &populate); \
  78. rv; })
  79. # else
  80. # define MY_DO_MMAP_PGOFF
  81. # define DO_MMAP_PGOFF(file, addr, len, prot, flags, pgoff) \
  82. KSYM(do_mmap_pgoff)(file, addr, len, prot, flags, pgoff)
  83. # endif /* kernel version < 3.9 */
  84. #endif /* !CONFIG_GRAPHENE_BULK_IPC && kernel version > 3.4.0 */
  85. #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
  86. # ifdef CONFIG_GRAPHENE_BULK_IPC
  87. # define FLUSH_TLB_MM_RANGE flush_tlb_mm_range
  88. # else
  89. # define MY_FLUSH_TLB_MM_RANGE
  90. # define FLUSH_TLB_MM_RANGE KSYM(flush_tlb_mm_range)
  91. # endif
  92. #else /* LINUX_VERSION_CODE < 3.7.0 */
  93. # if defined(CONFIG_GRAPHENE_BULK_IPC) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 2, 0)
  94. # define FLUSH_TLB_PAGE flush_tlb_page
  95. # else
  96. # define MY_FLUSH_TLB_PAGE
  97. # define FLUSH_TLB_PAGE KSYM(flush_tlb_page)
  98. # endif
  99. #endif
  100. #ifdef MY_DO_MMAP
  101. IMPORT_KSYM(do_mmap);
  102. #endif
  103. #ifdef MY_DO_MMAP_PGOFF
  104. IMPORT_KSYM(do_mmap_pgoff);
  105. #endif
  106. #ifdef MY_FLUSH_TLB_MM_RANGE
  107. IMPORT_KSYM(flush_tlb_mm_range);
  108. #endif
  109. #ifdef MY_FLUSH_TLB_PAGE
  110. IMPORT_KSYM(flush_tlb_page);
  111. #endif
  112. #ifndef gipc_get_session
  113. u64 (*my_gipc_get_session) (struct task_struct *) = NULL;
  114. #endif
  115. struct gipc_queue {
  116. struct list_head list;
  117. s64 token;
  118. u64 owner;
  119. atomic_t count;
  120. struct mutex send_lock, recv_lock;
  121. wait_queue_head_t send, recv;
  122. volatile int next, last;
  123. struct {
  124. struct page *page;
  125. struct file *file;
  126. u64 pgoff;
  127. } pages[PAGE_QUEUE];
  128. };
  129. struct gipc_send_buffer {
  130. unsigned long page_bit_map[PAGE_BITS];
  131. struct page *pages[PAGE_QUEUE];
  132. struct vm_area_struct *vmas[PAGE_QUEUE];
  133. struct file *files[PAGE_QUEUE];
  134. unsigned long pgoffs[PAGE_QUEUE];
  135. };
  136. struct {
  137. spinlock_t lock;
  138. /*
  139. * For now, just make them monotonically increasing. XXX: At
  140. * some point, do something smarter for security.
  141. */
  142. u64 max_token;
  143. struct list_head channels; // gipc_queue structs
  144. } gdev;
  145. #ifdef gipc_get_session
  146. #define GIPC_OWNER gipc_get_session(current)
  147. #else
  148. #define GIPC_OWNER (my_gipc_get_session ? my_gipc_get_session(current) : 0)
  149. #endif
  150. static inline struct gipc_queue * create_gipc_queue(struct file *creator)
  151. {
  152. struct gipc_queue *gq = kmem_cache_alloc(gipc_queue_cachep, GFP_KERNEL);
  153. if (!gq)
  154. return gq;
  155. memset(gq, 0, sizeof(*gq));
  156. INIT_LIST_HEAD(&gq->list);
  157. mutex_init(&gq->send_lock);
  158. mutex_init(&gq->recv_lock);
  159. init_waitqueue_head(&gq->send);
  160. init_waitqueue_head(&gq->recv);
  161. gq->owner = GIPC_OWNER;
  162. creator->private_data = gq;
  163. atomic_set(&gq->count, 1);
  164. spin_lock(&gdev.lock);
  165. list_add(&gq->list, &gdev.channels);
  166. gq->token = gdev.max_token++;
  167. spin_unlock(&gdev.lock);
  168. return gq;
  169. }
  170. static inline void release_gipc_queue(struct gipc_queue *gq, bool locked)
  171. {
  172. int idx;
  173. if (!atomic_dec_and_test(&gq->count))
  174. return;
  175. if (!locked)
  176. spin_lock(&gdev.lock);
  177. while (gq->next != gq->last) {
  178. idx = gq->next;
  179. if (gq->pages[idx].page) {
  180. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
  181. put_page(gq->pages[idx].page);
  182. #else
  183. page_cache_release(gq->pages[idx].page);
  184. #endif
  185. gq->pages[idx].page = NULL;
  186. }
  187. if (gq->pages[idx].file) {
  188. fput_atomic(gq->pages[idx].file);
  189. gq->pages[idx].file = NULL;
  190. gq->pages[idx].pgoff = 0;
  191. }
  192. gq->next++;
  193. gq->next &= (PAGE_QUEUE - 1);
  194. }
  195. list_del(&gq->list);
  196. if (!locked)
  197. spin_unlock(&gdev.lock);
  198. kmem_cache_free(gipc_queue_cachep, gq);
  199. }
  200. #if defined(SPLIT_RSS_COUNTING)
  201. static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
  202. {
  203. struct task_struct *task = current;
  204. if (likely(task->mm == mm))
  205. task->rss_stat.count[member] += val;
  206. else
  207. add_mm_counter(mm, member, val);
  208. }
  209. #else
  210. #define add_mm_counter_fast(mm, member, val) add_mm_counter(mm, member, val)
  211. #endif
  212. #define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
  213. #define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
  214. inline int make_page_cow(struct mm_struct *mm, struct vm_area_struct *vma,
  215. unsigned long addr)
  216. {
  217. pgd_t *pgd;
  218. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  219. p4d_t *p4d;
  220. #endif
  221. pud_t *pud;
  222. pmd_t *pmd;
  223. pte_t *pte;
  224. spinlock_t *ptl;
  225. pgd = pgd_offset(mm, addr);
  226. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  227. goto no_page;
  228. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  229. p4d = p4d_offset(pgd, addr);
  230. if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
  231. goto no_page;
  232. pud = pud_offset(p4d, addr);
  233. #else
  234. pud = pud_offset(pgd, addr);
  235. #endif
  236. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  237. goto no_page;
  238. pmd = pmd_offset(pud, addr);
  239. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  240. goto no_page;
  241. BUG_ON(pmd_trans_huge(*pmd));
  242. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  243. if (!pte_present(*pte)) {
  244. spin_unlock(ptl);
  245. goto no_page;
  246. }
  247. ptep_set_wrprotect(mm, addr, pte);
  248. spin_unlock(ptl);
  249. DEBUG("make page COW at %lx\n", addr);
  250. return 0;
  251. no_page:
  252. return -EFAULT;
  253. }
  254. static void fill_page_bit_map(struct mm_struct *mm,
  255. unsigned long addr, unsigned long nr_pages,
  256. unsigned long page_bit_map[PAGE_BITS])
  257. {
  258. int i = 0;
  259. DEBUG("GIPC_SEND fill_page_bit_map %lx - %lx\n",
  260. addr, addr + (nr_pages << PAGE_SHIFT));
  261. do {
  262. struct vm_area_struct *vma;
  263. pgd_t *pgd;
  264. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  265. p4d_t *p4d;
  266. #endif
  267. pud_t *pud;
  268. pmd_t *pmd;
  269. pte_t *pte;
  270. spinlock_t *ptl;
  271. bool has_page = false;
  272. vma = find_vma(mm, addr);
  273. if (!vma)
  274. goto next;
  275. BUG_ON(vma->vm_flags & VM_HUGETLB);
  276. pgd = pgd_offset(mm, addr);
  277. if (pgd_none(*pgd) || pgd_bad(*pgd))
  278. goto next;
  279. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
  280. p4d = p4d_offset(pgd, addr);
  281. if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
  282. goto next;
  283. pud = pud_offset(p4d, addr);
  284. #else
  285. pud = pud_offset(pgd, addr);
  286. #endif
  287. if (pud_none(*pud) || pud_bad(*pud))
  288. goto next;
  289. pmd = pmd_offset(pud, addr);
  290. if (pmd_none(*pmd))
  291. goto next;
  292. if (unlikely(pmd_trans_huge(*pmd))) {
  293. has_page = true;
  294. goto next;
  295. }
  296. if (pmd_bad(*pmd))
  297. goto next;
  298. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  299. if (pte_none(*pte))
  300. goto next_locked;
  301. /*
  302. if (unlikely(!pte_present(*pte)) && pte_file(*pte))
  303. goto next_locked;
  304. */
  305. has_page = true;
  306. next_locked:
  307. spin_unlock(ptl);
  308. next:
  309. if (has_page) {
  310. DEBUG("found a page at %lx\n", addr);
  311. set_bit(i, page_bit_map);
  312. } else {
  313. clear_bit(i, page_bit_map);
  314. }
  315. } while (i++, addr += PAGE_SIZE, i < nr_pages);
  316. }
  317. static int get_pages (struct task_struct *task, unsigned long start,
  318. unsigned long nr_pages,
  319. unsigned long page_bit_map[PAGE_BITS],
  320. struct page *pages[PAGE_QUEUE],
  321. struct vm_area_struct *vmas[PAGE_QUEUE])
  322. {
  323. struct mm_struct *mm = task->mm;
  324. struct vm_area_struct *vma = NULL;
  325. unsigned long addr = start, nr;
  326. int i = 0, j, rv;
  327. while (i < nr_pages) {
  328. unsigned long flushed, vmflags;
  329. int last = i;
  330. if (test_bit(last, page_bit_map)) {
  331. i = find_next_zero_bit(page_bit_map, PAGE_QUEUE,
  332. last + 1);
  333. if (i > nr_pages)
  334. i = nr_pages;
  335. nr = i - last;
  336. DEBUG("GIPC_SEND get_user_pages %ld pages at %lx\n",
  337. addr, nr);
  338. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
  339. rv = get_user_pages(addr, nr,
  340. FOLL_GET|FOLL_FORCE|FOLL_SPLIT,
  341. pages + last, vmas + last);
  342. #else
  343. rv = __get_user_pages(task, mm, addr, nr,
  344. FOLL_GET|FOLL_FORCE|FOLL_SPLIT,
  345. pages + last, vmas + last, NULL);
  346. #endif
  347. if (rv <= 0) {
  348. printk(KERN_ERR "Graphene error: "
  349. "get_user_pages at 0x%016lx-0x%016lx\n",
  350. addr, addr + (nr << PAGE_SHIFT));
  351. return rv;
  352. }
  353. if (rv != nr) {
  354. printk(KERN_ERR "Graphene error: "
  355. "get_user_pages at 0x%016lx\n",
  356. addr + (rv << PAGE_SHIFT));
  357. return -EACCES;
  358. }
  359. flushed = addr;
  360. vmflags = 0;
  361. for (j = 0; j < nr; j++) {
  362. unsigned long target = addr + (j << PAGE_SHIFT);
  363. /* Mark source COW */
  364. rv = make_page_cow(mm, vmas[last + j],
  365. target);
  366. if (rv)
  367. return rv;
  368. if (PageAnon(pages[last + j])) {
  369. /* Fix up the counters */
  370. inc_mm_counter_fast(mm, MM_FILEPAGES);
  371. dec_mm_counter_fast(mm, MM_ANONPAGES);
  372. pages[last + j]->mapping = NULL;
  373. }
  374. #ifdef FLUSH_TLB_MM_RANGE
  375. if (vmflags == vmas[last + j]->vm_flags)
  376. continue;
  377. if (flushed < target)
  378. FLUSH_TLB_MM_RANGE(mm, flushed, target,
  379. vmflags);
  380. flushed = target;
  381. vmflags = vmas[last + j]->vm_flags;
  382. #else
  383. FLUSH_TLB_PAGE(vmas[last + j], target);
  384. #endif
  385. }
  386. #ifdef FLUSH_TLB_MM_RANGE
  387. if (flushed < addr + (nr << PAGE_SHIFT))
  388. FLUSH_TLB_MM_RANGE(mm, flushed,
  389. addr + (nr << PAGE_SHIFT),
  390. vmflags);
  391. #endif
  392. vma = vmas[i - 1];
  393. addr += nr << PAGE_SHIFT;
  394. } else {
  395. /* This is the case where a page (or pages) are not
  396. * currently mapped.
  397. * Handle the hole appropriately. */
  398. i = find_next_bit(page_bit_map, PAGE_QUEUE, last + 1);
  399. if (i > nr_pages)
  400. i = nr_pages;
  401. nr = i - last;
  402. DEBUG("GIPC_SEND skip %ld pages at %lx\n", addr, nr);
  403. for (j = 0; j < nr; j++) {
  404. if (!vma) {
  405. vma = find_vma(mm, addr);
  406. } else {
  407. /* DEP 6/17/13 - these addresses should
  408. * be monotonically increasing. */
  409. for (; vma && addr >= vma->vm_end;
  410. vma = vma->vm_next);
  411. /* Leverage monotonic increasing vmas
  412. * to more quickly detect holes in the
  413. * address space. */
  414. if (vma && addr < vma->vm_start)
  415. vma = NULL;
  416. }
  417. pages[last + j] = NULL;
  418. vmas[last + j] = vma;
  419. addr += PAGE_SIZE;
  420. }
  421. }
  422. }
  423. return i;
  424. }
  425. static int do_gipc_send(struct task_struct *task, struct gipc_queue *gq,
  426. struct gipc_send_buffer *gbuf,
  427. unsigned long __user *uaddr, unsigned long __user *ulen,
  428. unsigned long *copied_pages)
  429. {
  430. struct mm_struct *mm = task->mm;
  431. unsigned long addr, len, nr_pages;
  432. int rv, i;
  433. DEBUG("GIPC_SEND uaddr = %p, ulen = %p\n", uaddr, ulen);
  434. rv = copy_from_user(&addr, uaddr, sizeof(unsigned long));
  435. if (rv) {
  436. printk(KERN_ALERT "Graphene SEND: bad buffer %p\n", uaddr);
  437. return -EFAULT;
  438. }
  439. rv = copy_from_user(&len, ulen, sizeof(unsigned long));
  440. if (rv) {
  441. printk(KERN_ALERT "Graphene SEND: bad buffer %p\n", ulen);
  442. return -EFAULT;
  443. }
  444. if (addr > addr + len) {
  445. printk(KERN_ALERT "Graphene SEND: attempt to send %p - %p "
  446. " by thread %d FAIL: bad argument\n",
  447. (void *) addr, (void *) (addr + len), task->pid);
  448. return -EINVAL;
  449. }
  450. DEBUG("GIPC_SEND addr = %lx, len = %ld\n", addr, len);
  451. nr_pages = len >> PAGE_SHIFT;
  452. if (!access_ok(VERIFY_READ, addr, len)) {
  453. printk(KERN_ALERT "Graphene SEND:"
  454. " attempt to send %p - %p (%ld pages) "
  455. " by thread %d FAIL: bad permission\n",
  456. (void *) addr, (void *) (addr + len), nr_pages,
  457. task->pid);
  458. return -EFAULT;
  459. }
  460. DEBUG(" %p - %p (%ld pages) sent by thread %d\n",
  461. (void *) addr, (void *) (addr + len), nr_pages, task->pid);
  462. while (nr_pages) {
  463. unsigned long nr =
  464. (nr_pages <= PAGE_QUEUE) ? nr_pages : PAGE_QUEUE;
  465. /* for each of these addresses - check if
  466. * demand faulting will be triggered
  467. * if vma is present, but there is no page
  468. * present(pmd/pud not present or PTE_PRESENT
  469. * is off) then get_user_pages will trigger
  470. * the creation of those */
  471. down_write(&mm->mmap_sem);
  472. fill_page_bit_map(mm, addr, nr, gbuf->page_bit_map);
  473. rv = get_pages(task, addr, nr,
  474. gbuf->page_bit_map,
  475. gbuf->pages,
  476. gbuf->vmas);
  477. if (rv < 0) {
  478. up_write(&mm->mmap_sem);
  479. break;
  480. }
  481. for (i = 0; i < nr; i++) {
  482. BUG_ON((!gbuf->vmas[i]) && (!!gbuf->pages[i]));
  483. if (gbuf->vmas[i] && gbuf->vmas[i]->vm_file) {
  484. gbuf->files[i] = get_file(gbuf->vmas[i]->vm_file);
  485. gbuf->pgoffs[i] =
  486. ((addr - gbuf->vmas[i]->vm_start) >> PAGE_SHIFT)
  487. + gbuf->vmas[i]->vm_pgoff;
  488. } else {
  489. gbuf->files[i] = NULL;
  490. gbuf->pgoffs[i] = 0;
  491. }
  492. addr += PAGE_SIZE;
  493. }
  494. up_write(&mm->mmap_sem);
  495. for (i = 0; i < nr ; i++) {
  496. /* Put in the pending buffer*/
  497. if (((gq->last + 1) & (PAGE_QUEUE - 1)) == gq->next) {
  498. /* The blocking condition for send
  499. * and recv can't both be true! */
  500. wake_up_all(&gq->recv);
  501. wait_event_interruptible(gq->send,
  502. ((gq->last + 1) & (PAGE_QUEUE - 1)) != gq->next);
  503. if (signal_pending(task)) {
  504. rv = -ERESTARTSYS;
  505. goto out;
  506. }
  507. }
  508. gq->pages[gq->last].page = gbuf->pages[i];
  509. gq->pages[gq->last].file = gbuf->files[i];
  510. gq->pages[gq->last].pgoff = gbuf->pgoffs[i];
  511. gq->last++;
  512. gq->last &= PAGE_QUEUE - 1;
  513. (*copied_pages)++;
  514. }
  515. wake_up_all(&gq->recv);
  516. nr_pages -= nr;
  517. }
  518. out:
  519. return rv;
  520. }
  521. static inline
  522. int recv_next (struct task_struct *task, struct gipc_queue *gq)
  523. {
  524. if (gq->next == gq->last) {
  525. /* The blocking condition for send & recv can't both be true */
  526. wake_up_all(&gq->send);
  527. wait_event_interruptible(gq->recv, gq->next != gq->last);
  528. if (signal_pending(task))
  529. return -ERESTARTSYS;
  530. }
  531. return gq->next;
  532. }
  533. static int do_gipc_recv(struct task_struct *task, struct gipc_queue *gq,
  534. unsigned long __user *uaddr, unsigned long __user *ulen,
  535. unsigned long __user *uprot,
  536. unsigned long *copied_pages)
  537. {
  538. struct mm_struct *mm = task->mm;
  539. struct vm_area_struct *vma = NULL;
  540. unsigned long start, addr, len, nr_pages, prot, pgoff;
  541. struct page *page = NULL;
  542. struct file *file = NULL;
  543. int i = 0, rv;
  544. rv = copy_from_user(&addr, uaddr, sizeof(unsigned long));
  545. if (rv) {
  546. printk(KERN_ALERT "Graphene RECV: bad buffer %p\n", uaddr);
  547. return -EFAULT;
  548. }
  549. rv = copy_from_user(&len, ulen, sizeof(unsigned long));
  550. if (rv) {
  551. printk(KERN_ALERT "Graphene RECV: bad buffer %p\n", ulen);
  552. return -EFAULT;
  553. }
  554. rv = copy_from_user(&prot, uprot, sizeof(unsigned long));
  555. if (rv) {
  556. printk(KERN_ALERT "Graphene RECV: bad buffer %p\n", uprot);
  557. return -EFAULT;
  558. }
  559. nr_pages = len >> PAGE_SHIFT;
  560. start = addr;
  561. down_write(&mm->mmap_sem);
  562. while (i < nr_pages) {
  563. int found = recv_next(task, gq);
  564. int need_map = 1;
  565. if (found < 0) {
  566. rv = found;
  567. goto finish;
  568. }
  569. page = gq->pages[found].page;
  570. file = gq->pages[found].file;
  571. pgoff = gq->pages[found].pgoff;
  572. gq->next++;
  573. gq->next &= PAGE_QUEUE - 1;
  574. wake_up_all(&gq->send);
  575. if (vma) {
  576. need_map = 0;
  577. if (vma->vm_file != file)
  578. need_map = 1;
  579. if (file && vma->vm_start +
  580. ((pgoff - vma->vm_pgoff) << PAGE_SHIFT)
  581. != addr)
  582. need_map = 1;
  583. if (prot != (vma->vm_flags & (VM_READ|VM_WRITE|VM_EXEC)))
  584. need_map = 1;
  585. }
  586. if (need_map) {
  587. unsigned long flags = MAP_PRIVATE;
  588. if (addr)
  589. flags |= MAP_FIXED;
  590. if (file)
  591. flags |= MAP_FILE;
  592. else
  593. flags |= MAP_ANONYMOUS;
  594. addr = DO_MMAP_PGOFF(file, addr,
  595. (nr_pages - i) << PAGE_SHIFT,
  596. prot, flags, pgoff);
  597. if (IS_ERR_VALUE(addr)) {
  598. rv = PTR_ERR((void *) addr);
  599. printk(KERN_ERR
  600. "Graphene error: failed to mmap (%d)\n",
  601. -rv);
  602. goto finish;
  603. }
  604. if (file)
  605. DEBUG("map %08lx-%08lx file %p\n", addr,
  606. addr + ((nr_pages - i) << PAGE_SHIFT),
  607. file);
  608. else
  609. DEBUG("map %08lx-%08lx\n", addr,
  610. addr + ((nr_pages - i) << PAGE_SHIFT));
  611. if (!start)
  612. start = addr;
  613. vma = find_vma(mm, addr);
  614. if (!vma) {
  615. printk(KERN_ERR
  616. "Graphene error: can't find vma at %p\n",
  617. (void *) addr);
  618. rv = -ENOENT;
  619. goto finish;
  620. }
  621. } else {
  622. BUG_ON(!vma);
  623. }
  624. if (page) {
  625. rv = vm_insert_page(vma, addr, page);
  626. if (rv) {
  627. printk(KERN_ERR "Graphene error: "
  628. "fail to insert page %d\n", rv);
  629. goto finish;
  630. }
  631. rv = make_page_cow(mm, vma, addr);
  632. if (rv) {
  633. printk(KERN_ERR "Graphene error: "
  634. "can't make vma copy-on-write at %p\n",
  635. (void *) addr);
  636. goto finish;
  637. }
  638. }
  639. finish:
  640. /* Drop the kernel's reference to this page */
  641. if (page)
  642. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
  643. put_page(page);
  644. #else
  645. page_cache_release(page);
  646. #endif
  647. if (file)
  648. fput_atomic(file);
  649. if (rv)
  650. break;
  651. i++;
  652. addr += PAGE_SIZE;
  653. (*copied_pages)++;
  654. }
  655. up_write(&mm->mmap_sem);
  656. if (i)
  657. DEBUG(" %p - %p (%d pages) received by thread %d\n",
  658. (void *) start, (void *) start + (i << PAGE_SHIFT), i,
  659. task->pid);
  660. if (start) {
  661. rv = copy_to_user(uaddr, &start, sizeof(unsigned long));
  662. if (rv) {
  663. printk(KERN_ERR "Graphene error: bad buffer %p\n",
  664. uaddr);
  665. return -EFAULT;
  666. }
  667. }
  668. return rv;
  669. }
  670. static long gipc_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  671. {
  672. struct task_struct *task = current;
  673. struct gipc_queue *gq = NULL;
  674. long rv = 0;
  675. switch (cmd) {
  676. case GIPC_SEND: {
  677. struct gipc_send gs;
  678. struct gipc_send_buffer *gbuf;
  679. int i;
  680. unsigned long nr_pages = 0;
  681. rv = copy_from_user(&gs, (void *) arg, sizeof(gs));
  682. if (rv) {
  683. printk(KERN_ALERT "Graphene SEND: bad buffer %p\n",
  684. (void *) arg);
  685. return -EFAULT;
  686. }
  687. /* Find/allocate the gipc_pages struct for our recipient */
  688. gq = (struct gipc_queue *) file->private_data;
  689. if (!gq)
  690. return -EFAULT;
  691. gbuf = kmem_cache_alloc(gipc_send_buffer_cachep, GFP_KERNEL);
  692. if (!gbuf)
  693. return -ENOMEM;
  694. DEBUG("GIPC_SEND %ld entries to token %lld by thread %d\n",
  695. gs.entries, gq->token, task->pid);
  696. mutex_lock(&gq->send_lock);
  697. for (i = 0; i < gs.entries; i++) {
  698. rv = do_gipc_send(task, gq, gbuf, gs.addr + i,
  699. gs.len + i, &nr_pages);
  700. if (rv < 0)
  701. break;
  702. }
  703. mutex_unlock(&gq->send_lock);
  704. DEBUG("GIPC_SEND return to thread %d, %ld pages are sent\n",
  705. task->pid, nr_pages);
  706. kmem_cache_free(gipc_send_buffer_cachep, gbuf);
  707. rv = nr_pages ? : rv;
  708. break;
  709. }
  710. case GIPC_RECV: {
  711. struct gipc_recv gr;
  712. int i;
  713. unsigned long nr_pages = 0;
  714. rv = copy_from_user(&gr, (void *) arg, sizeof(gr));
  715. if (rv) {
  716. printk(KERN_ERR "Graphene error: bad buffer %p\n",
  717. (void *) arg);
  718. return -EFAULT;
  719. }
  720. gq = (struct gipc_queue *) file->private_data;
  721. if (!gq)
  722. return -EBADF;
  723. DEBUG("GIPC_RECV %ld entries to token %lld by thread %d\n",
  724. gr.entries, gq->token, task->pid);
  725. mutex_lock(&gq->recv_lock);
  726. for (i = 0; i < gr.entries; i++) {
  727. rv = do_gipc_recv(task, gq, gr.addr + i, gr.len + i,
  728. gr.prot + i, &nr_pages);
  729. if (rv < 0)
  730. break;
  731. }
  732. mutex_unlock(&gq->recv_lock);
  733. DEBUG("GIPC_RECV return to thread %d, %ld pages are received\n",
  734. task->pid, nr_pages);
  735. rv = nr_pages ? : rv;
  736. break;
  737. }
  738. case GIPC_CREATE: {
  739. gq = create_gipc_queue(file);
  740. if (!gq) {
  741. rv = -ENOMEM;
  742. break;
  743. }
  744. DEBUG("GIPC_CREATE token %lld by thread %d\n", gq->token,
  745. task->pid);
  746. rv = gq->token;
  747. break;
  748. }
  749. case GIPC_JOIN: {
  750. struct gipc_queue *q;
  751. u64 token = arg;
  752. u64 session = GIPC_OWNER;
  753. if (file->private_data != NULL)
  754. return -EBUSY;
  755. /* Search for this token */
  756. spin_lock(&gdev.lock);
  757. list_for_each_entry(q, &gdev.channels, list) {
  758. if (q->token == token) {
  759. gq = q;
  760. break;
  761. }
  762. }
  763. /* Fail if we didn't find it */
  764. if (!gq) {
  765. spin_unlock(&gdev.lock);
  766. return -ENOENT;
  767. }
  768. if (gq->owner != session) {
  769. spin_unlock(&gdev.lock);
  770. return -EPERM;
  771. }
  772. atomic_inc(&gq->count);
  773. file->private_data = gq;
  774. /* Hold the lock until we allocate so only one process
  775. * gets the queue */
  776. spin_unlock(&gdev.lock);
  777. DEBUG("GIPC_JOIN token %lld by thread %d\n", token, task->pid);
  778. rv = 0;
  779. break;
  780. }
  781. default:
  782. printk(KERN_ALERT "Graphene unknown ioctl %u %lu\n", cmd, arg);
  783. rv = -ENOSYS;
  784. break;
  785. }
  786. return rv;
  787. }
  788. static int gipc_release(struct inode *inode, struct file *file)
  789. {
  790. struct gipc_queue *gq = (struct gipc_queue *) file->private_data;
  791. if (!gq)
  792. return 0;
  793. file->private_data = NULL;
  794. release_gipc_queue(gq, false);
  795. return 0;
  796. }
  797. static int gipc_open(struct inode *inode, struct file *file)
  798. {
  799. file->private_data = NULL;
  800. return 0;
  801. }
  802. static struct file_operations gipc_fops = {
  803. .owner = THIS_MODULE,
  804. .release = gipc_release,
  805. .open = gipc_open,
  806. .unlocked_ioctl = gipc_ioctl,
  807. .compat_ioctl = gipc_ioctl,
  808. .llseek = noop_llseek,
  809. };
  810. static struct miscdevice gipc_dev = {
  811. .minor = GIPC_MINOR,
  812. .name = "gipc",
  813. .fops = &gipc_fops,
  814. .mode = 0666,
  815. };
  816. static int __init gipc_init(void)
  817. {
  818. int rv = 0;
  819. #ifdef MY_DO_MMAP
  820. LOOKUP_KSYM(do_mmap);
  821. #endif
  822. #ifdef MY_DO_MMAP_PGOFF
  823. LOOKUP_KSYM(do_mmap_pgoff);
  824. #endif
  825. #ifdef MY_FLUSH_TLB_MM_RANGE
  826. LOOKUP_KSYM(flush_tlb_mm_range);
  827. #endif
  828. #ifdef MY_FLUSH_TLB_PAGE
  829. LOOKUP_KSYM(flush_tlb_page);
  830. #endif
  831. #ifndef gipc_get_session
  832. my_gipc_get_session = (void *) kallsyms_lookup_name("gipc_get_session");
  833. #endif
  834. /* Register the kmem cache */
  835. gipc_queue_cachep = kmem_cache_create("gipc_queue",
  836. sizeof(struct gipc_queue),
  837. 0,
  838. SLAB_HWCACHE_ALIGN|
  839. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
  840. SLAB_TYPESAFE_BY_RCU,
  841. #else
  842. SLAB_DESTROY_BY_RCU,
  843. #endif
  844. NULL);
  845. if (!gipc_queue_cachep) {
  846. printk(KERN_ERR "Graphene error: "
  847. "failed to create a gipc queues cache\n");
  848. return -ENOMEM;
  849. }
  850. gipc_send_buffer_cachep = kmem_cache_create("gipc_send_buffer",
  851. sizeof(struct gipc_send_buffer),
  852. 0,
  853. SLAB_HWCACHE_ALIGN|
  854. #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
  855. SLAB_TYPESAFE_BY_RCU,
  856. #else
  857. SLAB_DESTROY_BY_RCU,
  858. #endif
  859. NULL);
  860. if (!gipc_send_buffer_cachep) {
  861. printk(KERN_ERR "Graphene error: "
  862. "failed to create a gipc buffers cache\n");
  863. return -ENOMEM;
  864. }
  865. INIT_LIST_HEAD(&gdev.channels);
  866. spin_lock_init(&gdev.lock);
  867. gdev.max_token = 1;
  868. rv = misc_register(&gipc_dev);
  869. if (rv) {
  870. printk(KERN_ERR "Graphene error: "
  871. "failed to add a char device (rv=%d)\n", rv);
  872. return rv;
  873. }
  874. printk(KERN_ALERT "Graphene IPC: Hello, world\n");
  875. return 0;
  876. }
  877. static void __exit gipc_exit(void)
  878. {
  879. struct gipc_queue *gq, *n;
  880. spin_lock(&gdev.lock);
  881. list_for_each_entry_safe(gq, n, &gdev.channels, list)
  882. release_gipc_queue(gq, true);
  883. spin_unlock(&gdev.lock);
  884. misc_deregister(&gipc_dev);
  885. kmem_cache_destroy(gipc_queue_cachep);
  886. printk(KERN_ALERT "Graphene IPC: Goodbye, cruel world\n");
  887. }
  888. module_init(gipc_init);
  889. module_exit(gipc_exit);