[LibOS] Use %gs register for LibOS TCB (shim_tcb)

For binaries statically linked against Glibc, %fs register cannot be
used for LibOS TCB (shim_tcb) because it is already used by Glibc.
Thus, this commit moves shim_tcb into PAL TCB. Also, now that LibOS
doesn't access Glibc TCB (__libc_tcb), we make it an opaque pointer
used only for clean up.

LibOS/shim/include/shim_thread.h

@@ -84,6 +84,7 @@ struct shim_thread {
     void * stack, * stack_top, * stack_red;
     __libc_tcb_t * tcb;
     bool user_tcb; /* is tcb assigned by user? */
+    shim_tcb_t * shim_tcb;
     void * frameptr;
 
     REFTYPE ref_count;
@@ -121,16 +122,16 @@ int init_thread (void);
 
 static inline struct shim_thread * shim_thread_self(void)
 {
-    struct shim_thread * __self;
-    __asm__ ("movq %%fs:%c1,%q0" : "=r" (__self)
-             : "i" (offsetof(__libc_tcb_t, shim_tcb.tp)));
-    return __self;
+    /* TODO: optimize to use single movq %gs:<offset> */
+    shim_tcb_t * shim_tcb = shim_get_tls();
+    return shim_tcb->tp;
 }
 
 static inline struct shim_thread * save_shim_thread_self(struct shim_thread * __self)
 {
-    __asm__ ("movq %q0,%%fs:%c1" : : "r" (__self),
-             "i" (offsetof(__libc_tcb_t, shim_tcb.tp)));
+    /* TODO: optimize to use single movq %gs:<offset> */
+    shim_tcb_t * shim_tcb = shim_get_tls();
+    shim_tcb->tp = __self;
     return __self;
 }
 
@@ -185,6 +186,7 @@ void set_cur_thread (struct shim_thread * thread)
     IDTYPE tid = 0;
 
     if (thread) {
+        __libc_tcb_t * libc_tcb = tcb->tp ? tcb->tp->tcb : NULL;
         if (tcb->tp && tcb->tp != thread)
             put_thread(tcb->tp);
 
@@ -192,7 +194,9 @@ void set_cur_thread (struct shim_thread * thread)
             get_thread(thread);
 
         tcb->tp = thread;
-        thread->tcb = container_of(tcb, __libc_tcb_t, shim_tcb);
+        if (libc_tcb)
+            thread->tcb = libc_tcb;
+        thread->shim_tcb = tcb;
         tid = thread->tid;
 
         if (!is_internal(thread) && !thread->signal_logs)
@@ -328,8 +332,6 @@ struct clone_args {
     void * stack;
 };
 
-int clone_implementation_wrapper(struct clone_args * arg);
-
 void * allocate_stack (size_t size, size_t protect_size, bool user);
 
 static inline __attribute__((always_inline))

LibOS/shim/include/shim_tls.h

@@ -70,52 +70,27 @@ struct shim_tcb {
 
 #ifdef IN_SHIM
 
-/*
- * This struct must match the one defined in glibc/nptl/sysdeps/x86_64/tls.h
- * The first 10 members(from tcb to __unused1) are used by Glibc-internal,
- * they are NOT used by Graphene.
- * But Graphene needs to preserve the correct offset of shim_tcb so we have to
- * duplicate these 10 fields from the original Glibc struct.
- */
-struct __libc_tcb_t;
-typedef struct __libc_tcb_t __libc_tcb_t;
-struct __libc_tcb_t
-{
-    __libc_tcb_t *          tcb;
-    void *                  dtv, * self;
-    int                     mthreads, gscope;
-    uintptr_t               sysinfo, sg, pg;
-    unsigned long int       vgetcpu_cache[2];
-    int                     __unused1;
-    shim_tcb_t              shim_tcb;
-};
-
 #include <stddef.h>
 
 void init_tcb (shim_tcb_t * tcb);
+struct __libc_tcb_t;
+typedef struct __libc_tcb_t __libc_tcb_t;
 
-static inline bool shim_tls_check_canary(void)
-{
-    uint64_t __canary;
-    __asm__ ("movq %%fs:%c1,%q0" : "=r" (__canary)
-             : "i" (offsetof(__libc_tcb_t, shim_tcb.canary)));
-    return __canary == SHIM_TLS_CANARY;
-}
+/* don't define struct __libc_tcb_t. just type to point to libc tls
+ * LibOS doesn't access this structure as it's private to libc.
+ */
 
 static inline shim_tcb_t * shim_get_tls(void)
 {
-    shim_tcb_t *__self;
-    __asm__ ("movq %%fs:%c1,%q0" : "=r" (__self)
-             : "i" (offsetof(__libc_tcb_t, shim_tcb.self)));
-    return __self;
+    PAL_TCB * tcb = pal_get_tcb();
+    return (shim_tcb_t*)tcb->libos_tcb;
 }
 
-static inline __libc_tcb_t * shim_libc_tcb(void)
+static inline bool shim_tls_check_canary(void)
 {
-    __libc_tcb_t *__self;
-    __asm__ ("movq %%fs:%c1,%q0" : "=r" (__self)
-             : "i" (offsetof(__libc_tcb_t, tcb)));
-    return __self;
+    /* TODO: optimize to use single movq %gs:<offset> */
+    shim_tcb_t * shim_tcb = shim_get_tcb();
+    return shim_tcb->canary == SHIM_TLS_CANARY;
 }
 
 #endif /* IN_SHIM */

LibOS/shim/src/bookkeep/shim_thread.c

@@ -20,6 +20,7 @@
  * This file contains codes to maintain bookkeeping of threads in library OS.
  */
 
+#include <shim_defs.h>
 #include <shim_internal.h>
 #include <shim_thread.h>
 #include <shim_handle.h>
@@ -688,10 +689,10 @@ BEGIN_RS_FUNC(thread)
     if (thread->cwd)
         get_dentry(thread->cwd);
 
-    DEBUG_RS("tid=%d,tgid=%d,parent=%d,stack=%p,frameptr=%p,tcb=%p",
+    DEBUG_RS("tid=%d,tgid=%d,parent=%d,stack=%p,frameptr=%p,tcb=%p,shim_tcb=%p",
              thread->tid, thread->tgid,
              thread->parent ? thread->parent->tid : thread->tid,
-             thread->stack, thread->frameptr, thread->tcb);
+             thread->stack, thread->frameptr, thread->tcb, thread->shim_tcb);
 }
 END_RS_FUNC(thread)
 
@@ -707,27 +708,27 @@ BEGIN_CP_FUNC(running_thread)
     DO_CP(thread, thread, &new_thread);
     ADD_CP_FUNC_ENTRY((ptr_t) new_thread - base);
 
-    if (!thread->user_tcb && thread->tcb) {
-        ptr_t toff = ADD_CP_OFFSET(sizeof(__libc_tcb_t));
-        new_thread->tcb = (void *) (base + toff);
-        memcpy(new_thread->tcb, thread->tcb, sizeof(__libc_tcb_t));
+    if (thread->shim_tcb) {
+        ptr_t toff = ADD_CP_OFFSET(sizeof(shim_tcb_t));
+        new_thread->shim_tcb = (void *)(base + toff);
+        memcpy(new_thread->shim_tcb, thread->shim_tcb, sizeof(shim_tcb_t));
     }
 }
 END_CP_FUNC(running_thread)
 
-int resume_wrapper (void * param)
+static int resume_wrapper (void * param)
 {
     struct shim_thread * thread = (struct shim_thread *) param;
     assert(thread);
 
     __libc_tcb_t * libc_tcb = thread->tcb;
     assert(libc_tcb);
-    shim_tcb_t * tcb = &libc_tcb->shim_tcb;
+    shim_tcb_t * tcb = thread->shim_tcb;
     assert(tcb->context.regs && tcb->context.regs->rsp);
 
     thread->in_vm = thread->is_alive = true;
     allocate_tls(libc_tcb, thread->user_tcb, thread);
-    debug_setbuf(tcb, true);
+    debug_setbuf(tcb, false);
     debug("set tcb to %p\n", libc_tcb);
 
     object_wait_with_retry(thread_start_event);
@@ -747,6 +748,8 @@ BEGIN_RS_FUNC(running_thread)
 
     if (!thread->user_tcb)
         CP_REBASE(thread->tcb);
+    if (thread->shim_tcb)
+        CP_REBASE(thread->shim_tcb);
 
     if (thread->set_child_tid) {
         /* CLONE_CHILD_SETTID */
@@ -764,10 +767,15 @@ BEGIN_RS_FUNC(running_thread)
 
         thread->pal_handle = handle;
     } else {
+        if (thread->shim_tcb) {
+            memcpy(shim_get_tls(), thread->shim_tcb, sizeof(shim_tcb_t));
+            thread->shim_tcb = shim_get_tls();
+        }
+        debug_setbuf(thread->shim_tcb, false);
         __libc_tcb_t * libc_tcb = thread->tcb;
 
         if (libc_tcb) {
-            shim_tcb_t * tcb = &libc_tcb->shim_tcb;
+            shim_tcb_t * tcb = thread->shim_tcb;
             assert(tcb->context.regs && tcb->context.regs->rsp);
             tcb->debug_buf = shim_get_tls()->debug_buf;
             allocate_tls(libc_tcb, thread->user_tcb, thread);
@@ -783,9 +791,11 @@ BEGIN_RS_FUNC(running_thread)
              * frameptr = NULL
              * tcb = NULL
              * user_tcb = false
+             * shim_tcb = NULL
              * in_vm = false
              */
-            init_tcb(&shim_libc_tcb()->shim_tcb);
+            thread->shim_tcb = shim_get_tls();
+            init_tcb(thread->shim_tcb);
             set_cur_thread(thread);
         }
 

LibOS/shim/src/generated-offsets.c

@@ -7,7 +7,7 @@
 
 void dummy(void)
 {
-    OFFSET_T(SHIM_TCB_OFFSET, __libc_tcb_t, shim_tcb);
+    OFFSET_T(SHIM_TCB_OFFSET, PAL_TCB, libos_tcb);
     OFFSET_T(TCB_REGS, shim_tcb_t, context.regs);
     OFFSET(SHIM_REGS_RSP, shim_regs, rsp);
     OFFSET(SHIM_REGS_R15, shim_regs, r15);

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -791,10 +791,10 @@ static void shim_ipc_helper_prepare(void* arg) {
     if (!arg)
         return;
 
-    __libc_tcb_t tcb;
-    allocate_tls(&tcb, false, self);
-    debug_setbuf(&tcb.shim_tcb, true);
-    debug("Set tcb to %p\n", &tcb);
+    __libc_tcb_t* tcb = NULL;
+    allocate_tls(tcb, false, self);
+    debug_setbuf(shim_get_tls(), true);
+    debug("Set tcb to %p\n", tcb);
 
     lock(&ipc_helper_lock);
     bool notme = (self != ipc_helper_thread);

LibOS/shim/src/shim_async.c

@@ -144,10 +144,10 @@ static void shim_async_helper(void * arg) {
     if (!arg)
         return;
 
-    __libc_tcb_t tcb;
-    allocate_tls(&tcb, false, self);
-    debug_setbuf(&tcb.shim_tcb, true);
-    debug("Set tcb to %p\n", &tcb);
+    __libc_tcb_t* tcb = NULL;
+    allocate_tls(tcb, false, self);
+    debug_setbuf(shim_get_tls(), true);
+    debug("Set tcb to %p\n", tcb);
 
     lock(&async_helper_lock);
     bool notme = (self != async_helper_thread);

LibOS/shim/src/shim_init.c

@@ -20,6 +20,7 @@
  * This file contains entry and exit functions of library OS.
  */
 
+#include <shim_defs.h>
 #include <shim_internal.h>
 #include <shim_table.h>
 #include <shim_tls.h>
@@ -40,6 +41,10 @@
 #include <asm/unistd.h>
 #include <asm/fcntl.h>
 
+static_assert(sizeof(shim_tcb_t) <= PAL_LIBOS_TCB_SIZE,
+              "shim_tcb_t does not fit into PAL_TCB; "
+              "please increase PAL_LIBOS_TCB_SIZE");
+
 size_t g_pal_alloc_align;
 
 /* The following constants will help matching glibc version with compatible
@@ -200,32 +205,22 @@ void init_tcb (shim_tcb_t * tcb)
     tcb->self = tcb;
 }
 
-void copy_tcb (shim_tcb_t * new_tcb, const shim_tcb_t * old_tcb)
-{
-    memset(new_tcb, 0, sizeof(shim_tcb_t));
-    new_tcb->canary = SHIM_TLS_CANARY;
-    new_tcb->self = new_tcb;
-    new_tcb->tp   = old_tcb->tp;
-    memcpy(&new_tcb->context, &old_tcb->context, sizeof(struct shim_context));
-    new_tcb->tid  = old_tcb->tid;
-    new_tcb->debug_buf = old_tcb->debug_buf;
-}
-
 /* This function is used to allocate tls before interpreter start running */
 void allocate_tls (__libc_tcb_t * tcb, bool user, struct shim_thread * thread)
 {
-    assert(tcb);
-    tcb->tcb = tcb;
-    init_tcb(&tcb->shim_tcb);
+    shim_tcb_t * shim_tcb;
+    shim_tcb = shim_get_tls();
+    init_tcb(shim_tcb);
 
     if (thread) {
-        thread->tcb       = tcb;
-        thread->user_tcb  = user;
-        tcb->shim_tcb.tp  = thread;
-        tcb->shim_tcb.tid = thread->tid;
+        thread->tcb      = tcb;
+        thread->user_tcb = user;
+        thread->shim_tcb = shim_tcb;
+        shim_tcb->tp  = thread;
+        shim_tcb->tid = thread->tid;
     } else {
-        tcb->shim_tcb.tp  = NULL;
-        tcb->shim_tcb.tid = 0;
+        shim_tcb->tp  = NULL;
+        shim_tcb->tid = 0;
     }
 
     DkSegmentRegister(PAL_SEGMENT_FS, tcb);
@@ -234,14 +229,14 @@ void allocate_tls (__libc_tcb_t * tcb, bool user, struct shim_thread * thread)
 
 void populate_tls (__libc_tcb_t * tcb, bool user)
 {
-    assert(tcb);
-    tcb->tcb = tcb;
-    copy_tcb(&tcb->shim_tcb, shim_get_tls());
+    shim_tcb_t * shim_tcb;
+    shim_tcb = shim_get_tls();
 
-    struct shim_thread * thread = (struct shim_thread *) tcb->shim_tcb.tp;
+    struct shim_thread * thread = shim_tcb->tp;
     if (thread) {
         thread->tcb = tcb;
         thread->user_tcb = user;
+        thread->shim_tcb = shim_tcb;
     }
 
     DkSegmentRegister(PAL_SEGMENT_FS, tcb);
@@ -684,13 +679,12 @@ noreturn void* shim_init (int argc, void * args)
     cur_process.vmid = (IDTYPE) PAL_CB(process_id);
 
     /* create the initial TCB, shim can not be run without a tcb */
-    __libc_tcb_t tcb;
-    memset(&tcb, 0, sizeof(__libc_tcb_t));
-    allocate_tls(&tcb, false, NULL);
-    __disable_preempt(&tcb.shim_tcb); // Temporarily disable preemption for delaying any signal
-                                      // that arrives during initialization
-    debug_setbuf(&tcb.shim_tcb, true);
-    debug("set tcb to %p\n", &tcb);
+    __libc_tcb_t* tcb = NULL;
+    allocate_tls(tcb, false, NULL);
+    __disable_preempt(shim_get_tls()); // Temporarily disable preemption for delaying any signal
+                                       // that arrives during initialization
+    debug_setbuf(shim_get_tls(), true);
+    debug("set tcb to %p\n", tcb);
 
 #ifdef PROFILE
     unsigned long begin_time = GET_PROFILE_INTERVAL();

LibOS/shim/src/sys/shim_clone.c

@@ -114,25 +114,19 @@ static void fixup_child_context(struct shim_regs * regs)
  */
 #define PTHREAD_PADDING 2048
 
-int clone_implementation_wrapper(struct clone_args * arg)
+static int clone_implementation_wrapper(struct clone_args * arg)
 {
     //The child thread created by PAL is now running on the
     //PAL allocated stack. We need to switch the stack to use
     //the user provided stack.
 
-    int stack_allocated = 0;
-
     /* We acquired ownership of arg->thread from the caller, hence there is
      * no need to call get_thread. */
     struct shim_thread* my_thread = arg->thread;
     assert(my_thread);
 
-    if (!my_thread->tcb) {
-        stack_allocated = 1;
-        my_thread->tcb = __alloca(sizeof(__libc_tcb_t) + PTHREAD_PADDING);
-    }
     allocate_tls(my_thread->tcb, my_thread->user_tcb, my_thread); /* set up TCB */
-    shim_tcb_t * tcb = &my_thread->tcb->shim_tcb;
+    shim_tcb_t * tcb = my_thread->shim_tcb;
 
     /* only now we can call LibOS/PAL functions because they require a set-up TCB;
      * do not move the below functions before allocate_tls()! */
@@ -142,9 +136,9 @@ int clone_implementation_wrapper(struct clone_args * arg)
     __disable_preempt(tcb); // Temporarily disable preemption, because the preemption
                             // will be re-enabled when the thread starts.
     debug_setbuf(tcb, true);
-    debug("set tcb to %p (stack allocated? %d)\n", my_thread->tcb, stack_allocated);
+    debug("set tcb to %p\n", my_thread->tcb);
 
-    struct shim_regs regs = *arg->parent->tcb->shim_tcb.context.regs;
+    struct shim_regs regs = *arg->parent->shim_tcb->context.regs;
     if (my_thread->set_child_tid) {
         *(my_thread->set_child_tid) = my_thread->tid;
         my_thread->set_child_tid = NULL;
@@ -335,8 +329,9 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
         } else {
             thread->tcb = tcb = self->tcb;
             old_shim_tcb = __alloca(sizeof(shim_tcb_t));
-            memcpy(old_shim_tcb, &tcb->shim_tcb, sizeof(shim_tcb_t));
+            memcpy(old_shim_tcb, self->shim_tcb, sizeof(shim_tcb_t));
             thread->user_tcb = self->user_tcb;
+            thread->shim_tcb = self->shim_tcb;
         }
 
         if (user_stack_addr) {
@@ -344,8 +339,8 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
             lookup_vma(PAGE_ALIGN_DOWN_PTR(user_stack_addr), &vma);
             thread->stack_top = vma.addr + vma.length;
             thread->stack_red = thread->stack = vma.addr;
-            parent_stack = (void *)tcb->shim_tcb.context.regs->rsp;
-            tcb->shim_tcb.context.regs->rsp = (unsigned long)user_stack_addr;
+            parent_stack = (void *)self->shim_tcb->context.regs->rsp;
+            thread->shim_tcb->context.regs->rsp = (unsigned long)user_stack_addr;
         }
 
         thread->is_alive = true;
@@ -354,10 +349,12 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
         set_as_child(self, thread);
 
         ret = do_migrate_process(&migrate_fork, NULL, NULL, thread);
-        if (old_shim_tcb)
-            memcpy(&tcb->shim_tcb, old_shim_tcb, sizeof(tcb->shim_tcb));
+        if (old_shim_tcb) {
+            thread->shim_tcb = NULL;
+            memcpy(self->shim_tcb, old_shim_tcb, sizeof(*self->shim_tcb));
+        }
         if (parent_stack)
-            tcb->shim_tcb.context.regs->rsp = (unsigned long)parent_stack;
+            self->shim_tcb->context.regs->rsp = (unsigned long)parent_stack;
         if (ret < 0)
             goto failed;
 

LibOS/shim/src/sys/shim_exec.c

@@ -87,16 +87,8 @@ noreturn static void __shim_do_execve_rtld (struct execve_rtld_arg * __arg)
     struct shim_thread * cur_thread = get_cur_thread();
     int ret = 0;
 
-#ifdef SHIM_TCB_USE_GS
     /* libc tcb is not needed because PAL provides storage for shim_tcb */
     __libc_tcb_t* tcb = NULL;
-#else
-# define LIBC_TCB_ALLOC_SIZE    (sizeof(__libc_tcb_t) + __alignof__(__libc_tcb_t))
-    __libc_tcb_t* tcb = ALIGN_UP_PTR(
-        cur_thread->stack_top - LIBC_TCB_ALLOC_SIZE,
-        __alignof__(*tcb));
-    memset(tcb, 0, sizeof(*tcb));
-#endif
     populate_tls(tcb, false);
     debug("set tcb to %p\n", tcb);
 
@@ -220,12 +212,7 @@ static int shim_do_execve_rtld (struct shim_handle * hdl, const char ** argv,
     int * new_argcp = &new_argc;
     const char ** new_argp;
     elf_auxv_t * new_auxp;
-#ifdef SHIM_TCB_USE_GS
     size_t reserve = 0;
-#else
-    /* reserve __libc_tcb_t for startup use. see __shim_do_execve_rtld() */
-    size_t reserve = LIBC_TCB_ALLOC_SIZE;
-#endif
     if ((ret = init_stack(argv, envp, &new_argcp, &new_argp, &new_auxp,
                           reserve)) < 0)
         return ret;
@@ -506,6 +493,7 @@ err:
     void * stack     = cur_thread->stack;
     void * stack_top = cur_thread->stack_top;
     __libc_tcb_t * tcb = cur_thread->tcb;
+    shim_tcb_t * shim_tcb = cur_thread->shim_tcb;
     bool   user_tcb  = cur_thread->user_tcb;
     void * frameptr  = cur_thread->frameptr;
 
@@ -514,6 +502,7 @@ err:
     cur_thread->frameptr  = NULL;
     cur_thread->tcb       = NULL;
     cur_thread->user_tcb  = false;
+    cur_thread->shim_tcb  = NULL;
     cur_thread->in_vm     = false;
     unlock(&cur_thread->lock);
 
@@ -525,6 +514,7 @@ err:
     cur_thread->frameptr    = frameptr;
     cur_thread->tcb         = tcb;
     cur_thread->user_tcb    = user_tcb;
+    cur_thread->shim_tcb    = shim_tcb;
 
     if (ret < 0) {
         /* execve failed, so reanimate this thread as if nothing happened */

LibOS/shim/src/sys/shim_fork.c

@@ -86,6 +86,7 @@ int shim_do_fork (void)
 
     new_thread->tcb      = cur_thread->tcb;
     new_thread->user_tcb = cur_thread->user_tcb;
+    new_thread->shim_tcb = cur_thread->shim_tcb;
     new_thread->tgid     = new_thread->tid;
     new_thread->in_vm    = false;
     new_thread->is_alive = true;
@@ -100,6 +101,7 @@ int shim_do_fork (void)
     lock(&new_thread->lock);
     struct shim_handle_map * handle_map = new_thread->handle_map;
     new_thread->handle_map = NULL;
+    new_thread->shim_tcb = NULL;
     unlock(&new_thread->lock);
     if (handle_map)
         put_handle_map(handle_map);

LibOS/shim/src/syscallas.S

@@ -20,6 +20,7 @@
  * This file contains the entry point of system call table in library OS.
  */
 
+#include <shim_defs.h>
 #include <shim_unistd_defs.h>
 
 #include "asm-offsets.h"
@@ -76,7 +77,7 @@ syscalldb:
         cmp $0, %rbx
         je isundef
 
-        movq %rbp, %fs:(SHIM_TCB_OFFSET + TCB_REGS)
+        movq %rbp, %gs:(SHIM_TCB_OFFSET + TCB_REGS)
 
         /* Translating x86_64 kernel calling convention to user-space
          * calling convention */
@@ -84,7 +85,7 @@ syscalldb:
         andq $~0xF, %rsp  # Required by System V AMD64 ABI.
         call *%rbx
 
-        movq $0, %fs:(SHIM_TCB_OFFSET + TCB_REGS)
+        movq $0, %gs:(SHIM_TCB_OFFSET + TCB_REGS)
 
 ret:
         movq %rbp, %rsp