[LibOS] Move shim_thread:fs_base to shim_context::fs_base

fs_base is now an opaque value of FS register. It belongs to shim_context.

LibOS/shim/include/shim_tcb.h

@@ -28,6 +28,7 @@ struct shim_regs {
 
 struct shim_context {
     struct shim_regs *      regs;
+    unsigned long           fs_base;
     struct shim_context *   next;
     uint64_t                enter_time;
     struct atomic_int       preempt;

LibOS/shim/include/shim_thread.h

@@ -82,7 +82,6 @@ struct shim_thread {
     struct shim_handle * exec;
 
     void * stack, * stack_top, * stack_red;
-    unsigned long fs_base;
     shim_tcb_t * shim_tcb;
     void * frameptr;
 
@@ -185,7 +184,6 @@ void set_cur_thread (struct shim_thread * thread)
     IDTYPE tid = 0;
 
     if (thread) {
-        unsigned long fs_base = tcb->tp ? tcb->tp->fs_base : 0;
         if (tcb->tp && tcb->tp != thread)
             put_thread(tcb->tp);
 
@@ -193,7 +191,6 @@ void set_cur_thread (struct shim_thread * thread)
             get_thread(thread);
 
         tcb->tp = thread;
-        thread->fs_base = fs_base;
         thread->shim_tcb = tcb;
         tid = thread->tid;
 
@@ -328,6 +325,7 @@ struct clone_args {
     PAL_HANDLE initialize_event;
     struct shim_thread * parent, * thread;
     void * stack;
+    unsigned long fs_base;
 };
 
 void * allocate_stack (size_t size, size_t protect_size, bool user);

LibOS/shim/src/bookkeep/shim_thread.c

@@ -721,13 +721,20 @@ static int resume_wrapper (void * param)
     struct shim_thread * thread = (struct shim_thread *) param;
     assert(thread);
 
-    unsigned long fs_base = thread->fs_base;
+    /* initialize the current shim_tcb_t (= shim_get_tcb())
+       based on saved thread->shim_tcb */
+    shim_tcb_t* saved_tcb = thread->shim_tcb;
+    assert(saved_tcb->context.regs && saved_tcb->context.regs->rsp);
+    unsigned long fs_base = saved_tcb->context.fs_base;
     assert(fs_base);
-    shim_tcb_t * tcb = thread->shim_tcb;
-    assert(tcb->context.regs && tcb->context.regs->rsp);
+    init_fs_base(fs_base, thread);
 
     thread->in_vm = thread->is_alive = true;
-    init_fs_base(fs_base, thread);
+
+    shim_tcb_t* tcb = shim_get_tcb();
+    tcb->context.regs = saved_tcb->context.regs;
+    tcb->context.enter_time = saved_tcb->context.enter_time;
+    tcb->context.preempt = saved_tcb->context.preempt;
     debug_setbuf(tcb, false);
     debug("set fs_base to 0x%lx\n", fs_base);
 
@@ -765,22 +772,18 @@ BEGIN_RS_FUNC(running_thread)
 
         thread->pal_handle = handle;
     } else {
-        if (thread->shim_tcb) {
-            memcpy(shim_get_tcb(), thread->shim_tcb, sizeof(shim_tcb_t));
-            thread->shim_tcb = shim_get_tcb();
-        }
-        debug_setbuf(thread->shim_tcb, false);
-        unsigned long fs_base = thread->fs_base;
+        shim_tcb_t* saved_tcb = thread->shim_tcb;
+        if (saved_tcb) {
+            /* fork case */
+            shim_tcb_t* tcb = shim_get_tcb();
+            memcpy(tcb, saved_tcb, sizeof(*tcb));
 
-        if (fs_base) {
-            shim_tcb_t * tcb = thread->shim_tcb;
             assert(tcb->context.regs && tcb->context.regs->rsp);
-            tcb->debug_buf = shim_get_tcb()->debug_buf;
-            init_fs_base(fs_base, thread);
+            init_fs_base(tcb->context.fs_base, thread);
             /* Temporarily disable preemption until the thread resumes. */
             __disable_preempt(tcb);
-            debug_setprefix(tcb);
-            debug("after resume, set tcb to 0x%lx\n", fs_base);
+            debug_setbuf(tcb, false);
+            debug("after resume, set tcb to 0x%lx\n", tcb->context.fs_base);
         } else {
             /*
              * In execve case, the following holds:
@@ -793,6 +796,7 @@ BEGIN_RS_FUNC(running_thread)
              */
             thread->shim_tcb = shim_get_tcb();
             init_tcb(thread->shim_tcb);
+            debug_setbuf(thread->shim_tcb, false);
             set_cur_thread(thread);
         }
 

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -791,8 +791,7 @@ static void shim_ipc_helper_prepare(void* arg) {
     if (!arg)
         return;
 
-    unsigned long fs_base = 0;
-    init_fs_base(fs_base, self);
+    init_fs_base(0, self);
     debug_setbuf(shim_get_tcb(), true);
 
     lock(&ipc_helper_lock);

LibOS/shim/src/shim_async.c

@@ -144,8 +144,7 @@ static void shim_async_helper(void * arg) {
     if (!arg)
         return;
 
-    unsigned long fs_base = 0;
-    init_fs_base(fs_base, self);
+    init_fs_base(0, self);
     debug_setbuf(shim_get_tcb(), true);
 
     lock(&async_helper_lock);

LibOS/shim/src/shim_checkpoint.c

@@ -1304,7 +1304,9 @@ void restore_context (struct shim_context * context)
     shim_tcb_t * tcb = shim_get_tcb();
     __enable_preempt(tcb);
 
+    unsigned long fs_base = context->fs_base;
     memset(context, 0, sizeof(struct shim_context));
+    context->fs_base = fs_base;
 
     __asm__ volatile("movq %0, %%rsp\r\n"
                      "addq $2 * 8, %%rsp\r\n"    /* skip orig_rax and rsp */

LibOS/shim/src/shim_init.c

@@ -212,7 +212,6 @@ void init_fs_base (unsigned long fs_base, struct shim_thread * thread)
     init_tcb(shim_tcb);
 
     if (thread) {
-        thread->fs_base = fs_base;
         thread->shim_tcb = shim_tcb;
         shim_tcb->tp = thread;
         shim_tcb->tid = thread->tid;
@@ -221,6 +220,7 @@ void init_fs_base (unsigned long fs_base, struct shim_thread * thread)
         shim_tcb->tid = 0;
     }
 
+    shim_tcb->context.fs_base = fs_base;
     DkSegmentRegister(PAL_SEGMENT_FS, (PAL_PTR)fs_base);
     assert(shim_tcb_check_canary());
 }
@@ -231,10 +231,10 @@ void update_fs_base (unsigned long fs_base)
 
     struct shim_thread * thread = shim_tcb->tp;
     if (thread) {
-        thread->fs_base = fs_base;
         thread->shim_tcb = shim_tcb;
     }
 
+    shim_tcb->context.fs_base = fs_base;
     DkSegmentRegister(PAL_SEGMENT_FS, (PAL_PTR)fs_base);
     assert(shim_tcb_check_canary());
 }
@@ -675,8 +675,7 @@ noreturn void* shim_init (int argc, void * args)
     cur_process.vmid = (IDTYPE) PAL_CB(process_id);
 
     /* create the initial TCB, shim can not be run without a tcb */
-    unsigned long fs_base = 0;
-    init_fs_base(fs_base, NULL);
+    init_fs_base(0, NULL);
     __disable_preempt(shim_get_tcb()); // Temporarily disable preemption for delaying any signal
                                        // that arrives during initialization
     debug_setbuf(shim_get_tcb(), true);

LibOS/shim/src/sys/shim_clone.c

@@ -125,7 +125,7 @@ static int clone_implementation_wrapper(struct clone_args * arg)
     struct shim_thread* my_thread = arg->thread;
     assert(my_thread);
 
-    init_fs_base(my_thread->fs_base, my_thread); /* set up TCB */
+    init_fs_base(arg->fs_base, my_thread);
     shim_tcb_t * tcb = my_thread->shim_tcb;
 
     /* only now we can call LibOS/PAL functions because they require a set-up TCB;
@@ -136,7 +136,7 @@ static int clone_implementation_wrapper(struct clone_args * arg)
     __disable_preempt(tcb); // Temporarily disable preemption, because the preemption
                             // will be re-enabled when the thread starts.
     debug_setbuf(tcb, true);
-    debug("set fs_base to 0x%lx\n", my_thread->fs_base);
+    debug("set fs_base to 0x%lx\n", tcb->context.fs_base);
 
     struct shim_regs regs = *arg->parent->shim_tcb->context.regs;
     if (my_thread->set_child_tid) {
@@ -290,14 +290,13 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
         /* Implemented in shim_futex.c: release_clear_child_id */
         thread->clear_child_tid = parent_tidptr;
 
+    unsigned long fs_base = 0;
     if (flags & CLONE_SETTLS) {
         if (!tls) {
             ret = -EINVAL;
             goto failed;
         }
-        thread->fs_base = (unsigned long)tls;
-    } else {
-        thread->fs_base = 0;
+        fs_base = (unsigned long)tls;
     }
 
     if (!(flags & CLONE_THREAD))
@@ -319,18 +318,16 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
     }
 
     if (!(flags & CLONE_VM)) {
-        unsigned long fs_base;
-        shim_tcb_t * old_shim_tcb = NULL;
         void * parent_stack = NULL;
 
-        if (thread->fs_base) {
-            fs_base = thread->fs_base;
-        } else {
-            thread->fs_base = fs_base = self->fs_base;
-            old_shim_tcb = __alloca(sizeof(shim_tcb_t));
-            memcpy(old_shim_tcb, self->shim_tcb, sizeof(shim_tcb_t));
-            thread->shim_tcb = self->shim_tcb;
+        if (!fs_base) {
+            fs_base = self->shim_tcb->context.fs_base;
         }
+        /* associate cpu context to new forking thread for migration */
+        shim_tcb_t shim_tcb;
+        memcpy(&shim_tcb, self->shim_tcb, sizeof(shim_tcb_t));
+        shim_tcb.context.fs_base = fs_base;
+        thread->shim_tcb = &shim_tcb;
 
         if (user_stack_addr) {
             struct shim_vma_val vma;
@@ -347,10 +344,8 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
         set_as_child(self, thread);
 
         ret = do_migrate_process(&migrate_fork, NULL, NULL, thread);
-        if (old_shim_tcb) {
-            thread->shim_tcb = NULL;
-            memcpy(self->shim_tcb, old_shim_tcb, sizeof(*self->shim_tcb));
-        }
+        thread->shim_tcb = NULL; /* cpu context of forked thread isn't
+                                  * needed any more */
         if (parent_stack)
             self->shim_tcb->context.regs->rsp = (unsigned long)parent_stack;
         if (ret < 0)
@@ -394,6 +389,7 @@ int shim_do_clone (int flags, void * user_stack_addr, int * parent_tidptr,
     new_args.thread    = thread;
     new_args.parent    = self;
     new_args.stack     = user_stack_addr;
+    new_args.fs_base   = fs_base;
 
     // Invoke DkThreadCreate to spawn off a child process using the actual
     // "clone" system call. DkThreadCreate allocates a stack for the child

LibOS/shim/src/sys/shim_exec.c

@@ -489,14 +489,12 @@ err:
 
     void * stack     = cur_thread->stack;
     void * stack_top = cur_thread->stack_top;
-    unsigned long fs_base = cur_thread->fs_base;
     shim_tcb_t * shim_tcb = cur_thread->shim_tcb;
     void * frameptr  = cur_thread->frameptr;
 
     cur_thread->stack     = NULL;
     cur_thread->stack_top = NULL;
     cur_thread->frameptr  = NULL;
-    cur_thread->fs_base   = 0;
     cur_thread->shim_tcb  = NULL;
     cur_thread->in_vm     = false;
     unlock(&cur_thread->lock);
@@ -507,7 +505,6 @@ err:
     cur_thread->stack       = stack;
     cur_thread->stack_top   = stack_top;
     cur_thread->frameptr    = frameptr;
-    cur_thread->fs_base     = fs_base;
     cur_thread->shim_tcb    = shim_tcb;
 
     if (ret < 0) {

LibOS/shim/src/sys/shim_fork.c

@@ -84,7 +84,6 @@ int shim_do_fork (void)
     if (!new_thread)
         return -ENOMEM;
 
-    new_thread->fs_base  = cur_thread->fs_base;
     new_thread->shim_tcb = cur_thread->shim_tcb;
     new_thread->tgid     = new_thread->tid;
     new_thread->in_vm    = false;

LibOS/shim/src/sys/shim_vfork.c

@@ -76,7 +76,6 @@ int shim_do_vfork(void) {
     new_thread->is_alive  = true;
     new_thread->stack     = cur_thread->stack;
     new_thread->stack_top = cur_thread->stack_top;
-    new_thread->fs_base   = cur_thread->fs_base;
     cur_thread->stack     = dummy_stack;
     cur_thread->stack_top = dummy_stack + stack_size;
     cur_thread->frameptr  = NULL;