[Pal/{Linux, Linux-SGX}] Introduce PAL_TCB as a common structure

Introduce PAL_TCB common structure for Linux and Linux-SGX PALs such
that LibOS can stash its tcb into PAL_TCB. This commit is a preparation
for statically-linked binary support where the LibOS tcb can be embedded
into per-application-thread structure.

Additionally, SHIM_TCB_USE_GS is introduced as a compile-time option to
enable/disable this feature.

Pal/src/host/Linux-SGX/generated-offsets.c

@@ -64,6 +64,7 @@ void dummy(void)
     DEFINE(SGX_CONTEXT_SIZE, sizeof(sgx_context_t));
 
     /* struct enclave_tls */
+    OFFSET(SGX_COMMON_SELF, enclave_tls, common.self);
     OFFSET(SGX_ENCLAVE_SIZE, enclave_tls, enclave_size);
     OFFSET(SGX_TCS_OFFSET, enclave_tls, tcs_offset);
     OFFSET(SGX_INITIAL_STACK_OFFSET, enclave_tls, initial_stack_offset);

Pal/src/host/Linux-SGX/sgx_main.c

@@ -461,6 +461,10 @@ int initialize_enclave (struct pal_enclave * enclave)
 
             for (uint32_t t = 0 ; t < enclave->thread_num ; t++) {
                 struct enclave_tls * gs = data + pagesize * t;
+                memset(gs, 0, pagesize);
+                assert(sizeof(*gs) <= pagesize);
+                gs->common.self = (PAL_TCB *)(
+                    tls_area->addr + pagesize * t + enclave_secs.baseaddr);
                 gs->enclave_size = enclave->size;
                 gs->tcs_offset = tcs_area->addr + pagesize * t;
                 gs->initial_stack_offset =

Pal/src/host/Linux-SGX/sgx_tls.h

@@ -1,6 +1,8 @@
 #ifndef __SGX_TLS_H__
 #define __SGX_TLS_H__
 
+#include <pal.h>
+
 /*
  * Beside the classic thread local storage (like ustack, thread, etc.) the TLS
  * area is also used to pass parameters needed during enclave or thread
@@ -8,26 +10,30 @@
  * of them are identical for all threads (like enclave_size).
  */
 struct enclave_tls {
-    uint64_t enclave_size;
-    uint64_t tcs_offset;
-    uint64_t initial_stack_offset;
-    void *   aep;
-    void *   ssa;
-    sgx_arch_gpr_t * gpr;
-    void *   exit_target;
-    void *   fsbase;
-    void *   stack;
-    void *   ustack_top;
-    void *   ustack;
-    struct pal_handle_thread * thread;
-    uint64_t ocall_prepared;
-    uint64_t ecall_called;
-    uint64_t ready_for_exceptions;
-    uint64_t manifest_size;
-    void *   heap_min;
-    void *   heap_max;
-    void *   exec_addr;
-    uint64_t exec_size;
+    PAL_TCB common;
+    struct {
+        /* private to Linux-SGX PAL */
+        uint64_t enclave_size;
+        uint64_t tcs_offset;
+        uint64_t initial_stack_offset;
+        void*    aep;
+        void*    ssa;
+        sgx_arch_gpr_t* gpr;
+        void*    exit_target;
+        void*    fsbase;
+        void*    stack;
+        void*    ustack_top;
+        void*    ustack;
+        struct pal_handle_thread* thread;
+        uint64_t ocall_prepared;
+        uint64_t ecall_called;
+        uint64_t ready_for_exceptions;
+        uint64_t manifest_size;
+        void*    heap_min;
+        void*    heap_max;
+        void*    exec_addr;
+        uint64_t exec_size;
+    };
 };
 
 #ifndef DEBUG
@@ -39,15 +45,23 @@ extern uint64_t dummy_debug_variable;
     ({                                                              \
         struct enclave_tls * tmp;                                   \
         uint64_t val;                                               \
+        _Static_assert(sizeof(tmp->member) == 8,                    \
+                       "sgx_tls member should have 8-byte type");   \
         __asm__ ("movq %%gs:%c1, %q0": "=r" (val)                   \
              : "i" (offsetof(struct enclave_tls, member)));         \
         (__typeof(tmp->member)) val;                                \
     })
 #  define SET_ENCLAVE_TLS(member, value)                            \
     do {                                                            \
+        struct enclave_tls * tmp;                                   \
+        _Static_assert(sizeof(tmp->member) == 8,                    \
+                       "sgx_tls member should have 8-byte type");   \
+        _Static_assert(sizeof(value) == 8,                          \
+                       "only 8-byte type can be set to sgx_tls");  \
         __asm__ ("movq %q0, %%gs:%c1":: "r" (value),                \
              "i" (offsetof(struct enclave_tls, member)));           \
     } while (0)
+
 # endif
 
 #endif /* __SGX_TLS_H__ */

Pal/src/host/Linux-SGX/signer/pal-sgx-sign

@@ -420,6 +420,7 @@ def gen_area_content(attr, areas):
         set_tcs_field(t, TCS_FSLIMIT, '<L', 0xfff)
         set_tcs_field(t, TCS_GSLIMIT, '<L', 0xfff)
 
+        set_tls_field(t, SGX_COMMON_SELF, tls_area.addr + PAGESIZE * t + baseaddr())
         set_tls_field(t, SGX_ENCLAVE_SIZE, attr['enclave_size'])
         set_tls_field(t, SGX_TCS_OFFSET, tcs_area.addr + TCS_SIZE * t)
         set_tls_field(t, SGX_INITIAL_STACK_OFFSET, stacks[t].addr + stacks[t].size)

Pal/src/host/Linux/db_exception.c

@@ -271,7 +271,7 @@ static void _DkTerminateSighandler (int signum, siginfo_t * info,
     // If the signal arrives in the middle of a PAL call, add the event
     // to pending in the current TCB.
     if (ADDR_IN_PAL(rip)) {
-        PAL_TCB * tcb = get_tcb();
+        PAL_TCB_LINUX * tcb = get_tcb_linux();
         assert(tcb);
         if (!tcb->pending_event) {
             // Use the preserved pending event slot
@@ -313,7 +313,7 @@ static void _DkPipeSighandler (int signum, siginfo_t * info,
  */
 void __check_pending_event (void)
 {
-    PAL_TCB * tcb = get_tcb();
+    PAL_TCB_LINUX * tcb = get_tcb_linux();
     assert(tcb);
     if (tcb->pending_event) {
         int event = tcb->pending_event;

Pal/src/host/Linux/db_main.c

@@ -239,8 +239,8 @@ void pal_linux_main (void * args)
         INIT_FAIL(PAL_ERROR_NOMEM, "Out of memory");
 
     // Initialize TCB at the top of the alternative stack.
-    PAL_TCB * tcb  = alt_stack + ALT_STACK_SIZE - sizeof(PAL_TCB);
-    tcb->self      = tcb;
+    PAL_TCB_LINUX * tcb = alt_stack + ALT_STACK_SIZE - sizeof(PAL_TCB_LINUX);
+    tcb->common.self = &tcb->common;
     tcb->handle    = first_thread;
     tcb->alt_stack = alt_stack; // Stack bottom
     tcb->callback  = NULL;

Pal/src/host/Linux/db_threading.c

@@ -50,7 +50,7 @@
  */
 int pal_thread_init (void * tcbptr)
 {
-    PAL_TCB * tcb = tcbptr;
+    PAL_TCB_LINUX * tcb = tcbptr;
     int ret;
 
     ret = INLINE_SYSCALL(arch_prctl, 2, ARCH_SET_GS, tcb);
@@ -99,8 +99,8 @@ int _DkThreadCreate (PAL_HANDLE * handle, int (*callback) (void *),
     SET_HANDLE_TYPE(hdl, thread);
 
     // Initialize TCB at the top of the alternative stack.
-    PAL_TCB * tcb  = child_stack + ALT_STACK_SIZE - sizeof(PAL_TCB);
-    tcb->self      = tcb;
+    PAL_TCB_LINUX * tcb  = child_stack + ALT_STACK_SIZE - sizeof(PAL_TCB_LINUX);
+    tcb->common.self = &tcb->common;
     tcb->handle    = hdl;
     tcb->alt_stack = child_stack; // Stack bottom
     tcb->callback  = callback;
@@ -169,7 +169,7 @@ void _DkThreadYieldExecution (void)
 /* _DkThreadExit for internal use: Thread exiting */
 noreturn void _DkThreadExit (void)
 {
-    PAL_TCB* tcb = get_tcb();
+    PAL_TCB_LINUX* tcb = get_tcb_linux();
     PAL_HANDLE handle = tcb->handle;
 
     if (tcb->alt_stack) {

Pal/src/host/Linux/pal_linux.h

@@ -188,27 +188,25 @@ struct event_queue {
 };
 
 DEFINE_LISTP(event_queue);
-typedef struct pal_tcb {
-    struct pal_tcb *  self;
-    int               pending_event;
-    LISTP_TYPE(event_queue) pending_queue;
-    PAL_HANDLE        handle;
-    void *            alt_stack;
-    int               (*callback) (void *);
-    void *            param;
-} PAL_TCB;
+typedef struct pal_tcb_linux {
+    PAL_TCB common;
+    struct {
+        /* private to Linux PAL */
+        int         pending_event;
+        LISTP_TYPE(event_queue) pending_queue;
+        PAL_HANDLE  handle;
+        void *      alt_stack;
+        int         (*callback) (void *);
+        void *      param;
+    };
+} PAL_TCB_LINUX;
 
 noreturn void pal_linux_main (void * args);
 int pal_thread_init (void * tcbptr);
 
-static inline PAL_TCB * get_tcb (void)
+static inline PAL_TCB_LINUX * get_tcb_linux (void)
 {
-    PAL_TCB * tcb;
-    __asm__ (
-         "movq %%gs:%c1,%q0"
-         : "=r" (tcb)
-         : "i" (offsetof(PAL_TCB, self)));
-    return tcb;
+    return (PAL_TCB_LINUX*)pal_get_tcb();
 }
 
 #endif /* PAL_LINUX_H */

Pal/src/pal.h

@@ -84,6 +84,34 @@ typedef union pal_handle
 
 #endif /* !IN_PAL */
 
+/* TODO: introduce configuration system in long term and
+         make SHIM_TCB_USE_GS easily configurable without source code
+         modification */
+//#define SHIM_TCB_USE_GS 1
+#undef SHIM_TCB_USE_GS
+
+#if defined(IN_PAL) || defined(SHIM_TCB_USE_GS)
+#define PAL_LIBOS_TCB_SIZE  256
+
+typedef struct pal_tcb {
+    struct pal_tcb * self;
+    /* uint64_t for alignment */
+#ifdef SHIM_TCB_USE_GS
+    uint64_t libos_tcb[(PAL_LIBOS_TCB_SIZE + sizeof(uint64_t) - 1) / sizeof(uint64_t)];
+#endif
+    /* data private to PAL implementation follows this struct. */
+} PAL_TCB;
+
+static inline PAL_TCB * pal_get_tcb (void)
+{
+    PAL_TCB * tcb;
+    __asm__ ("movq %%gs:%c1,%q0"
+             : "=r" (tcb)
+             : "i" (offsetof(struct pal_tcb, self)));
+    return tcb;
+}
+#endif
+
 typedef struct {
 #ifdef __x86_64__
     PAL_NUM r8, r9, r10, r11, r12, r13, r14, r15;