[LibOS] Preserve RFLAGS in syscall_wrapper

To match Linux syscall ABI, syscall_wrapper should preserve RFLAGS.

LibOS/shim/include/shim_internal.h

@@ -179,6 +179,7 @@ static inline void do_pause (void);
 void handle_signal (bool delayed_only);
 long convert_pal_errno (long err);
 void syscall_wrapper(void);
+void syscall_wrapper_after_syscalldb(void);
 
 #define PAL_ERRNO  convert_pal_errno(PAL_NATIVE_ERRNO)
 

LibOS/shim/src/sys/shim_clone.c

@@ -41,6 +41,30 @@
 #include <linux/sched.h>
 #include <asm/prctl.h>
 
+void __attribute__((weak)) syscall_wrapper_after_syscalldb(void)
+{
+    /*
+     * workaround for linking.
+     * syscalldb.S is excluded for libsysdb_debug.so so it fails to link
+     * due to missing syscall_wrapper_after_syscalldb.
+     */
+}
+
+/*
+ * See syscall_wrapper @ syscalldb.S and illegal_upcall() @ shim_signal.c
+ * for details.
+ * child thread can _not_ use parent stack. So return right after syscall
+ * instruction as if syscall_wrapper is executed.
+ */
+static void fixup_child_context(struct shim_context * context)
+{
+    if (context->ret_ip == &syscall_wrapper_after_syscalldb) {
+        context->sp += RED_ZONE_SIZE;
+        context->regs->rflags = context->regs->r11;
+        context->ret_ip = (void*)context->regs->rcx;
+    }
+}
+
 /* from **sysdeps/unix/sysv/linux/x86_64/clone.S:
    The userland implementation is:
    int clone (int (*fn)(void *arg), void *child_stack, int flags, void *arg),
@@ -146,6 +170,7 @@ int clone_implementation_wrapper(struct clone_args * arg)
     tcb->context.regs = &regs;
     tcb->context.sp = stack;
     tcb->context.ret_ip = return_pc;
+    fixup_child_context(&tcb->context);
 
     restore_context(&tcb->context);
     return 0;

LibOS/shim/src/syscallas.S

@@ -30,7 +30,8 @@
         .extern shim_table, debug_unsupp
         .global syscall_wrapper
         .type syscall_wrapper, @function
-
+        .global syscall_wrapper_after_syscalldb
+        .type syscall_wrapper_after_syscalldb, @function
 
 syscalldb:
         .cfi_startproc
@@ -121,27 +122,37 @@ isundef:
         /*
          * syscall_wrapper: emulate syscall instruction
          *   prohibited in e.g. Linux-SGX PAL which raises a SIGILL exception
+         * See illegal_upcall() @ shim_signal.c and
+         *     fixup_child_context() @ shim_clone.c
          *
          * input:
          * %rcx: Instruction address to continue app execution after trapped
          *       syscall instruction
          * %r11: rflags on entering syscall
-         *
-         * FIXME: preserve rflags.
-         *        remember that clone-child can't use parent stack.
          */
 syscall_wrapper:
         .cfi_startproc
-
+        .cfi_def_cfa %rsp, 0
+        # %rcx is used as input for returning %rip
+        .cfi_register %rip, %rcx
+        # %r11 is used as input to keep %rflags
+        .cfi_register %rflags, %r11
         subq $RED_ZONE_SIZE, %rsp
+        .cfi_adjust_cfa_offset RED_ZONE_SIZE
         callq *syscalldb@GOTPCREL(%rip)
+syscall_wrapper_after_syscalldb:
         addq $RED_ZONE_SIZE, %rsp
-#if 0
-        # TODO: once clone emulation is fixed, remove this #if 0
+        .cfi_adjust_cfa_offset -RED_ZONE_SIZE
+        # restore %rflags for syscall abi compatibility.
+        # This must be done after "addq $RED_ZONE_SIZE, %rsp" above
+        # which destroys %rflags
         xchg %r11, (%rsp)
+        .cfi_offset %rflags, 0
         popfq
+        .cfi_adjust_cfa_offset -8
+        .cfi_same_value %rflags
         pushq %r11
-#endif
+        .cfi_adjust_cfa_offset 8
         jmp *%rcx
 
         .cfi_endproc