[LibOS] Align stack on 16B during user executable start

x86-64 ABI requires 16B alignment of stack on function calls.
To conform to ABI, align stack when jumping to the entry point
of the user executable.

LibOS/shim/include/shim_thread.h

@@ -322,9 +322,6 @@ struct clone_args {
 int clone_implementation_wrapper(struct clone_args * arg);
 
 void * allocate_stack (size_t size, size_t protect_size, bool user);
-int populate_user_stack (void * stack, size_t stack_size,
-                         int nauxv, elf_auxv_t ** auxpp,
-                         const char *** argvp, const char *** envpp);
 
 static inline __attribute__((always_inline))
 bool check_stack_size (struct shim_thread * cur_thread, int size)
@@ -350,7 +347,8 @@ bool check_on_stack (struct shim_thread * cur_thread, void * mem)
     return (mem <= cur_thread->stack_top && mem > cur_thread->stack);
 }
 
-int init_stack (const char ** argv, const char ** envp, const char *** argpp,
+int init_stack (const char ** argv, const char ** envp,
+                int ** argcpp, const char *** argpp,
                 int nauxv, elf_auxv_t ** auxpp);
 
 #endif /* _SHIM_THREAD_H_ */

LibOS/shim/include/shim_utils.h

@@ -216,7 +216,8 @@ int check_elf_object (struct shim_handle * file);
 int load_elf_object (struct shim_handle * file, void * addr, size_t mapped);
 int load_elf_interp (struct shim_handle * exec);
 int free_elf_interp (void);
-int execute_elf_object (struct shim_handle * exec, int argc, const char ** argp,
+int execute_elf_object (struct shim_handle * exec,
+                        int * argcp, const char ** argp,
                         int nauxv, elf_auxv_t * auxp);
 int remove_loaded_libraries (void);
 

LibOS/shim/src/elf/shim_rtld.c

@@ -1558,11 +1558,13 @@ int register_library (const char * name, unsigned long load_address)
     return 0;
 }
 
-int execute_elf_object (struct shim_handle * exec, int argc, const char ** argp,
+int execute_elf_object (struct shim_handle * exec,
+                        int * argcp, const char ** argp,
                         int nauxv, ElfW(auxv_t) * auxp)
 {
     struct link_map * exec_map = __search_map_by_handle(exec);
     assert(exec_map);
+    assert((void*)argcp + sizeof(long) == argp || argp == NULL);
 
     auxp[0].a_type = AT_PHDR;
     auxp[0].a_un.a_val = (__typeof(auxp[0].a_un.a_val)) exec_map->l_phdr;
@@ -1585,14 +1587,11 @@ int execute_elf_object (struct shim_handle * exec, int argc, const char ** argp,
 #if defined(__x86_64__)
     asm volatile (
                     "movq %%rbx, %%rsp\r\n"
-                    "pushq %%rdi\r\n"
                     "jmp *%%rax\r\n"
-
                     :
                     : "a"(entry),
-                    "b"(argp),
-                    "D"(argc),
-                    "d"(0)
+                      "b"(argcp),
+                      "d"(0)
                     : "memory");
 #else
 # error "architecture not supported"

LibOS/shim/src/shim_init.c

@@ -281,12 +281,15 @@ void * allocate_stack (size_t size, size_t protect_size, bool user)
     return stack;
 }
 
-int populate_user_stack (void * stack, size_t stack_size,
-                         int nauxv, elf_auxv_t ** auxpp,
-                         const char *** argvp, const char *** envpp)
+static int populate_user_stack (void * stack, size_t stack_size,
+                                int nauxv, elf_auxv_t ** auxpp,
+                                int ** argcpp,
+                                const char *** argvp, const char *** envpp)
 {
+    const int argc = **argcpp;
     const char ** argv = *argvp, ** envp = *envpp;
     const char ** new_argv = NULL, ** new_envp = NULL;
+    elf_auxv_t *new_auxp = NULL;
     void * stack_bottom = stack;
     void * stack_top = stack + stack_size;
 
@@ -298,6 +301,10 @@ int populate_user_stack (void * stack, size_t stack_size,
     ({ if ((stack_bottom += (size)) > stack_top) return -ENOMEM;    \
        stack_bottom - (size); })
 
+    /* ld.so expects argc as long on stack, not int. */
+    long * argcp = ALLOCATE_BOTTOM(sizeof(long));
+    *argcp = **argcpp;
+
     if (!argv) {
         *(const char **) ALLOCATE_BOTTOM(sizeof(const char *)) = NULL;
         goto copy_envp;
@@ -325,27 +332,34 @@ copy_envp:
     if (!new_envp)
         *(const char **) ALLOCATE_BOTTOM(sizeof(const char *)) = NULL;
 
-    stack_bottom = (void *) ((unsigned long) stack_bottom & ~7UL);
-    *((unsigned long *) ALLOCATE_TOP(sizeof(unsigned long))) = 0;
-
     if (nauxv) {
-        elf_auxv_t * old_auxp = *auxpp;
-        *auxpp = ALLOCATE_TOP(sizeof(elf_auxv_t) * nauxv);
-        if (old_auxp)
-            memcpy(*auxpp, old_auxp, nauxv * sizeof(elf_auxv_t));
+        new_auxp = ALLOCATE_BOTTOM(sizeof(elf_auxv_t) * nauxv);
+        if (*auxpp)
+            memcpy(new_auxp, *auxpp, nauxv * sizeof(elf_auxv_t));
     }
 
-    memmove(stack_top - (stack_bottom - stack), stack, stack_bottom - stack);
-    if (new_argv)
-        *argvp = (void *) new_argv + (stack_top - stack_bottom);
-    if (new_envp)
-        *envpp = (void *) new_envp + (stack_top - stack_bottom);
+    /* x86_64 ABI requires 16 bytes alignment on stack on every function
+       call. */
+    size_t move_size = stack_bottom - stack;
+    *argcpp = stack_top - move_size;
+    *argcpp = ALIGN_DOWN_PTR(*argcpp, 16UL);
+    **argcpp = argc;
+    size_t shift = (void*)(*argcpp) - stack;
+
+    memmove(*argcpp, stack, move_size);
+    *argvp = new_argv ? (void *) new_argv + shift : NULL;
+    *envpp = new_envp ? (void *) new_envp + shift : NULL;
+    *auxpp = new_auxp ? (void *) new_auxp + shift : NULL;
+
+    /* clear working area at the bottom */
+    memset(stack, 0, shift);
     return 0;
 }
 
 unsigned long sys_stack_size = 0;
 
-int init_stack (const char ** argv, const char ** envp, const char *** argpp,
+int init_stack (const char ** argv, const char ** envp,
+                int ** argcpp, const char *** argpp,
                 int nauxv, elf_auxv_t ** auxpp)
 {
     if (!sys_stack_size) {
@@ -371,7 +385,7 @@ int init_stack (const char ** argv, const char ** envp, const char *** argpp,
         envp = initial_envp;
 
     int ret = populate_user_stack(stack, sys_stack_size,
-                                  nauxv, auxpp, &argv, &envp);
+                                  nauxv, auxpp, argcpp, &argv, &envp);
     if (ret < 0)
         return ret;
 
@@ -690,6 +704,7 @@ int shim_init (int argc, void * args, void ** return_stack)
 
     create_lock(__master_lock);
 
+    int * argcp = &argc;
     const char ** argv, ** envp, ** argp = NULL;
     elf_auxv_t * auxp;
 
@@ -763,7 +778,7 @@ restore:
     RUN_INIT(init_mount);
     RUN_INIT(init_important_handles);
     RUN_INIT(init_async);
-    RUN_INIT(init_stack, argv, envp, &argp, nauxv, &auxp);
+    RUN_INIT(init_stack, argv, envp, &argcp, &argp, nauxv, &auxp);
     RUN_INIT(init_loader);
     RUN_INIT(init_ipc_helper);
     RUN_INIT(init_signal);
@@ -813,7 +828,7 @@ restore:
 
     if (cur_thread->exec)
         execute_elf_object(cur_thread->exec,
-                           argc, argp, nauxv, auxp);
+                           argcp, argp, nauxv, auxp);
 
     *return_stack = initial_stack;
     return 0;

LibOS/shim/src/sys/shim_exec.c

@@ -67,6 +67,7 @@ DEFINE_PROFILE_INTERVAL(load_new_executable_for_exec, exec_rtld);
 static void * old_stack_top, * old_stack, * old_stack_red;
 static const char ** new_argp;
 static int           new_argc;
+static int *         new_argcp;
 static elf_auxv_t *  new_auxp;
 
 #define REQUIRED_ELF_AUXV       6
@@ -110,7 +111,8 @@ int shim_do_execve_rtld (struct shim_handle * hdl, const char ** argv,
     new_argc = 0;
     for (const char ** a = argv ; *a ; a++, new_argc++);
 
-    if ((ret = init_stack(argv, envp, &new_argp,
+    new_argcp = &new_argc;
+    if ((ret = init_stack(argv, envp, &new_argcp, &new_argp,
                           REQUIRED_ELF_AUXV, &new_auxp)) < 0)
         return ret;
 
@@ -195,7 +197,7 @@ retry_dump_vmas:
 #endif
 
     debug("execve: start execution\n");
-    execute_elf_object(cur_thread->exec, new_argc, new_argp,
+    execute_elf_object(cur_thread->exec, new_argcp, new_argp,
                        REQUIRED_ELF_AUXV, new_auxp);
 
     return 0;