[LibOS,Pal] Force intermediate process to wait for child after execve()

Previously, Graphene emulated execve() by silently terminating the
intermediate process and creating a new child one. For example,
`bash -c ls` would spawn a new Graphene process with `ls` and exit
the unnecessary "intermediate" process `bash`. This deviation from
standard execve() behavior resulted in the host shell becoming
detached from the Graphene process.

This commit simply forces this intermediate process to wait() until
the child terminates. This way the host shell stays attached, and
all timings and pipes behave correctly.

LibOS/shim/src/shim_init.c

@@ -1159,6 +1159,12 @@ noreturn void shim_clean_and_exit(int exit_code) {
     shim_stdio = NULL;
     debug("process %u exited with status %d\n", cur_process.vmid & 0xFFFF, cur_process.exit_code);
     MASTER_LOCK();
+
+    if (cur_process.exit_code == PAL_WAIT_FOR_CHILDREN_EXIT) {
+        /* user application specified magic exit code; this should be an extremely rare case */
+        debug("exit status collides with Graphene-internal magic status; changed to 1\n");
+        cur_process.exit_code = 1;
+    }
     DkProcessExit(cur_process.exit_code);
 }
 

LibOS/shim/src/sys/shim_exec.c

@@ -521,14 +521,14 @@ reopen:
         return ret;
     }
 
-    /* This "temporary" process must die quietly, not sending any messages to not confuse the parent
-     * and the execve'ed child */
+    /* this "temporary" process must die quietly, not sending any messages to not confuse the parent
+     * and the execve'ed child, but it must still be around until the child finally exits (because
+     * its parent in turn may wait on it, e.g., `bash -c ls`) */
     debug(
-        "Temporary process %u exited after emulating execve (by forking new process to replace this"
-        " one)\n",
-        cur_process.vmid & 0xFFFF);
+        "Temporary process %u exits after emulating execve (by forking new process to replace this"
+        " one); will wait for forked process to exit...\n", cur_process.vmid & 0xFFFF);
     MASTER_LOCK();
-    DkProcessExit(0);
+    DkProcessExit(PAL_WAIT_FOR_CHILDREN_EXIT);
 
     return 0;
 }

LibOS/shim/test/apps

@@ -1 +1 @@
-Subproject commit aa2a35a11bf4e32aeddb31e7bf248bc186ba7aaa
+Subproject commit ac1c0da453b2f95131eb917055206e204542325d

Pal/include/pal/pal.h

@@ -388,6 +388,13 @@ DkVirtualMemoryProtect (PAL_PTR addr, PAL_NUM size, PAL_FLG prot);
 PAL_HANDLE
 DkProcessCreate (PAL_STR uri, PAL_STR * args);
 
+
+/* Magic exit code that instructs the exiting process to wait for its children. Required for a
+ * corner case when the parent exec's the child in a new Graphene process: for correctness,
+ * the parent cannot immediately exit since it may have a parent that waits on it. We hope that
+ * no sane application will pick this magic number as its exit code. */
+#define PAL_WAIT_FOR_CHILDREN_EXIT (1024 * 1024)
+
 noreturn void
 DkProcessExit (PAL_NUM exitCode);
 

Pal/src/host/Linux-SGX/sgx_enclave.c

@@ -21,6 +21,21 @@ static long sgx_ocall_exit(void* pms)
     ms_ocall_exit_t * ms = (ms_ocall_exit_t *) pms;
     ODEBUG(OCALL_EXIT, NULL);
 
+    if (ms->ms_is_exitgroup && ms->ms_exitcode == PAL_WAIT_FOR_CHILDREN_EXIT) {
+        /* this is a "temporary" process exiting after execve'ing a child process: it must still
+         * be around until the child finally exits (because its parent in turn may wait on it) */
+        SGX_DBG(DBG_I, "Temporary process exits after emulating execve, wait for child to exit\n");
+
+        int ret = INLINE_SYSCALL(wait4, 4, /*any child*/-1, /*wstatus=*/NULL, /*options=*/0,
+                                 /*rusage=*/NULL);
+        if (IS_ERR(ret)) {
+            /* it's too late to recover from errors, just log it and continue with dying */
+            SGX_DBG(DBG_I, "Temporary process waited for child to exit but received error %d\n", ret);
+        }
+
+        ms->ms_exitcode = 0;
+    }
+
     if (ms->ms_exitcode != (int) ((uint8_t) ms->ms_exitcode)) {
         SGX_DBG(DBG_E, "Saturation error in exit code %d, getting rounded down to %u\n",
                 ms->ms_exitcode, (uint8_t) ms->ms_exitcode);

Pal/src/host/Linux/db_process.c

@@ -422,6 +422,14 @@ no_data:
 
 noreturn void _DkProcessExit (int exitcode)
 {
+    if (exitcode == PAL_WAIT_FOR_CHILDREN_EXIT) {
+        /* this is a "temporary" process exiting after execve'ing a child process: it must still
+         * be around until the child finally exits (because its parent in turn may wait on it) */
+        INLINE_SYSCALL(wait4, 4, /*any child*/-1, /*wstatus=*/NULL, /*options=*/0,
+                       /*rusage=*/NULL);
+        exitcode = 0;
+    }
+
     INLINE_SYSCALL(exit_group, 1, exitcode);
     while (true) {
         /* nothing */;