[LibOS, Pal] Correctly migrate eventfd descriptors on fork()

LibOS/shim/src/shim_checkpoint.c

@@ -788,10 +788,12 @@ int send_handles_on_stream (PAL_HANDLE stream, struct shim_cp_store * store)
     entries  += cnt;
     nentries -= cnt;
 
-    for (int i = 0 ; i < nentries ; i++)
+    for (int i = 0 ; i < nentries ; i++) {
+        /* We need to abort migration from parent to child if DkSendHandle() returned error,
+         * otherwise the application may fail. */
         if (!DkSendHandle(stream, entries[i]->handle))
-            entries[i]->handle = NULL;
-
+            return -EINVAL;
+    }
     return 0;
 }
 
@@ -828,10 +830,12 @@ int receive_handles_on_stream (struct palhdl_header * hdr, ptr_t base,
         entry = entries[i];
         if (entry->handle) {
             PAL_HANDLE hdl = DkReceiveHandle(PAL_CB(parent_process));
-            if (hdl) {
-                *entry->phandle = hdl;
-                continue;
-            }
+            /* We need to abort migration from parent to child if DkReceiveHandle() returned error,
+             * otherwise the application may fail. */
+            if (!hdl)
+                return -EINVAL;
+
+            *entry->phandle = hdl;
         }
     }
 

LibOS/shim/test/regression/eventfd.c

@@ -4,24 +4,40 @@
 #include <pthread.h>
 
 #include <signal.h>
+#include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdarg.h>
 #include <string.h>
 #include <sys/eventfd.h>
 #include <sys/time.h>
 #include <sys/types.h>
+#include <sys/wait.h>
 #include <unistd.h>
 
 #define MAX_EFDS 3
 
-int efds[MAX_EFDS] = { 0 };
+#define CLOSE_EFD_AND_EXIT(efd) \
+    do {                        \
+        close(efd);             \
+        return 1;               \
+    } while (0)
+
+#define EXIT_IF_ERROR(efd, bytes, prefix)           \
+    do {                                            \
+        if ((bytes) != sizeof(uint64_t)) {          \
+            perror(prefix);                         \
+            printf("error at line-%d\n", __LINE__); \
+            CLOSE_EFD_AND_EXIT(efd);                \
+        }                                           \
+    } while (0)
+
+int efds[MAX_EFDS] = {0};
 
 void* write_eventfd_thread(void* arg) {
     uint64_t count = 10;
 
-    int* efds = (int*) arg;
+    int* efds = (int*)arg;
 
     if (!arg) {
         printf("arg is NULL\n");
@@ -48,9 +64,9 @@ void* write_eventfd_thread(void* arg) {
 int eventfd_using_poll() {
     int ret = 0;
     struct pollfd pollfds[MAX_EFDS];
-    pthread_t tid = 0;
-    uint64_t count = 0;
-    int poll_ret = 0;
+    pthread_t tid    = 0;
+    uint64_t count   = 0;
+    int poll_ret     = 0;
     int nread_events = 0;
 
     for (int i = 0; i < MAX_EFDS; i++) {
@@ -63,7 +79,7 @@ int eventfd_using_poll() {
 
         printf("efd = %d\n", efds[i]);
 
-        pollfds[i].fd = efds[i];
+        pollfds[i].fd     = efds[i];
         pollfds[i].events = POLLIN;
     }
 
@@ -71,7 +87,8 @@ int eventfd_using_poll() {
 
     if (ret != 0) {
         perror("error in thread creation\n");
-        return 1;
+        ret = 1;
+        goto out;
     }
 
     while (1) {
@@ -91,32 +108,38 @@ int eventfd_using_poll() {
         for (int i = 0; i < MAX_EFDS; i++) {
             if (pollfds[i].revents & POLLIN) {
                 pollfds[i].revents = 0;
-                errno = 0;
+                errno              = 0;
                 read(pollfds[i].fd, &count, sizeof(count));
                 printf("fd set=%d\n", pollfds[i].fd);
-                printf("efd = %d, count: %lu, errno=%d\n", pollfds[i].fd,
-                        count, errno);
+                printf("efd = %d, count: %lu, errno=%d\n", pollfds[i].fd, count, errno);
                 nread_events++;
             }
         }
     }
 
-    if (nread_events == MAX_EFDS) {
+    if (nread_events == MAX_EFDS)
         printf("%s completed successfully\n", __func__);
-    } else
+    else
         printf("%s: nread_events=%d, MAX_EFDS=%d\n", __func__, nread_events, MAX_EFDS);
 
     pthread_join(tid, NULL);
+
+out:
+    for (int i = 0; i < MAX_EFDS; i++) {
+        close(efds[i]);
+    }
+
     return ret;
 }
 
-/* This function used to test various flags supported while creating eventfd descriptors.
- * Note: EFD_SEMAPHORE has not been tested.
+/* This function used to test various flags supported while creating eventfd
+ * descriptors.
  * To support regression testing, positive value returned for error case. */
 int eventfd_using_various_flags() {
-    uint64_t count = 0;
-    int efd = 0;
-    int eventfd_flags[] = { 0, EFD_NONBLOCK, EFD_CLOEXEC, EFD_NONBLOCK | EFD_CLOEXEC };
+    uint64_t count      = 0;
+    int efd             = 0;
+    ssize_t bytes       = 0;
+    int eventfd_flags[] = {0, EFD_SEMAPHORE, EFD_NONBLOCK, EFD_CLOEXEC};
 
     for (int i = 0; i < sizeof(eventfd_flags) / sizeof(int); i++) {
         printf("iteration #-%d, flags=%d\n", i, eventfd_flags[i]);
@@ -130,19 +153,46 @@ int eventfd_using_various_flags() {
         }
 
         count = 5;
-        eventfd_write(efd, count);
-        eventfd_write(efd, count);
+        bytes = write(efd, &count, sizeof(count));
+        EXIT_IF_ERROR(efd, bytes, "write");
+
+        bytes = write(efd, &count, sizeof(count));
+        EXIT_IF_ERROR(efd, bytes, "write");
+
+        count = 0;
+        errno = 0;
+        if (eventfd_flags[i] & EFD_SEMAPHORE) {
+            uint64_t prev_count = 0;
+            bytes               = read(efd, &prev_count, sizeof(prev_count));
+            EXIT_IF_ERROR(efd, bytes, "read");
+
+            bytes = read(efd, &count, sizeof(count));
+            EXIT_IF_ERROR(efd, bytes, "read");
+
+            if (prev_count != 1 || count != 1) {
+                printf("flag->EFD_SEMAPHORE, error, prev_count=%lu, new count=%lu\n", prev_count,
+                       count);
+                close(efd);
+                return 1;
+            }
+            continue;
+        }
+
         count = 0;
         errno = 0;
-        eventfd_read(efd, &count);
-        printf("efd = %d, count: %lu, errno=%d\n", efd, count, errno);
+        bytes = read(efd, &count, sizeof(count));
+        EXIT_IF_ERROR(efd, bytes, "read");
+        if (count != 10) {
+            printf("%d: efd = %d, count: %lu, errno=%d\n", __LINE__, efd, count, errno);
+            CLOSE_EFD_AND_EXIT(efd);
+        }
 
         /* calling the second read would block if flags doesn't have EFD_NONBLOCK */
         if (eventfd_flags[i] & EFD_NONBLOCK) {
             count = 0;
             errno = 0;
-            eventfd_read(efd, &count);
-            printf("efd = %d, count: %lu, errno=%d\n", efd, count, errno);
+            read(efd, &count, sizeof(count));
+            printf("%d: efd = %d, count: %lu, errno=%d\n", __LINE__, efd, count, errno);
         }
 
         close(efd);
@@ -153,11 +203,59 @@ int eventfd_using_various_flags() {
     return 0;
 }
 
+int eventfd_using_fork() {
+    int status     = 0;
+    int efd        = 0;
+    uint64_t count = 0;
+
+    efd = eventfd(0, EFD_NONBLOCK);
+
+    if (efd < 0) {
+        perror("eventfd failed");
+        return 1;
+    }
+
+    pid_t pid = fork();
+
+    if (pid == 0) {
+        // child process
+        count = 5;
+        write(efd, &count, sizeof(count));
+        exit(0);
+    } else if (pid > 0) {
+        // parent process
+        waitpid(pid, &status, 0);
+
+        if (WIFSIGNALED(status)) {
+            perror("child was terminated by signal");
+            CLOSE_EFD_AND_EXIT(efd);
+        }
+
+        count = 0;
+        read(efd, &count, sizeof(count));
+        if (count != 5) {
+            printf("parent-pid=%d, efd = %d, count: %lu, errno=%d\n", getpid(), efd, count, errno);
+            CLOSE_EFD_AND_EXIT(efd);
+        }
+
+    } else {
+        perror("fork error");
+        CLOSE_EFD_AND_EXIT(efd);
+    }
+
+    close(efd);
+
+    printf("%s completed successfully\n", __func__);
+
+    return 0;
+}
+
 int main(int argc, char* argv[]) {
     int ret = 0;
 
     ret = eventfd_using_poll();
     ret += eventfd_using_various_flags();
+    ret += eventfd_using_fork();
 
     return ret;
 }

LibOS/shim/test/regression/test_libos.py

@@ -331,6 +331,7 @@ class TC_30_Syscall(RegressionTestCase):
         # Eventfd Test
         self.assertIn('eventfd_using_poll completed successfully', stdout)
         self.assertIn('eventfd_using_various_flags completed successfully', stdout)
+        self.assertIn('eventfd_using_fork completed successfully', stdout)
 
     def test_080_sched(self):
         stdout, stderr = self.run_binary(['sched'])

Pal/src/host/Linux-SGX/db_streams.c

@@ -136,6 +136,7 @@ int handle_serialize(PAL_HANDLE handle, void** data) {
             break;
         case pal_type_gipc:
         case pal_type_process:
+        case pal_type_eventfd:
             break;
         default:
             return -PAL_ERROR_INVAL;
@@ -234,6 +235,7 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) {
         }
         case pal_type_gipc:
         case pal_type_process:
+        case pal_type_eventfd:
             hdl = malloc_copy(hdl_data, hdlsz);
             break;
         default:

Pal/src/host/Linux/db_streams.c

@@ -149,6 +149,7 @@ int handle_serialize(PAL_HANDLE handle, void** data) {
             break;
         case pal_type_gipc:
         case pal_type_process:
+        case pal_type_eventfd:
             break;
         default:
             return -PAL_ERROR_INVAL;
@@ -246,6 +247,7 @@ int handle_deserialize(PAL_HANDLE* handle, const void* data, int size) {
         }
         case pal_type_gipc:
         case pal_type_process:
+        case pal_type_eventfd:
             hdl = malloc_copy(hdl_data, hdlsz);
             break;
         default:

Pal/src/pal_internal.h

@@ -127,7 +127,7 @@ extern const struct handle_ops * pal_handle_ops [];
 static inline const struct handle_ops * HANDLE_OPS (PAL_HANDLE handle)
 {
     int _type = PAL_GET_TYPE(handle);
-    if (_type >= PAL_HANDLE_TYPE_BOUND)
+    if (_type < 0 || _type >= PAL_HANDLE_TYPE_BOUND)
         return NULL;
     return pal_handle_ops[_type];
 }