[LibOS, Pal/{Linux,Linux-SGX}] Remove DkObjectsWaitAny()

Now Graphene supports an improved version of DkObjectsWaitAny() with
correct polling semantics -- DkObjectsWaitEvents(). This makes
DkObjectsWaitAny() obsolete. This commit removes DkObjectsWaitAny()
and replaces it with:
- DkSynchronizationObjectWait() to wait on a single synchronization
  object like mutex or event.
- DkStreamsWaitEvents() to wait on stream-like objects (this is the
  renamed DkObjectsWaitEvents()).

The corresponding tests are fixed to use the new PAL interfaces.
Also, IPC helper and Async helper threads are significantly refactored
to make better use of DkStreamsWaitEvents().

Documentation/oldwiki/PAL-Host-ABI.md

@@ -520,19 +520,19 @@ This API clears a notification event or a synchronization event.
 
 ### Objects
 
-#### DkObjectsWaitAny
+#### DkSynchronizationObjectWait
 
     #define NO_TIMEOUT ((PAL_NUM)-1)
-    PAL_HANDLE DkObjectsWaitAny(PAL_NUM count, PAL_HANDLE* handle_array, PAL_NUM timeout_us);
+    PAL_BOL DkSynchronizationObjectWait(PAL_HANDLE handle, PAL_NUM timeout_us);
 
-This API polls an array of handles and returns one handle with recent activity. `timeout_us` is
-the maximum time that the API should wait (in microseconds), or `NO_TIMEOUT` to indicate it is to
-be blocked until at least one handle is ready.
+This API waits on a synchronization handle and returns true if this handle's event was triggered and
+false otherwise. `timeout_us` is the maximum time that the API should wait (in microseconds), or
+`NO_TIMEOUT` to indicate it is to be blocked until the handle's event is triggered.
 
-#### DkObjectsWaitEvents
+#### DkStreamsWaitEvents
 
     #define NO_TIMEOUT ((PAL_NUM)-1)
-    PAL_BOL DkObjectsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
+    PAL_BOL DkStreamsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
                                 PAL_FLG* ret_events, PAL_NUM timeout_us);
 
 This API polls an array of handles with user-defined events `events` and returns polled-handles'
@@ -540,9 +540,6 @@ events in `ret_events`. `timeout_us` is the maximum time that the API should wai
 microseconds), or `NO_TIMEOUT` to indicate it is to be blocked until at least one handle is ready.
 It returns true if there was an event on at least one handle and false otherwise.
 
-This API is a more efficient version of `DkObjectsWaitAny()` and closely resembles Linux poll
-semantics. Therefore, `DkObjectsWaitAny()` should be considered deprecated.
-
 #### DkObjectClose
 
     void DkObjectClose(PAL_HANDLE objectHandle);

Documentation/oldwiki/Signal-Handling-in-Graphene.md

@@ -678,14 +678,14 @@ shim_do_alarm(seconds)                          ... no alive host thread ...
 | | + + <creates new thread in host>  +------>  shim_async_helper()
 | |                                             +
 | | set_event(async_helper_event)               | while (true):
-| | +                                           |   DkObjectsWaitAny(array =
+| | +                                           |   DkStreamsWaitEvents(array =
 + + + DkStreamWrite(async_helper_event) +-+     |      { global async_helper_event },
                                           |     |      timeout = <some-constant>)
 ... app-thread code continues ...         |     |   ...
                                           |     |
                                           +-->  |   event = async_list.pop()
                                                 |
-                                                |   DkObjectsWaitAny(...,
+                                                |   DkStreamsWaitEvents(...,
                                                 |      timeout = event.expire_time)
                                                 |
                                                 |   ... sleep until timeout ...

LibOS/shim/include/shim_internal.h

@@ -548,7 +548,9 @@ static void lock(struct shim_lock* l)
     debug("try lock(%s=%p) %s:%d\n", name, l, file, line);
 #endif
 
-    while (!DkObjectsWaitAny(1, &l->lock, NO_TIMEOUT));
+    while (!DkSynchronizationObjectWait(l->lock, NO_TIMEOUT))
+        /* nop */;
+
     l->owner = tcb->tid;
 #if DEBUG_LOCK == 1
     debug("lock(%s=%p) by %s:%d\n", name, l, file, line);
@@ -654,7 +656,7 @@ static inline void wait_event (AEVENTTYPE * e)
         char byte;
         int n = 0;
         do {
-            if (!DkObjectsWaitAny(1, &e->event, NO_TIMEOUT))
+            if (!DkSynchronizationObjectWait(e->event, NO_TIMEOUT))
                 continue;
 
             n = DkStreamRead(e->event, 0, 1, &byte, NULL, 0);

LibOS/shim/include/shim_thread.h

@@ -232,7 +232,7 @@ static inline int thread_sleep (uint64_t timeout_us)
     if (!event)
         return -EINVAL;
 
-    if ( NULL == DkObjectsWaitAny(1, &event, timeout_us))
+    if (!DkSynchronizationObjectWait(event, timeout_us))
         return -PAL_ERRNO;
 
     return 0;

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -618,12 +618,12 @@ out:
  * thread calls receive_ipc_message() if a message arrives on port.
  *
  * Other threads add and remove IPC ports via add_ipc_xxx() and del_ipc_xxx() functions. These ports
- * are added to port_list which the IPC helper thread consults before each new DkObjectsWaitAny().
+ * are added to port_list which the IPC helper thread consults before each DkStreamsWaitEvents().
  *
  * Note that ports are copied from global port_list to local object_list. This is because ports may
  * be removed from port_list by other threads while IPC helper thread is waiting on
- * DkObjectsWaitAny(). For this reason IPC thread also get references to all current ports and puts
- * them after handling all ports in object_list.
+ * DkStreamsWaitEvents(). For this reason IPC thread also get references to all current ports and
+ * puts them after handling all ports in object_list.
  *
  * Previous implementation went to great lengths to keep changes to the list of current ports to a
  * minimum (instead of repopulating the list before each wait like in current code). Unfortunately,
@@ -634,25 +634,22 @@ noreturn static void shim_ipc_helper(void* dummy) {
     __UNUSED(dummy);
     struct shim_thread* self = get_cur_thread();
 
-    PAL_HANDLE polled = NULL;
-
     /* Initialize two lists:
-     * - object_list collects IPC port objects and is the main handled list
-     * - palhandle_list collects corresponding PAL handles of IPC port objects and is needed for
-     *   DkObjectsWaitAny(.., <array-of-PAL-handles>, ..) interface; palhandle_list always contains
-     *   at least install_new_event
-     *
-     * We allocate these two lists on the heap so they do not overflow the limited PAL stack. We
-     * grow them at runtime if needed.
-     */
-    size_t object_list_size    = 0;
-    size_t object_list_maxsize = 32;
-    struct shim_ipc_port** object_list =
-        malloc(sizeof(struct shim_ipc_port*) * object_list_maxsize);
-    PAL_HANDLE* palhandle_list = malloc(sizeof(PAL_HANDLE) * (1 + object_list_maxsize));
-
-    PAL_HANDLE install_new_event_hdl = event_handle(&install_new_event);
-    palhandle_list[0]                = install_new_event_hdl;
+     * - `ports` collects IPC port objects and is the main list we process here
+     * - `pals` collects PAL handles of IPC port objects; always contains install_new_event */
+    size_t ports_cnt = 0;
+    size_t ports_max_cnt = 32;
+    struct shim_ipc_port** ports = malloc(sizeof(*ports) * ports_max_cnt);
+    PAL_HANDLE* pals = malloc(sizeof(*pals) * (1 + ports_max_cnt));
+
+    /* allocate one memory region to hold two PAL_FLG arrays: events and revents */
+    PAL_FLG* pal_events = malloc(sizeof(*pal_events) * (1 + ports_max_cnt) * 2);
+    PAL_FLG* ret_events = pal_events + 1 + ports_max_cnt;
+
+    PAL_HANDLE install_new_event_pal = event_handle(&install_new_event);
+    pals[0]       = install_new_event_pal;
+    pal_events[0] = PAL_WAIT_READ;
+    ret_events[0] = 0;
 
     while (true) {
         lock(&ipc_helper_lock);
@@ -661,94 +658,45 @@ noreturn static void shim_ipc_helper(void* dummy) {
             unlock(&ipc_helper_lock);
             break;
         }
-        unlock(&ipc_helper_lock);
-
-        struct shim_ipc_port* polled_port = NULL;
-
-        if (polled == install_new_event_hdl) {
-            /* some thread wants to install new event; this event is found in object_list below, so
-             * just re-init install_new_event */
-            debug("New IPC event was requested (port was added/removed)\n");
-            clear_event(&install_new_event);
-        } else {
-            /* it is not install_new_event handle, so must be one of ports */
-            for (size_t i = 0; i < object_list_size; i++)
-                if (polled == object_list[i]->pal_handle) {
-                    polled_port = object_list[i];
-                    break;
-                }
-        }
-
-        if (polled_port) {
-            if (polled_port->type & IPC_PORT_SERVER) {
-                /* if polled port is server port, accept a client, create client port, and add it to
-                 * port list */
-                PAL_HANDLE client = DkStreamWaitForClient(polled_port->pal_handle);
-                if (client) {
-                    /* type of client port is the same as original server port but with LISTEN (for
-                     * remote client) and without SERVER (this port doesn't wait for new clients) */
-                    IDTYPE client_type = (polled_port->type & ~IPC_PORT_SERVER) | IPC_PORT_LISTEN;
-                    add_ipc_port_by_id(polled_port->vmid, client, client_type, NULL, NULL);
-                } else {
-                    debug("Port %p (handle %p) was removed during accepting client\n", polled_port,
-                          polled_port->pal_handle);
-                    del_ipc_port_fini(polled_port, -ECHILD);
-                }
-            } else {
-                PAL_STREAM_ATTR attr;
-                if (DkStreamAttributesQueryByHandle(polled_port->pal_handle, &attr)) {
-                    /* can read on this port, so receive messages */
-                    if (attr.readable) {
-                        /* NOTE: IPC helper thread does not handle failures currently */
-                        receive_ipc_message(polled_port);
-                    }
-
-                    if (attr.disconnected) {
-                        debug("Port %p (handle %p) disconnected\n", polled_port,
-                              polled_port->pal_handle);
-                        del_ipc_port_fini(polled_port, -ECONNRESET);
-                    }
-                } else {
-                    debug("Port %p (handle %p) was removed during attr querying\n", polled_port,
-                          polled_port->pal_handle);
-                    del_ipc_port_fini(polled_port, -PAL_ERRNO);
-                }
-            }
-        }
-
-        /* done handling ports; put their references so they can be freed */
-        for (size_t i = 0; i < object_list_size; i++)
-            put_ipc_port(object_list[i]);
 
-        lock(&ipc_helper_lock);
-
-        /* iterate through all ports to repopulate object_list */
-        object_list_size = 0;
+        /* iterate through all known ports from `port_list` to repopulate `ports` */
+        ports_cnt = 0;
         struct shim_ipc_port* port;
         struct shim_ipc_port* tmp;
         LISTP_FOR_EACH_ENTRY_SAFE(port, tmp, &port_list, list) {
             /* get port reference so it is not freed while we wait on/handle it */
             __get_ipc_port(port);
 
-            if (object_list_size == object_list_maxsize) {
-                /* grow object_list and palhandle_list to accomodate more objects */
-                struct shim_ipc_port** tmp_array =
-                    malloc(sizeof(struct shim_ipc_port*) * (object_list_maxsize * 2));
-                PAL_HANDLE* tmp_pal_array =
-                    malloc(sizeof(PAL_HANDLE) * (1 + object_list_maxsize * 2));
-                memcpy(tmp_array, object_list, sizeof(struct shim_ipc_port*) * (object_list_size));
-                memcpy(tmp_pal_array, palhandle_list, sizeof(PAL_HANDLE) * (1 + object_list_size));
-                object_list_maxsize *= 2;
-                free(object_list);
-                free(palhandle_list);
-                object_list    = tmp_array;
-                palhandle_list = tmp_pal_array;
+            if (ports_cnt == ports_max_cnt) {
+                /* grow `ports` and `pals` to accommodate more objects */
+                struct shim_ipc_port** tmp_ports = malloc(sizeof(*tmp_ports) * ports_max_cnt * 2);
+                PAL_HANDLE* tmp_pals    = malloc(sizeof(*tmp_pals) * (1 + ports_max_cnt * 2));
+                PAL_FLG* tmp_pal_events = malloc(sizeof(*tmp_pal_events) * (2 + ports_max_cnt * 4));
+                PAL_FLG* tmp_ret_events = tmp_pal_events + 1 + ports_max_cnt * 2;
+
+                memcpy(tmp_ports, ports, sizeof(*tmp_ports) * ports_max_cnt);
+                memcpy(tmp_pals, pals, sizeof(*tmp_pals) * (1 + ports_max_cnt));
+                memcpy(tmp_pal_events, pal_events, sizeof(*tmp_pal_events) * (1 + ports_max_cnt));
+                memcpy(tmp_ret_events, ret_events, sizeof(*tmp_ret_events) * (1 + ports_max_cnt));
+
+                ports_max_cnt *= 2;
+
+                free(ports);
+                free(pals);
+                free(pal_events);
+
+                ports      = tmp_ports;
+                pals       = tmp_pals;
+                pal_events = tmp_pal_events;
+                ret_events = tmp_ret_events;
             }
 
-            /* re-add this port to object_list and palhandle_list */
-            object_list[object_list_size]        = port;
-            palhandle_list[object_list_size + 1] = port->pal_handle;
-            object_list_size++;
+            /* re-add this port to ports/pals/events */
+            ports[ports_cnt]          = port;
+            pals[ports_cnt + 1]       = port->pal_handle;
+            pal_events[ports_cnt + 1] = PAL_WAIT_READ;
+            ret_events[ports_cnt + 1] = 0;
+            ports_cnt++;
 
             debug("Listening to process %u on port %p (handle %p, type %04x)\n",
                   port->vmid & 0xFFFF, port, port->pal_handle, port->type);
@@ -756,16 +704,67 @@ noreturn static void shim_ipc_helper(void* dummy) {
 
         unlock(&ipc_helper_lock);
 
-        /* wait on collected ports' PAL handles + install_new_event_hdl */
-        polled = DkObjectsWaitAny(object_list_size + 1, palhandle_list, NO_TIMEOUT);
-    }
+        /* wait on collected ports' PAL handles + install_new_event_pal */
+        PAL_BOL polled = DkStreamsWaitEvents(ports_cnt + 1, pals, pal_events, ret_events, NO_TIMEOUT);
+
+        for (size_t i = 0; polled && i < ports_cnt + 1; i++) {
+            if (ret_events[i]) {
+                if (pals[i] == install_new_event_pal) {
+                    /* some thread wants to install new event; this event is found in `ports`, so
+                     * just re-init install_new_event */
+                    debug("New IPC event was requested (port was added/removed)\n");
+                    clear_event(&install_new_event);
+                    continue;
+                }
 
-    /* IPC thread exits; put acquired port references so they can be freed */
-    for (size_t i = 0; i < object_list_size; i++)
-        put_ipc_port(object_list[i]);
+                /* it is not install_new_event handle, so must be one of ports */
+                assert(i > 0);
+                struct shim_ipc_port* polled_port = ports[i - 1];
+                assert(polled_port);
+
+                if (polled_port->type & IPC_PORT_SERVER) {
+                    /* server port: accept client, create client port, and add it to port list */
+                    PAL_HANDLE client = DkStreamWaitForClient(polled_port->pal_handle);
+                    if (client) {
+                        /* type of client port is the same as original server port but with LISTEN
+                         * (for remote client) and without SERVER (doesn't wait for new clients) */
+                        IDTYPE client_type = (polled_port->type & ~IPC_PORT_SERVER) | IPC_PORT_LISTEN;
+                        add_ipc_port_by_id(polled_port->vmid, client, client_type, NULL, NULL);
+                    } else {
+                        debug("Port %p (handle %p) was removed during accepting client\n",
+                              polled_port, polled_port->pal_handle);
+                        del_ipc_port_fini(polled_port, -ECHILD);
+                    }
+                } else {
+                    PAL_STREAM_ATTR attr;
+                    if (DkStreamAttributesQueryByHandle(polled_port->pal_handle, &attr)) {
+                        /* can read on this port, so receive messages */
+                        if (attr.readable) {
+                            /* NOTE: IPC helper thread does not handle failures currently */
+                            receive_ipc_message(polled_port);
+                        }
+                        if (attr.disconnected) {
+                            debug("Port %p (handle %p) disconnected\n",
+                                  polled_port, polled_port->pal_handle);
+                            del_ipc_port_fini(polled_port, -ECONNRESET);
+                        }
+                    } else {
+                        debug("Port %p (handle %p) was removed during attr querying\n",
+                              polled_port, polled_port->pal_handle);
+                        del_ipc_port_fini(polled_port, -PAL_ERRNO);
+                    }
+                }
+            }
+        }
+
+        /* done handling ports; put their references so they can be freed */
+        for (size_t i = 0; i < ports_cnt; i++)
+            put_ipc_port(ports[i]);
+    }
 
-    free(object_list);
-    free(palhandle_list);
+    free(ports);
+    free(pals);
+    free(pal_events);
 
     __disable_preempt(self->shim_tcb);
     put_thread(self);

LibOS/shim/src/shim_async.c

@@ -168,24 +168,22 @@ static void shim_async_helper(void* arg) {
      * to install a new event. */
     uint64_t idle_cycles = 0;
 
-    PAL_HANDLE polled = NULL;
+    /* init `pals` so that it always contains at least install_new_event */
+    size_t pals_max_cnt = 32;
+    PAL_HANDLE* pals = malloc(sizeof(*pals) * (1 + pals_max_cnt));
 
-    /* init object_list so that it always contains at least install_new_event */
-    size_t object_list_size = 32;
-    PAL_HANDLE* object_list = malloc(sizeof(PAL_HANDLE) * (1 + object_list_size));
+    /* allocate one memory region to hold two PAL_FLG arrays: events and revents */
+    PAL_FLG* pal_events = malloc(sizeof(*pal_events) * (1 + pals_max_cnt) * 2);
+    PAL_FLG* ret_events = pal_events + 1 + pals_max_cnt;
 
-    PAL_HANDLE install_new_event_hdl = event_handle(&install_new_event);
-    object_list[0]                   = install_new_event_hdl;
+    PAL_HANDLE install_new_event_pal = event_handle(&install_new_event);
+    pals[0]       = install_new_event_pal;
+    pal_events[0] = PAL_WAIT_READ;
+    ret_events[0] = 0;
 
     while (true) {
         uint64_t now = DkSystemTimeQuery();
 
-        if (polled == install_new_event_hdl) {
-            /* Some thread wants to install new event; this event is found
-             * in async_list below, so just re-init install_new_event. */
-            clear_event(&install_new_event);
-        }
-
         lock(&async_helper_lock);
         if (async_helper_state != HELPER_ALIVE) {
             async_helper_thread = NULL;
@@ -193,62 +191,38 @@ static void shim_async_helper(void* arg) {
             break;
         }
 
-        LISTP_TYPE(async_event) triggered;
-        INIT_LISTP(&triggered);
-
-    again:;
-        /* Iterate through all async IO events and alarm/timer events to:
-         *   - call callbacks for all triggered events, and
-         *   - repopulate object_list with async IO events (if any), and
-         *   - find the next expiring alarm/timer (if any) */
         uint64_t next_expire_time = 0;
-        size_t object_num         = 0;
+        size_t pals_cnt            = 0;
 
         struct async_event* tmp;
         struct async_event* n;
         LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) {
-            /* First check if this event was triggered; there are three types:
-             *   1. Exited child:  trigger callback and remove from the list;
-             *   2. IO events:     trigger callback and keep in the list;
-             *   3. alarms/timers: trigger callback and remove from the list. */
-            if (tmp->callback == &cleanup_thread) {
-                debug("Thread exited, cleaning up\n");
-                LISTP_DEL(tmp, &async_list, list);
-                LISTP_ADD_TAIL(tmp, &triggered, list);
-                continue;
-            } else if (polled && tmp->object == polled) {
-                debug("Async IO event triggered at %lu\n", now);
-                unlock(&async_helper_lock);
-                /* FIXME: potential race condition when
-                 * ioctl(FIOASYNC, off) and cleanup on fd-close are
-                 * correctly implemented. tmp can be freed at the same
-                 * time. */
-                tmp->callback(tmp->caller, tmp->arg);
-                /* async_list may be changed because async_helper_lock is
-                 * released; list traverse cannot be continued. */
-                polled = NULL;
-                lock(&async_helper_lock);
-                goto again;
-            } else if (tmp->expire_time && tmp->expire_time <= now) {
-                debug("Async alarm/timer triggered at %lu (expired at %lu)\n", now,
-                      tmp->expire_time);
-                LISTP_DEL(tmp, &async_list, list);
-                LISTP_ADD_TAIL(tmp, &triggered, list);
-                continue;
-            }
-
-            /* Now re-add this IO event to the list or re-add this timer */
+            /* repopulate `pals` with IO events and find the next expiring alarm/timer */
             if (tmp->object) {
-                if (object_num == object_list_size) {
-                    /* grow object_list to accomodate more objects */
-                    PAL_HANDLE* tmp_array = malloc(sizeof(PAL_HANDLE) * (1 + object_list_size * 2));
-                    memcpy(tmp_array, object_list, sizeof(PAL_HANDLE) * (1 + object_list_size));
-                    object_list_size *= 2;
-                    free(object_list);
-                    object_list = tmp_array;
+                if (pals_cnt == pals_max_cnt) {
+                    /* grow `pals` to accommodate more objects */
+                    PAL_HANDLE* tmp_pals    = malloc(sizeof(*tmp_pals) * (1 + pals_max_cnt * 2));
+                    PAL_FLG* tmp_pal_events = malloc(sizeof(*tmp_pal_events) * (2 + pals_max_cnt * 4));
+                    PAL_FLG* tmp_ret_events = tmp_pal_events + 1 + pals_max_cnt * 2;
+
+                    memcpy(tmp_pals, pals, sizeof(*tmp_pals) * (1 + pals_max_cnt));
+                    memcpy(tmp_pal_events, pal_events, sizeof(*tmp_pal_events) * (1 + pals_max_cnt));
+                    memcpy(tmp_ret_events, ret_events, sizeof(*tmp_ret_events) * (1 + pals_max_cnt));
+
+                    pals_max_cnt *= 2;
+
+                    free(pals);
+                    free(pal_events);
+
+                    pals = tmp_pals;
+                    pal_events = tmp_pal_events;
+                    ret_events = tmp_ret_events;
                 }
-                object_list[object_num + 1] = tmp->object;
-                object_num++;
+
+                pals[pals_cnt + 1]       = tmp->object;
+                pal_events[pals_cnt + 1] = PAL_WAIT_READ;
+                ret_events[pals_cnt + 1] = 0;
+                pals_cnt++;
             } else if (tmp->expire_time && tmp->expire_time > now) {
                 if (!next_expire_time || next_expire_time > tmp->expire_time) {
                     /* use time of the next expiring alarm/timer */
@@ -257,22 +231,12 @@ static void shim_async_helper(void* arg) {
             }
         }
 
-        if (!LISTP_EMPTY(&triggered)) {
-            unlock(&async_helper_lock);
-            LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &triggered, list) {
-                LISTP_DEL(tmp, &triggered, list);
-                tmp->callback(tmp->caller, tmp->arg);
-                free(tmp);
-            }
-            lock(&async_helper_lock);
-        }
-
         uint64_t sleep_time;
         if (next_expire_time) {
             sleep_time  = next_expire_time - now;
             idle_cycles = 0;
-        } else if (object_num) {
-            sleep_time  = NO_TIMEOUT;
+        } else if (pals_cnt) {
+            sleep_time = NO_TIMEOUT;
             idle_cycles = 0;
         } else {
             /* no async IO events and no timers/alarms: thread is idling */
@@ -290,13 +254,70 @@ static void shim_async_helper(void* arg) {
         unlock(&async_helper_lock);
 
         /* wait on async IO events + install_new_event + next expiring alarm/timer */
-        polled = DkObjectsWaitAny(object_num + 1, object_list, sleep_time);
+        PAL_BOL polled = DkStreamsWaitEvents(pals_cnt + 1, pals, pal_events, ret_events, sleep_time);
+
+        now = DkSystemTimeQuery();
+
+        LISTP_TYPE(async_event) triggered;
+        INIT_LISTP(&triggered);
+
+        /* acquire lock because we read/modify async_list below */
+        lock(&async_helper_lock);
+
+        for (size_t i = 0; polled && i < pals_cnt + 1; i++) {
+            if (ret_events[i]) {
+                if (pals[i] == install_new_event_pal) {
+                    /* some thread wants to install new event; this event is found in async_list,
+                     * so just re-init install_new_event */
+                    clear_event(&install_new_event);
+                    continue;
+                }
+
+                /* check if this event is an IO event found in async_list */
+                LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) {
+                    if (tmp->object == pals[i]) {
+                        debug("Async IO event triggered at %lu\n", now);
+                        LISTP_ADD_TAIL(tmp, &triggered, list);
+                        break;
+                    }
+                }
+            }
+        }
+
+        /* check if exit-child or alarm/timer events were triggered */
+        LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &async_list, list) {
+            if (tmp->callback == &cleanup_thread) {
+                debug("Thread exited, cleaning up\n");
+                LISTP_DEL(tmp, &async_list, list);
+                LISTP_ADD_TAIL(tmp, &triggered, list);
+            } else if (tmp->expire_time && tmp->expire_time <= now) {
+                debug("Alarm/timer triggered at %lu (expired at %lu)\n", now, tmp->expire_time);
+                LISTP_DEL(tmp, &async_list, list);
+                LISTP_ADD_TAIL(tmp, &triggered, list);
+            }
+        }
+
+        unlock(&async_helper_lock);
+
+        /* call callbacks for all triggered events */
+        if (!LISTP_EMPTY(&triggered)) {
+            LISTP_FOR_EACH_ENTRY_SAFE(tmp, n, &triggered, list) {
+                LISTP_DEL(tmp, &triggered, list);
+                tmp->callback(tmp->caller, tmp->arg);
+                if (!tmp->object) {
+                    /* this is a one-off exit-child or alarm/timer event */
+                    free(tmp);
+                }
+            }
+        }
     }
 
     __disable_preempt(self->shim_tcb);
     put_thread(self);
     debug("Async helper thread terminated\n");
-    free(object_list);
+
+    free(pals);
+    free(pal_events);
 
     DkThreadExit(/*clear_child_tid=*/NULL);
 }

LibOS/shim/src/shim_object.c

@@ -2,15 +2,15 @@
 #include <shim_internal.h>
 
 int object_wait_with_retry(PAL_HANDLE handle) {
-    PAL_HANDLE ret;
+    PAL_BOL ret;
     do {
-        ret = DkObjectsWaitAny(1, &handle, NO_TIMEOUT);
-    } while (ret == NULL &&
+        ret = DkSynchronizationObjectWait(handle, NO_TIMEOUT);
+    } while (!ret &&
              (PAL_NATIVE_ERRNO == PAL_ERROR_INTERRUPTED || PAL_NATIVE_ERRNO == PAL_ERROR_TRYAGAIN));
-    if (ret == NULL) {
+
+    if (!ret) {
         debug("waiting on %p resulted in error %s", handle, pal_strerror(PAL_NATIVE_ERRNO));
         return -PAL_NATIVE_ERRNO;
     }
-    assert(ret == handle);
     return 0;
 }

LibOS/shim/src/sys/shim_epoll.c

@@ -360,7 +360,7 @@ int shim_do_epoll_wait(int epfd, struct __kernel_epoll_event* events, int maxeve
         unlock(&epoll_hdl->lock);
 
         /* TODO: Timeout must be updated in case of retries; otherwise, we may wait for too long */
-        PAL_BOL polled = DkObjectsWaitEvents(pal_cnt + 1, pal_handles, pal_events, ret_events, timeout_ms * 1000);
+        PAL_BOL polled = DkStreamsWaitEvents(pal_cnt + 1, pal_handles, pal_events, ret_events, timeout_ms * 1000);
 
         lock(&epoll_hdl->lock);
         epoll->waiter_cnt--;

LibOS/shim/src/sys/shim_msgget.c

@@ -707,8 +707,7 @@ int get_sysv_msg(struct shim_msg_handle* msgq, long type, size_t size, void* dat
             break;
 
         unlock(&hdl->lock);
-        while (!DkObjectsWaitAny(1, &msgq->event, NO_TIMEOUT))
-            ;
+        object_wait_with_retry(msgq->event);
         lock(&hdl->lock);
 
         if (!msgq->owned)

LibOS/shim/src/sys/shim_poll.c

@@ -141,7 +141,7 @@ int shim_do_poll(struct pollfd* fds, nfds_t nfds, int timeout_ms) {
 
     unlock(&map->lock);
 
-    PAL_BOL polled = DkObjectsWaitEvents(pal_cnt, pals, pal_events, ret_events, timeout_us);
+    PAL_BOL polled = DkStreamsWaitEvents(pal_cnt, pals, pal_events, ret_events, timeout_us);
 
     /* update fds.revents, but only if something was actually polled */
     if (polled) {

Pal/regression/Event.c

@@ -49,7 +49,7 @@ int main() {
     unsigned long t_start = DkSystemTimeQuery();
 
     pal_printf("Testing wait with too short timeout...\n");
-    DkObjectsWaitAny(1, &event1, 1000000);
+    DkSynchronizationObjectWait(event1, 1000000);
     unsigned long t_wait1  = DkSystemTimeQuery();
     unsigned long dt_wait1 = t_wait1 - t_start;
     pal_printf("Wait returned after %lu us.\n", dt_wait1);
@@ -59,7 +59,7 @@ int main() {
     }
 
     pal_printf("Testing wait with long enough timeout...\n");
-    DkObjectsWaitAny(1, &event1, 5000000);
+    DkSynchronizationObjectWait(event1, 5000000);
     unsigned long t_wait2  = DkSystemTimeQuery();
     unsigned long dt_wait2 = t_wait2 - t_start;
     pal_printf("Wait returned after %lu us since start.\n", dt_wait2);

Pal/regression/Pipe.c

@@ -28,10 +28,6 @@ int main(int argc, char** argv, char** envp) {
         PAL_HANDLE pipe2 = DkStreamOpen("pipe:1", PAL_ACCESS_RDWR, 0, 0, 0);
 
         if (pipe2) {
-            // DEP 10/24/16: We should also be able to wait for a connection
-            //  on this handle
-            // PAL_HANDLE pipe3 = DkObjectsWaitAny(1, &pipe1, 0);
-
             PAL_HANDLE pipe3 = DkStreamWaitForClient(pipe1);
 
             if (pipe3) {

Pal/regression/Semaphore.c

@@ -13,12 +13,11 @@ void helper_timeout(PAL_NUM timeout) {
     }
 
     /* Wait on the binary semaphore with a timeout */
-    PAL_HANDLE rv = DkObjectsWaitAny(1, &sem1, timeout);
-    if (rv == NULL)
+    PAL_BOL rv = DkSynchronizationObjectWait(sem1, timeout);
+    if (rv == PAL_FALSE)
         pal_printf("Locked binary semaphore timed out (%ld).\n", timeout);
     else
-        pal_printf("Acquired locked binary semaphore!?! Got back %p; sem1 is %p (%ld)\n", rv, sem1,
-                   timeout);
+        pal_printf("Acquired locked binary semaphore!?! sem1 is %p (%ld)\n", sem1, timeout);
 
     DkObjectClose(sem1);
 }
@@ -34,11 +33,11 @@ void helper_success(PAL_NUM timeout) {
     }
 
     /* Wait on the binary semaphore with a timeout */
-    PAL_HANDLE rv = DkObjectsWaitAny(1, &sem1, timeout);
-    if (rv == sem1)
+    PAL_BOL rv = DkSynchronizationObjectWait(sem1, timeout);
+    if (rv == PAL_TRUE)
         pal_printf("Locked binary semaphore successfully (%ld).\n", timeout);
     else
-        pal_printf("Failed to lock binary semaphore: Got back %p; sem1 is %p\n", rv, sem1);
+        pal_printf("Failed to lock binary semaphore: sem1 is %p\n", sem1);
 
     DkObjectClose(sem1);
 }

Pal/regression/Symbols.c

@@ -34,6 +34,7 @@ int main(int argc, char** argv, char** envp) {
     PRINT_SYMBOL(DkStreamAttributesSetByHandle);
     PRINT_SYMBOL(DkStreamGetName);
     PRINT_SYMBOL(DkStreamChangeName);
+    PRINT_SYMBOL(DkStreamsWaitEvents);
 
     PRINT_SYMBOL(DkThreadCreate);
     PRINT_SYMBOL(DkThreadDelayExecution);
@@ -50,8 +51,8 @@ int main(int argc, char** argv, char** envp) {
     PRINT_SYMBOL(DkSynchronizationEventCreate);
     PRINT_SYMBOL(DkEventSet);
     PRINT_SYMBOL(DkEventClear);
+    PRINT_SYMBOL(DkSynchronizationObjectWait);
 
-    PRINT_SYMBOL(DkObjectsWaitAny);
     PRINT_SYMBOL(DkObjectClose);
 
     PRINT_SYMBOL(DkSystemTimeQuery);

Pal/regression/Thread.c

@@ -56,9 +56,11 @@ int main(int argc, const char** argv, const char** envp) {
         while (count1 < 100) {
             DkThreadYieldExecution();
         }
-        for (int i = 0; i < 300; i++) {
+        for (int i = 0; i < 500; i++) {
             DkThreadYieldExecution();
         }
+
+        __asm__ volatile("nop" ::: "memory");
         if (count1 == 100)
             pal_printf("Child Thread Exited\n");
     }

Pal/regression/test_pal.py

@@ -191,7 +191,8 @@ class TC_02_Symbols(RegressionTestCase):
         'DkSynchronizationEventCreate',
         'DkEventSet',
         'DkEventClear',
-        'DkObjectsWaitAny',
+        'DkSynchronizationObjectWait',
+        'DkStreamsWaitEvents',
         'DkObjectClose',
         'DkSystemTimeQuery',
         'DkRandomBitsRead',

Pal/src/db_object.c

@@ -68,39 +68,30 @@ void DkObjectClose(PAL_HANDLE objectHandle) {
     LEAVE_PAL_CALL();
 }
 
-/* PAL call DkObjectsWaitAny: wait for any of the handles in the handle array. The wait can be timed
- * out, unless NO_TIMEOUT is given for the timeout_us argument. */
-PAL_HANDLE
-DkObjectsWaitAny(PAL_NUM count, PAL_HANDLE* handle_array, PAL_NUM timeout_us) {
-    ENTER_PAL_CALL(DkObjectsWaitAny);
+/* Wait on a synchronization handle and return true if this handle's event was triggered,
+ * otherwise return false and additionally raise failure. */
+PAL_BOL DkSynchronizationObjectWait(PAL_HANDLE handle, PAL_NUM timeout_us) {
+    ENTER_PAL_CALL(DkSynchronizationObjectWait);
 
-    if (!count || !handle_array) {
+    if (!handle) {
         _DkRaiseFailure(PAL_ERROR_INVAL);
         LEAVE_PAL_CALL_RETURN(NULL);
     }
 
-    for (PAL_NUM i = 0; i < count; i++)
-        if (UNKNOWN_HANDLE(handle_array[i])) {
-            _DkRaiseFailure(PAL_ERROR_INVAL);
-            LEAVE_PAL_CALL_RETURN(NULL);
-        }
-
-    PAL_HANDLE polled = NULL;
-
-    int ret = _DkObjectsWaitAny(count, handle_array, timeout_us, &polled);
+    int ret = _DkSynchronizationObjectWait(handle, timeout_us);
     if (ret < 0) {
         _DkRaiseFailure(-ret);
-        polled = NULL;
+        LEAVE_PAL_CALL_RETURN(PAL_FALSE);
     }
 
-    LEAVE_PAL_CALL_RETURN(polled);
+    LEAVE_PAL_CALL_RETURN(PAL_TRUE);
 }
 
 /* Wait for user-specified events of handles in the handle array. The wait can be timed out, unless
  * NO_TIMEOUT is given in the timeout_us argument. Returns PAL_TRUE if waiting was successful. */
-PAL_BOL DkObjectsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
+PAL_BOL DkStreamsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
                             PAL_FLG* ret_events, PAL_NUM timeout_us) {
-    ENTER_PAL_CALL(DkObjectsWaitEvents);
+    ENTER_PAL_CALL(DkStreamsWaitEvents);
 
     if (!count || !handle_array || !events || !ret_events) {
         _DkRaiseFailure(PAL_ERROR_INVAL);
@@ -114,7 +105,7 @@ PAL_BOL DkObjectsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* ev
         }
     }
 
-    int ret = _DkObjectsWaitEvents(count, handle_array, events, ret_events, timeout_us);
+    int ret = _DkStreamsWaitEvents(count, handle_array, events, ret_events, timeout_us);
     if (ret < 0) {
         _DkRaiseFailure(-ret);
         LEAVE_PAL_CALL_RETURN(PAL_FALSE);

Pal/src/host/Linux-SGX/db_eventfd.c

@@ -125,8 +125,6 @@ static int64_t eventfd_pal_write(PAL_HANDLE handle, uint64_t offset, uint64_t le
         return unix_to_pal_error(ERRNO(bytes));
     }
 
-    /* whether fd is writable or not, gets updated here, to optimize polling logic in
-     * _DkObjectsWaitAny */
     if ((uint64_t)bytes == sizeof(uint64_t))
         HANDLE_HDR(handle)->flags |= writable;
     else

Pal/src/host/Linux-SGX/db_exception.c

@@ -331,8 +331,8 @@ void _DkHandleExternalEvent (PAL_NUM event, sgx_cpu_context_t * uc)
 {
     struct pal_frame * frame = get_frame(uc);
 
-    if (event == PAL_EVENT_RESUME &&
-        frame && frame->func == DkObjectsWaitAny)
+    if (event == PAL_EVENT_RESUME && frame &&
+        (frame->func == DkSynchronizationObjectWait || frame->func == DkStreamsWaitEvents))
         return;
 
     if (!frame) {

Pal/src/host/Linux-SGX/db_object.c

@@ -34,141 +34,22 @@
 #include "pal_linux_defs.h"
 #include "pal_linux_error.h"
 
-/* Wait for an event on any handle in the handle array and return this handle in `polled`. If no
- * ready-event handle was found, `polled` is set to NULL. */
-int _DkObjectsWaitAny(size_t count, PAL_HANDLE* handle_array, int64_t timeout_us,
-                      PAL_HANDLE* polled) {
-    int ret;
-    if (count == 0)
-        return 0;
-
-    if (count == 1 && handle_array[0] &&
-            (IS_HANDLE_TYPE(handle_array[0], mutex) || IS_HANDLE_TYPE(handle_array[0], event))) {
-        /* Special case of DkObjectsWaitAny(1, mutex/event, ...): perform a mutex-specific or
-         * event-specific wait() callback instead of host-OS poll. */
-        const struct handle_ops* ops = HANDLE_OPS(handle_array[0]);
-        assert(ops && ops->wait);
-
-        int rv = ops->wait(handle_array[0], timeout_us);
-        if (!rv)
-            *polled = handle_array[0];
-        return rv;
-    }
-
-    /* Normal case of not mutex/event: poll on all handles in the array (their handle types can be
-     * process, socket, pipe, device, file, eventfd). Note that this function is used only for
-     * Graphene-internal purposes, so we can allocate arrays on stack (since they are small). */
-    struct pollfd fds[count * MAX_FDS];
-    PAL_HANDLE hdls[count * MAX_FDS];
-
-    /* collect all FDs of all PAL handles that may report read/write events */
-    size_t nfds = 0;
-    for (size_t i = 0; i < count; i++) {
-        PAL_HANDLE hdl = handle_array[i];
-        if (!hdl)
-            continue;
-
-        /* ignore duplicate handles */
-        for (size_t j = 0; j < i; j++)
-            if (hdl == handle_array[j])
-                continue;
-
-        /* collect all internal-handle FDs (only those which are readable/writable) */
-        for (size_t j = 0; j < MAX_FDS; j++) {
-            PAL_FLG flags = HANDLE_HDR(hdl)->flags;
-
-            /* hdl might be a mutex/event/non-pollable object, simply ignore it */
-            if (hdl->generic.fds[j] == PAL_IDX_POISON)
-                continue;
-            if (flags & ERROR(j))
-                continue;
-
-            /* always ask host to wait for read event (if FD allows read events); however, no need
-             * to ask host to wait for write event if FD is already known to be writable */
-            int events = 0;
-            events |= (flags & RFD(j)) ? POLLIN : 0;
-            events |= ((flags & WFD(j)) && !(flags & WRITABLE(j))) ? POLLOUT : 0;
-
-            if (events) {
-                fds[nfds].fd      = hdl->generic.fds[j];
-                fds[nfds].events  = events;
-                fds[nfds].revents = 0;
-                hdls[nfds] = hdl;
-                nfds++;
-            }
-        }
-    }
-
-    if (!nfds) {
-        /* did not find any waitable FDs (probably because their events were already cached) */
-        ret = -PAL_ERROR_TRYAGAIN;
-        goto out;
-    }
-
-    ret = ocall_poll(fds, nfds, timeout_us);
+/* Wait on a synchronization handle and return 0 if this handle's event was triggered or error
+ * code otherwise (e.g., due to timeout). */
+int _DkSynchronizationObjectWait(PAL_HANDLE handle, int64_t timeout_us) {
+    assert(IS_HANDLE_TYPE(handle, mutex) || IS_HANDLE_TYPE(handle, event));
 
-    if (IS_ERR(ret)) {
-        switch (ERRNO(ret)) {
-            case EINTR:
-            case ERESTART:
-                ret = -PAL_ERROR_INTERRUPTED;
-                break;
-            default:
-                ret = unix_to_pal_error(ERRNO(ret));
-                break;
-        }
-        goto out;
-    }
-
-    if (!ret) {
-        /* timed out */
-        ret = -PAL_ERROR_TRYAGAIN;
-        goto out;
-    }
-
-    PAL_HANDLE polled_hdl = NULL;
+    const struct handle_ops* ops = HANDLE_OPS(handle);
+    if (!ops || !ops->wait)
+        return -PAL_ERROR_NOTIMPLEMENTED;
 
-    for (size_t i = 0; i < nfds; i++) {
-        if (!fds[i].revents)
-            continue;
-
-        /* One PAL handle can have MAX_FDS internal FDs, so we must select one handle (first one)
-         * from the ones on which the host reported events and then collect all revents on this
-         * handle's internal FDs. Note that this is very inefficient. Each DkObjectsWaitAny()
-         * returns only one of possibly many event-ready PAL handles. */
-        if (!polled_hdl)
-            polled_hdl = hdls[i];
-
-        if (polled_hdl != hdls[i])
-            continue;
-
-        for (size_t j = 0; j < MAX_FDS; j++) {
-            if (!(HANDLE_HDR(polled_hdl)->flags & (RFD(j) | WFD(j))))
-                continue;
-            if (polled_hdl->generic.fds[j] != (PAL_IDX)fds[i].fd)
-                continue;
-
-            /* found internal FD of PAL handle that corresponds to the FD of
-             * event-ready fds[i] */
-            if (fds[i].revents & POLLOUT)
-                HANDLE_HDR(polled_hdl)->flags |= WRITABLE(j);
-            if (fds[i].revents & (POLLHUP | POLLERR))
-                HANDLE_HDR(polled_hdl)->flags |= ERROR(j);
-        }
-    }
-
-    *polled = polled_hdl;
-    ret = polled_hdl ? 0 : -PAL_ERROR_TRYAGAIN;
-out:
-    return ret;
+    return ops->wait(handle, timeout_us);
 }
 
-/* Improved version of _DkObjectsWaitAny(): wait for specific events on all handles in the handle
- * array and return multiple events (including errors) reported by the host.
- * Returns 0 on success,
- * PAL error on failure. */
-int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events,
-                         PAL_FLG* ret_events, int64_t timeout_us) {
+/* Wait for specific events on all handles in the handle array and return multiple events
+ * (including errors) reported by the host. Return 0 on success, PAL error on failure. */
+int _DkStreamsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
+                         int64_t timeout_us) {
     int ret;
 
     if (count == 0)
@@ -250,12 +131,26 @@ int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events
             continue;
 
         size_t j = offsets[i];
+
+        /* update revents */
         if (fds[i].revents & POLLIN)
             ret_events[j] |= PAL_WAIT_READ;
         if (fds[i].revents & POLLOUT)
             ret_events[j] |= PAL_WAIT_WRITE;
         if (fds[i].revents & (POLLHUP | POLLERR | POLLNVAL))
             ret_events[j] |= PAL_WAIT_ERROR;
+
+        /* update handle's internal fields (flags) */
+        PAL_HANDLE hdl = handle_array[j];
+        assert(hdl);
+        for (size_t k = 0; k < MAX_FDS; k++) {
+            if (hdl->generic.fds[k] != (PAL_IDX)fds[i].fd)
+                continue;
+            if (fds[i].revents & POLLOUT)
+                HANDLE_HDR(hdl)->flags |= WRITABLE(k);
+            if (fds[i].revents & (POLLHUP|POLLERR|POLLNVAL))
+                HANDLE_HDR(hdl)->flags |= ERROR(k);
+        }
     }
 
     ret = 0;

Pal/src/host/Linux/db_eventfd.c

@@ -125,8 +125,6 @@ static int64_t eventfd_pal_write(PAL_HANDLE handle, uint64_t offset, uint64_t le
         return unix_to_pal_error(ERRNO(bytes));
     }
 
-    /* whether fd is writable or not, gets updated here, to optimize polling logic in
-     * _DkObjectsWaitAny */
     if ((uint64_t)bytes == sizeof(uint64_t))
         HANDLE_HDR(handle)->flags |= writable;
     else

Pal/src/host/Linux/db_object.c

@@ -34,149 +34,22 @@
 #include "pal_linux.h"
 #include "pal_linux_defs.h"
 
-/* Wait for an event on any handle in the handle array and return this handle in `polled`. If no
- * ready-event handle was found, `polled` is set to NULL. */
-int _DkObjectsWaitAny(size_t count, PAL_HANDLE* handle_array, int64_t timeout_us,
-                      PAL_HANDLE* polled) {
-    int ret;
-
-    if (count == 0)
-        return 0;
-
-    if (count == 1 && handle_array[0] &&
-        (IS_HANDLE_TYPE(handle_array[0], mutex) || IS_HANDLE_TYPE(handle_array[0], event))) {
-        /* Special case of DkObjectsWaitAny(1, mutex/event, ...): perform a mutex-specific or
-         * event-specific wait() callback instead of host-OS poll. */
-        const struct handle_ops* ops = HANDLE_OPS(handle_array[0]);
-        assert(ops && ops->wait);
-
-        int rv = ops->wait(handle_array[0], timeout_us);
-        if (!rv)
-            *polled = handle_array[0];
-        return rv;
-    }
-
-    /* Normal case of not mutex/event: poll on all handles in the array (their handle types can be
-     * process, socket, pipe, device, file, eventfd). Note that this function is used only for
-     * Graphene-internal purposes, so we can allocate arrays on stack (since they are small). */
-    struct pollfd fds[count * MAX_FDS];
-    PAL_HANDLE hdls[count * MAX_FDS];
-
-    /* collect all FDs of all PAL handles that may report read/write events */
-    size_t nfds = 0;
-    for (size_t i = 0; i < count; i++) {
-        PAL_HANDLE hdl = handle_array[i];
-        if (!hdl)
-            continue;
-
-        /* ignore duplicate handles */
-        for (size_t j = 0; j < i; j++)
-            if (hdl == handle_array[j])
-                continue;
-
-        /* collect all internal-handle FDs (only those which are readable/writable) */
-        for (size_t j = 0; j < MAX_FDS; j++) {
-            PAL_FLG flags = HANDLE_HDR(hdl)->flags;
-
-            /* hdl might be a mutex/event/non-pollable object, simply ignore it */
-            if (hdl->generic.fds[j] == PAL_IDX_POISON)
-                continue;
-            if (flags & ERROR(j))
-                continue;
-
-            /* always ask host to wait for read event (if FD allows read events); however, no need
-             * to ask host to wait for write event if FD is already known to be writable */
-            int events = 0;
-            events |= (flags & RFD(j)) ? POLLIN : 0;
-            events |= ((flags & WFD(j)) && !(flags & WRITABLE(j))) ? POLLOUT : 0;
-
-            if (events) {
-                fds[nfds].fd      = hdl->generic.fds[j];
-                fds[nfds].events  = events;
-                fds[nfds].revents = 0;
-                hdls[nfds] = hdl;
-                nfds++;
-            }
-        }
-    }
-
-    if (!nfds) {
-        /* did not find any waitable FDs (probably because their events were already cached) */
-        ret = -PAL_ERROR_TRYAGAIN;
-        goto out;
-    }
-
-    struct timespec timeout_ts;
-
-    if (timeout_us >= 0) {
-        int64_t sec        = timeout_us / 1000000;
-        int64_t microsec   = timeout_us - sec * 1000000;
-        timeout_ts.tv_sec  = sec;
-        timeout_ts.tv_nsec = microsec * 1000;
-    }
-
-    ret = INLINE_SYSCALL(ppoll, 5, fds, nfds, timeout_us >= 0 ? &timeout_ts : NULL, NULL, 0);
-
-    if (IS_ERR(ret)) {
-        switch (ERRNO(ret)) {
-            case EINTR:
-            case ERESTART:
-                ret = -PAL_ERROR_INTERRUPTED;
-                break;
-            default:
-                ret = unix_to_pal_error(ERRNO(ret));
-                break;
-        }
-        goto out;
-    }
-
-    if (!ret) {
-        /* timed out */
-        ret = -PAL_ERROR_TRYAGAIN;
-        goto out;
-    }
-
-    PAL_HANDLE polled_hdl = NULL;
-
-    for (size_t i = 0; i < nfds; i++) {
-        if (!fds[i].revents)
-            continue;
+/* Wait on a synchronization handle and return 0 if this handle's event was triggered or error
+ * code otherwise (e.g., due to timeout). */
+int _DkSynchronizationObjectWait(PAL_HANDLE handle, int64_t timeout_us) {
+    assert(IS_HANDLE_TYPE(handle, mutex) || IS_HANDLE_TYPE(handle, event));
 
-        /* One PAL handle can have MAX_FDS internal FDs, so we must select one handle (first found)
-         * from the ones on which the host reported events and then collect all revents on this
-         * handle's internal FDs. Note that this is very inefficient. Each DkObjectsWaitAny()
-         * returns only one of possibly many event-ready PAL handles. */
-        if (!polled_hdl)
-            polled_hdl = hdls[i];
-
-        if (polled_hdl != hdls[i])
-            continue;
-
-        for (size_t j = 0; j < MAX_FDS; j++) {
-            if (!(HANDLE_HDR(polled_hdl)->flags & (RFD(j) | WFD(j))))
-                continue;
-            if (polled_hdl->generic.fds[j] != (PAL_IDX)fds[i].fd)
-                continue;
+    const struct handle_ops* ops = HANDLE_OPS(handle);
+    if (!ops || !ops->wait)
+        return -PAL_ERROR_NOTIMPLEMENTED;
 
-            /* found internal FD of PAL handle that corresponds to the FD of event-ready fds[i] */
-            if (fds[i].revents & POLLOUT)
-                HANDLE_HDR(polled_hdl)->flags |= WRITABLE(j);
-            if (fds[i].revents & (POLLHUP | POLLERR))
-                HANDLE_HDR(polled_hdl)->flags |= ERROR(j);
-        }
-    }
-
-    *polled = polled_hdl;
-    ret = polled_hdl ? 0 : -PAL_ERROR_TRYAGAIN;
-out:
-    return ret;
+    return ops->wait(handle, timeout_us);
 }
 
-/* Improved version of _DkObjectsWaitAny(): wait for specific events on all handles in the handle
- * array and return multiple events (including errors) reported by the host. Returns 0 on success,
- * PAL error on failure. */
-int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events,
-                         PAL_FLG* ret_events, int64_t timeout_us) {
+/* Wait for specific events on all handles in the handle array and return multiple events
+ * (including errors) reported by the host. Return 0 on success, PAL error on failure. */
+int _DkStreamsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
+                         int64_t timeout_us) {
     int ret;
 
     if (count == 0)
@@ -267,12 +140,26 @@ int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events
             continue;
 
         size_t j = offsets[i];
+
+        /* update revents */
         if (fds[i].revents & POLLIN)
             ret_events[j] |= PAL_WAIT_READ;
         if (fds[i].revents & POLLOUT)
             ret_events[j] |= PAL_WAIT_WRITE;
         if (fds[i].revents & (POLLHUP | POLLERR | POLLNVAL))
             ret_events[j] |= PAL_WAIT_ERROR;
+
+        /* update handle's internal fields (flags) */
+        PAL_HANDLE hdl = handle_array[j];
+        assert(hdl);
+        for (size_t k = 0; k < MAX_FDS; k++) {
+            if (hdl->generic.fds[k] != (PAL_IDX)fds[i].fd)
+                continue;
+            if (fds[i].revents & POLLOUT)
+                HANDLE_HDR(hdl)->flags |= WRITABLE(k);
+            if (fds[i].revents & (POLLHUP|POLLERR|POLLNVAL))
+                HANDLE_HDR(hdl)->flags |= ERROR(k);
+        }
     }
 
     ret = 0;

Pal/src/host/Skeleton/db_object.c

@@ -27,15 +27,15 @@
 #include "pal_error.h"
 #include "pal_internal.h"
 
-/* _DkObjectsWaitAny for internal use. The function wait for any of the handle
-   in the handle array. timeout can be set for the wait. */
-int _DkObjectsWaitAny(size_t count, PAL_HANDLE* handle_array, int64_t timeout_us, PAL_HANDLE* polled) {
+/* Wait on a synchronization handle and return 0 if this handle's event was triggered
+ * or error code otherwise (e.g., due to timeout). */
+int _DkSynchronizationObjectWait(PAL_HANDLE handle, int64_t timeout_us) {
     return -PAL_ERROR_NOTIMPLEMENTED;
 }
 
-/* Improved version of _DkObjectsWaitAny(): wait for specific events on all handles in the handle
- * array and return multiple events (including errors) reported by the host. */
-int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
+/* Wait for specific events on all handles in the handle array and return multiple events
+ * (including errors) reported by the host. Return 0 on success, PAL error on failure. */
+int _DkStreamsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
                          int64_t timeout_us) {
     return -PAL_ERROR_NOTIMPLEMENTED;
 }

Pal/src/pal-symbols

@@ -12,8 +12,8 @@ DkSynchronizationEventCreate
 DkMutexRelease
 DkEventSet
 DkEventClear
-DkObjectsWaitAny
-DkObjectsWaitEvents
+DkSynchronizationObjectWait
+DkStreamsWaitEvents
 DkStreamOpen
 DkStreamRead
 DkStreamWrite

Pal/src/pal.h

@@ -477,28 +477,22 @@ DkNotificationEventCreate (PAL_BOL initialState);
 PAL_HANDLE
 DkSynchronizationEventCreate (PAL_BOL initialState);
 
-/* DkEventDestroy deprecated, replaced by DkObjectClose */
-
 void
 DkEventSet (PAL_HANDLE eventHandle);
 
-/* DkEventWait deprecated, replaced by DkObjectsWaitAny */
-
 void
 DkEventClear (PAL_HANDLE eventHandle);
 
 #define NO_TIMEOUT ((PAL_NUM)-1)
 
-/* Returns: NULL if the call times out, the ready handle on success */
-PAL_HANDLE
-DkObjectsWaitAny (PAL_NUM count, PAL_HANDLE* handle_array, PAL_NUM timeout_us);
+PAL_BOL DkSynchronizationObjectWait(PAL_HANDLE handle, PAL_NUM timeout_us);
 
 #define PAL_WAIT_SIGNAL     1   /* ignored in events */
 #define PAL_WAIT_READ       2
 #define PAL_WAIT_WRITE      4
 #define PAL_WAIT_ERROR      8   /* ignored in events */
 
-PAL_BOL DkObjectsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
+PAL_BOL DkStreamsWaitEvents(PAL_NUM count, PAL_HANDLE* handle_array, PAL_FLG* events,
                             PAL_FLG* ret_events, PAL_NUM timeout_us);
 
 /* Deprecate DkObjectReference */

Pal/src/pal_internal.h

@@ -325,8 +325,8 @@ int _DkVirtualMemoryProtect (void * addr, uint64_t size, int prot);
 /* DkObject calls */
 int _DkObjectReference (PAL_HANDLE objectHandle);
 int _DkObjectClose (PAL_HANDLE objectHandle);
-int _DkObjectsWaitAny(size_t count, PAL_HANDLE* handle_array, int64_t timeout_us, PAL_HANDLE* polled);
-int _DkObjectsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
+int _DkSynchronizationObjectWait(PAL_HANDLE handle, int64_t timeout_us);
+int _DkStreamsWaitEvents(size_t count, PAL_HANDLE* handle_array, PAL_FLG* events, PAL_FLG* ret_events,
                          int64_t timeout_us);
 
 /* DkException calls & structures */

Pal/test/Event.c

@@ -1,5 +1,3 @@
-/* This Hello World demostrate a simple multithread program */
-
 #include "pal.h"
 #include "pal_debug.h"
 
@@ -7,46 +5,45 @@ static PAL_HANDLE event1;
 
 int count = 0;
 
-int thread_1(void* args) {
+int thread_func(void* args) {
     DkThreadDelayExecution(1000);
 
-    pal_printf("In Thread 1\n");
+    pal_printf("In thread 1\n");
 
-    while (count < 100) {
+    while (count < 100)
         count++;
-    }
 
     DkEventSet(event1);
     DkThreadExit(/*clear_child_tid=*/NULL);
-
-    return 0;
+    return 0; /* NOTREACHED */
 }
 
 int main(int argc, char** argv) {
-    pal_printf("Enter Main Thread\n");
+    pal_printf("Enter main thread\n");
 
     PAL_HANDLE thd1;
 
     event1 = DkNotificationEventCreate(0);
-    if (event1 == NULL) {
+    if (!event1) {
         pal_printf("DkNotificationEventCreate failed\n");
         return -1;
     }
 
-    thd1 = DkThreadCreate(&thread_1, 0);
-
-    if (thd1 == NULL) {
+    thd1 = DkThreadCreate(&thread_func, 0);
+    if (!thd1) {
         pal_printf("DkThreadCreate failed\n");
         return -1;
     }
 
-    DkObjectsWaitAny(1, &event1, NO_TIMEOUT);
+    /* wait till thread thd1 is done */
+    DkSynchronizationObjectWait(event1, NO_TIMEOUT);
 
-    if (count < 100)
+    if (count != 100)
         return -1;
 
-    DkObjectsWaitAny(1, &event1, NO_TIMEOUT);
+    /* this wait should return immediately */
+    DkSynchronizationObjectWait(event1, NO_TIMEOUT);
 
-    pal_printf("Leave Main Thread\n");
+    pal_printf("Success, leave main thread\n");
     return 0;
 }

Pal/test/Select.c

@@ -1,24 +1,23 @@
-/* This Hello World demostrate a simple multithread program */
 #include "pal.h"
 #include "pal_debug.h"
 
 PAL_HANDLE wakeup;
 
-int thread(void* args) {
-    pal_printf("Enter Thread\n");
+int thread_func(void* args) {
+    pal_printf("Enter thread\n");
 
     DkThreadDelayExecution(3000000);
-    pal_printf("set event\n");
+    pal_printf("Thread sets event\n");
 
     char byte = 0;
     DkStreamWrite(wakeup, 0, 1, &byte, NULL);
 
-    pal_printf("Leave Thread\n");
+    pal_printf("Leave thread\n");
     return 0;
 }
 
-int main() {
-    pal_printf("Enter Main Thread\n");
+int main(int argc, char** argv) {
+    pal_printf("Enter main thread\n");
 
     PAL_HANDLE handles[3];
     handles[0] = DkStreamOpen("pipe:", PAL_ACCESS_RDWR, 0, 0, 0);
@@ -26,20 +25,26 @@ int main() {
     handles[2] = DkStreamOpen("pipe:", PAL_ACCESS_RDWR, 0, 0, 0);
     wakeup     = handles[2];
 
-    PAL_HANDLE thd = DkThreadCreate(&thread, NULL);
-
-    if (thd == NULL) {
+    PAL_HANDLE thd = DkThreadCreate(&thread_func, NULL);
+    if (!thd) {
         pal_printf("DkThreadCreate failed\n");
         return -1;
     }
 
-    pal_printf("wait on event\n");
+    pal_printf("Waiting on event\n");
+
+    PAL_FLG events[3]  = {PAL_WAIT_READ, PAL_WAIT_READ, PAL_WAIT_READ};
+    PAL_FLG revents[3] = {0, 0, 0};
 
-    PAL_HANDLE hdl = DkObjectsWaitAny(3, handles, NO_TIMEOUT);
+    PAL_BOL polled = DkStreamsWaitEvents(3, handles, events, revents, NO_TIMEOUT);
+    if (!polled) {
+        pal_printf("DkStreamsWaitEvents did not return any events\n");
+        return -1;
+    }
 
-    if (hdl == wakeup)
-        pal_printf("events is called\n");
+    if (revents[2])
+        pal_printf("Event was called\n");
 
-    pal_printf("Leave Main Thread\n");
+    pal_printf("Leave main thread\n");
     return 0;
 }

Pal/test/Server.c

@@ -1,4 +1,4 @@
-/* The server test program that accept multiple TCP connection at the same
+/* The server test program that accepts multiple TCP connections at the same
  * time. A port number is taken as argument. If a port is locked up, try
  * another one.
  *
@@ -7,7 +7,6 @@
  * Start the server:
  *  ../src/libpal.so file:./Server.manifest 4000
  *
- *
  * Run the client:
  *   nc localhost 4000
  *   [ type strings here, see them appear on the console ]
@@ -17,17 +16,18 @@
 #include "pal.h"
 #include "pal_debug.h"
 
+#define MAX_HANDLES 8
+
 int main(int argc, char** argv) {
     if (argc < 2) {
-        pal_printf("specify the port to open\n");
+        pal_printf("Specify the port to open!\n");
         return 0;
     }
 
     char uri[60];
-    snprintf(uri, 60, "tcp.srv:127.0.0.1:%s", argv[1]);
+    snprintf(uri, sizeof(uri), "tcp.srv:127.0.0.1:%s", argv[1]);
 
     PAL_HANDLE srv = DkStreamOpen(uri, PAL_ACCESS_RDWR, 0, PAL_CREATE_TRY, 0);
-
     if (srv == NULL) {
         pal_printf("DkStreamOpen failed\n");
         return -1;
@@ -39,51 +39,57 @@ int main(int argc, char** argv) {
         return -1;
     }
 
-    PAL_HANDLE hdls[8];
-    int nhdls = 1, i;
+    PAL_HANDLE hdls[MAX_HANDLES];
+    PAL_FLG events[MAX_HANDLES];
+    PAL_FLG revents[MAX_HANDLES];
+
+    int nhdls = 1;
     hdls[0]   = srv;
 
     while (1) {
-        PAL_HANDLE hdl = DkObjectsWaitAny(nhdls, hdls, NO_TIMEOUT);
+        for (int i = 0; i < MAX_HANDLES; i++) {
+            events[i]  = PAL_WAIT_READ | PAL_WAIT_WRITE;
+            revents[i] = 0;
+        }
 
-        if (!hdl)
+        PAL_BOL polled = DkStreamsWaitEvents(nhdls, hdls, events, revents, NO_TIMEOUT);
+        if (!polled)
             continue;
 
-        if (hdl == srv) {
-            hdl = DkStreamWaitForClient(srv);
-
-            if (!hdl)
-                continue;
-
-            if (nhdls >= 8) {
-                pal_printf("[ ] connection rejected\n");
-                DkObjectClose(hdl);
-                continue;
+        for (int i = 0; i < MAX_HANDLES; i++) {
+            if (revents[i]) {
+                if (hdls[i] == srv) {
+                    /* event on server -- must be client connecting */
+                    PAL_HANDLE client_hdl = DkStreamWaitForClient(srv);
+                    if (!client_hdl)
+                        continue;
+
+                    if (nhdls >= MAX_HANDLES) {
+                        pal_printf("[ ] connection rejected\n");
+                        DkObjectClose(client_hdl);
+                        continue;
+                    }
+
+                    pal_printf("[%d] receive new connection\n", nhdls);
+                    hdls[nhdls++] = client_hdl;
+                } else if (revents[i] & PAL_WAIT_READ) {
+                    /* event on client -- must read from client */
+                    int bytes = DkStreamRead(hdls[i], 0, 4096, buffer, NULL, 0);
+                    if (bytes == 0) {
+                        DkObjectClose(hdls[i]);
+                        for (int j = i + 1; j < nhdls; j++)
+                            hdls[j - 1] = hdls[j];
+                        nhdls--;
+                        continue;
+                    }
+                    int last_byte = bytes < 4096 ? bytes : 4095;
+                    ((char*)buffer)[last_byte] = 0;
+                    pal_printf("[%d] %s", i, (char*)buffer);
+
+                }
             }
-
-            pal_printf("[%d] receive new connection\n", nhdls);
-            hdls[nhdls++] = hdl;
-            continue;
         }
-
-        int cnt = 0;
-        for (i = 0; i < nhdls; i++)
-            if (hdls[i] == hdl)
-                cnt = i;
-
-        int bytes = DkStreamRead(hdl, 0, 4096, buffer, NULL, 0);
-
-        if (bytes == 0) {
-            DkObjectClose(hdls[cnt]);
-            if (cnt != nhdls - 1)
-                hdls[cnt] = hdls[nhdls - 1];
-            nhdls--;
-            continue;
-        }
-
-        ((char*)buffer)[bytes] = 0;
-
-        pal_printf("[%d] %s", cnt, (char*)buffer);
     }
+
     return 0;
 }

Pal/test/Wait.c

@@ -1,59 +1,65 @@
-/* This Hello World demostrate a simple multithread program */
-
 #include "pal.h"
 #include "pal_debug.h"
 
-PAL_HANDLE event1, event2;
+PAL_HANDLE event1;
+PAL_HANDLE event2;
 
-int thread_1(void* args) {
-    pal_printf("Enter Thread 1\n");
+int thread1_func(void* args) {
+    pal_printf("Enter thread 1\n");
 
     DkThreadDelayExecution(3000);
     DkEventSet(event1);
 
-    pal_printf("Leave Thread 1\n");
+    pal_printf("Leave thread 1\n");
     return 0;
 }
 
-int thread_2(void* args) {
-    pal_printf("Enter Thread 2\n");
+int thread2_func(void* args) {
+    pal_printf("Enter thread 2\n");
 
     DkThreadDelayExecution(5000);
     DkEventSet(event2);
 
-    pal_printf("Leave Thread 2\n");
+    pal_printf("Leave thread 2\n");
     return 0;
 }
 
-int main() {
+int main(int argc, char** argv) {
     pal_printf("Enter Main Thread\n");
 
-    PAL_HANDLE thd1, thd2;
+    PAL_HANDLE thd1;
+    PAL_HANDLE thd2;
 
     event1 = DkNotificationEventCreate(0);
     event2 = DkNotificationEventCreate(0);
 
-    thd1 = DkThreadCreate(&thread_1, 0);
-
-    if (thd1 == NULL) {
+    thd1 = DkThreadCreate(&thread1_func, 0);
+    if (!thd1) {
         pal_printf("DkThreadCreate failed\n");
         return -1;
     }
 
-    thd2 = DkThreadCreate(&thread_2, 0);
-
-    if (thd2 == NULL) {
+    thd2 = DkThreadCreate(&thread2_func, 0);
+    if (!thd2) {
         pal_printf("DkThreadCreate failed\n");
         return -1;
     }
 
-    PAL_HANDLE array[2];
-    array[0] = event1;
-    array[1] = event2;
+    PAL_HANDLE hdls[2] = {event1, event2};
+    PAL_FLG events[2]  = {PAL_WAIT_READ, PAL_WAIT_READ};
+    PAL_FLG revents[2] = {0, 0};
 
-    PAL_HANDLE hdl = DkObjectsWaitAny(2, array, NO_TIMEOUT);
-    pal_printf("event%d is set\n", hdl == event1 ? 1 : 2);
+    PAL_BOL polled = DkStreamsWaitEvents(2, hdls, events, revents, NO_TIMEOUT);
+    if (!polled) {
+        pal_printf("DkStreamsWaitEvents did not return any events\n");
+        return -1;
+    }
+
+    for (int i = 0; i < 2; i++) {
+        if (revents[i])
+            pal_printf("event%d is set\n", i + 1);
+    }
 
-    pal_printf("Leave Main Thread\n");
+    pal_printf("Leave main thread\n");
     return 0;
 }