[LibOS] Comprehensive cleanup of IPC helper thread (shim_ipc_helper.c)

- Removed unnecessary macros (IPC_PORT_IFPOLL, DEBUG_REF, etc).
- Simplified function signatures and changed to better names.
- Removed unused functions (del_ipc_port, del_ipc_port_by_id).
- Malloc instead of huge stack allocations.
- Removed complex logic of exit_with_ipc_helper(), now IPC thread exits
  similarly to Async helper thread.
- Removed ipc_port_pool hash list (used as perf optimization but
  providing no tangible benefit at the cost of high complexity).
- Simplified IPC helper thread states to only ALIVE & NOTALIVE.
- Removed unused broadcast_port.
- Reworked IPC helper thread's while-loop similarly to Async helper
  thread; removed perf optimization of keeping the same list of ports to
  listen on DkObjectsWaitAny(), instead simply repopulate this list
  every time (may become too slow if lots of IPC on hundreds of ports).

LibOS/shim/include/shim_ipc.h

@@ -55,7 +55,6 @@ struct shim_process {
 extern struct shim_process cur_process;
 
 #define IPC_MSG_MINIMAL_SIZE        48
-#define IPC_MSG_READAHEAD           96
 
 struct shim_ipc_msg {
     unsigned char       code;
@@ -91,18 +90,14 @@ struct shim_ipc_port {
     PAL_HANDLE          pal_handle;
 
     REFTYPE             ref_count;
-    LIST_TYPE(shim_ipc_port) hlist;
     LIST_TYPE(shim_ipc_port) list;
     LISTP_TYPE(shim_ipc_msg_obj) msgs;
     struct shim_lock    msgs_lock;
 
     port_fini           fini[MAX_IPC_PORT_FINI_CB];
 
-    bool                update, recent;
-    struct {
-        IDTYPE          type;
-        IDTYPE          vmid;
-    }                   info, private;
+    IDTYPE              type;
+    IDTYPE              vmid;
 };
 
 #define IPC_CALLBACK_ARGS   \
@@ -481,7 +476,7 @@ struct shim_ipc_info * create_ipc_port (IDTYPE vmid, bool listen);
 int create_ipc_location (struct shim_ipc_info ** pinfo);
 
 enum {
-    LISTEN,     /* listening */
+    LISTEN=0,   /* listening */
     SERVER,     /* connect as a server */
     KEEPALIVE,  /* keep the connetion alive */
     DIRCLD,     /* direct child */
@@ -500,21 +495,17 @@ enum {
     NS_PORT_TYPES(SYSV)
 };
 
-#define IPC_PORT_IFPOLL    (IPC_PORT_SERVER|IPC_PORT_LISTEN)
-
 /* general-purpose routines */
 void add_ipc_port_by_id (IDTYPE vmid, PAL_HANDLE hdl, IDTYPE type,
                          port_fini fini,
                          struct shim_ipc_port ** portptr);
 void add_ipc_port (struct shim_ipc_port * port, IDTYPE vmid, IDTYPE type,
                    port_fini fini);
-void del_ipc_port_by_id (IDTYPE vm_pid, IDTYPE type);
-void del_ipc_port (struct shim_ipc_port * port, IDTYPE type);
 void del_ipc_port_fini (struct shim_ipc_port * port, unsigned int exitcode);
 struct shim_ipc_port * lookup_ipc_port (IDTYPE vmid, IDTYPE type);
 void get_ipc_port (struct shim_ipc_port * port);
 void put_ipc_port (struct shim_ipc_port * port);
-void del_all_ipc_ports (IDTYPE type);
+void del_all_ipc_ports (void);
 
 struct shim_ipc_info * get_new_ipc_info (IDTYPE vmid, const char * uri,
                                          size_t len);
@@ -581,15 +572,12 @@ int send_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
                              void * private_data);
 int close_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
                               struct shim_ipc_port * port);
-int broadcast_ipc (struct shim_ipc_msg * msg, struct shim_ipc_port ** exclude,
-                   int exsize, int target_type);
+int broadcast_ipc (struct shim_ipc_msg * msg, int target_type,
+                   struct shim_ipc_port * exclude_port);
 struct shim_ipc_msg_obj * find_ipc_msg_duplex (struct shim_ipc_port * port,
                                                unsigned long seq);
-int receive_ipc_message (struct shim_ipc_port * port, unsigned long seq,
-                         struct shim_ipc_msg ** msg);
 
-/* for convenience */
-int __response_ipc_message (struct shim_ipc_port * port, IDTYPE dest,
+int send_response_ipc_message (struct shim_ipc_port * port, IDTYPE dest,
                             int ret, unsigned long seq);
 
 int do_ipc_duplex (struct shim_ipc_msg_obj * msg,
@@ -599,7 +587,7 @@ int do_ipc_duplex (struct shim_ipc_msg_obj * msg,
 void ipc_child_exit   (struct shim_ipc_port * port, IDTYPE vmid,
                        unsigned int exitcode);
 
-int exit_with_ipc_helper (bool handover, struct shim_thread ** ret);
+struct shim_thread* terminate_ipc_helper(void);
 
 #define IPC_FORCE_RECONNECT     ((void*)-1)
 

LibOS/shim/src/ipc/shim_ipc.c

@@ -48,8 +48,6 @@ struct shim_process cur_process;
 DEFINE_PROFILE_CATEGORY(ipc, );
 DEFINE_PROFILE_OCCURENCE(syscall_use_ipc, ipc);
 
-//#define DEBUG_REF
-
 int init_ipc_ports (void);
 int init_ns_pid    (void);
 int init_ns_sysv   (void);
@@ -115,14 +113,7 @@ struct shim_ipc_info * get_new_ipc_info (IDTYPE vmid, const char * uri,
 
 static void __get_ipc_info (struct shim_ipc_info * info)
 {
-#ifdef DEBUG_REF
-    int ref_count = REF_INC(info->ref_count);
-
-    debug("get port %p (vmid %u uri %s, ref_count = %d)\n", info,
-          info->vmid, qstrgetstr(&info->uri), ref_count);
-#else
     REF_INC(info->ref_count);
-#endif
 }
 
 void get_ipc_info (struct shim_ipc_info * info)
@@ -145,11 +136,6 @@ static void __put_ipc_info (struct shim_ipc_info * info)
 {
     int ref_count = REF_DEC(info->ref_count);
 
-#ifdef DEBUG_REF
-    debug("put port %p (vmid %u uri %s, ref_count = %d)\n", info,
-          info->vmid, qstrgetstr(&info->uri), ref_count);
-#endif
-
     if (ref_count)
         return;
 
@@ -161,11 +147,6 @@ void put_ipc_info (struct shim_ipc_info * info)
 {
     int ref_count = REF_DEC(info->ref_count);
 
-#ifdef DEBUG_REF
-    debug("put port %p (vmid %u uri %s, ref_count = %d)\n", info,
-          info->vmid, qstrgetstr(&info->uri), ref_count);
-#endif
-
     if (ref_count)
         return;
 
@@ -627,7 +608,7 @@ int ipc_checkpoint_send (const char * cpdir, IDTYPE cpsession)
     debug("ipc broadcast to all: IPC_CHECKPOINT(%u, %s)\n",
           cpsession, cpdir);
 
-    ret = broadcast_ipc(msg, NULL, 0, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT);
+    ret = broadcast_ipc(msg, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT, /*exclude_port*/ NULL);
     SAVE_PROFILE_INTERVAL(ipc_checkpoint_send);
     return ret;
 }
@@ -647,7 +628,7 @@ int ipc_checkpoint_callback (IPC_CALLBACK_ARGS)
         goto out;
 
     kill_all_threads(NULL, msgin->cpsession, SIGCP);
-    broadcast_ipc(msg, &port, 1, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD);
+    broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_checkpoint_callback);
     return ret;

LibOS/shim/src/ipc/shim_ipc_child.c

@@ -191,7 +191,7 @@ int ipc_cld_exit_send (IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned
 
     debug("ipc broadcast: IPC_CLD_EXIT(%u, %u, %d)\n", ppid, tid, exitcode);
 
-    ret = broadcast_ipc(msg, NULL, 0, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD);
+    ret = broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, /*exclude_port*/ NULL);
     SAVE_PROFILE_INTERVAL(ipc_cld_exit_send);
     return ret;
 }

LibOS/shim/src/ipc/shim_ipc_nsimpl.h

@@ -1204,7 +1204,7 @@ int NS_SEND(offer) (struct shim_ipc_port * port, IDTYPE dest, IDTYPE base,
     msg->seq     = seq;
 
     debug("ipc send to %u: " NS_CODE_STR(OFFER) "(%u, %u, %lu)\n",
-          port->info.vmid, base, size, lease);
+          port->vmid, base, size, lease);
     ret = send_ipc_message(msg, port);
     SAVE_PROFILE_INTERVAL(NS_SEND(offer));
     return ret;

LibOS/shim/src/ipc/shim_ipc_pid.c

@@ -87,7 +87,7 @@ int broadcast_signal (IDTYPE sender, int signum)
     debug("ipc send to %u: IPC_PID_KILL(%u, %d, %u, %d)\n", 0,
           sender, KILL_ALL, 0, signum);
 
-    ret = broadcast_ipc(msg, NULL, 0, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT);
+    ret = broadcast_ipc(msg, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT, /*exclude_port*/ NULL);
     SAVE_PROFILE_INTERVAL(ipc_pid_kill_send);
     return ret;
 }
@@ -152,7 +152,7 @@ int ipc_pid_kill_callback (IPC_CALLBACK_ARGS)
                                  true);
             break;
         case KILL_ALL:
-            broadcast_ipc(msg, &port, 1, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD);
+            broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, port);
             kill_all_threads(NULL, msgin->sender, msgin->signum);
             break;
     }

LibOS/shim/src/shim_async.c

@@ -276,12 +276,13 @@ static int create_async_helper(void) {
     if (async_helper_state == HELPER_ALIVE)
         return 0;
 
-    enable_locking();
-
     struct shim_thread* new = get_new_internal_thread();
     if (!new)
         return -ENOMEM;
 
+    async_helper_thread = new;
+    async_helper_state = HELPER_ALIVE;
+
     PAL_HANDLE handle = thread_create(shim_async_helper, new);
 
     if (!handle) {
@@ -292,8 +293,6 @@ static int create_async_helper(void) {
     }
 
     new->pal_handle = handle;
-    async_helper_thread = new;
-    async_helper_state = HELPER_ALIVE;
     return 0;
 }
 

LibOS/shim/src/shim_init.c

@@ -1157,7 +1157,7 @@ int shim_clean (int err)
     }
 #endif
 
-    del_all_ipc_ports(0);
+    del_all_ipc_ports();
 
     if (shim_stdio && shim_stdio != (PAL_HANDLE) -1)
         DkObjectClose(shim_stdio);

LibOS/shim/src/sys/shim_exit.c

@@ -147,19 +147,15 @@ int try_process_exit (int error_code, int term_signal)
          */
         put_thread(async_thread); /* free resources of the thread */
 
-    struct shim_thread * ipc_thread;
-    int ret = exit_with_ipc_helper(true, &ipc_thread);
+    struct shim_thread * ipc_thread = terminate_ipc_helper();
     if (ipc_thread)
         /* TODO: wait for the thread to exit in host.
          * This is tracked by the following issue.
          * https://github.com/oscarlab/graphene/issues/440
          */
         put_thread(ipc_thread); /* free resources of the thread */
-    if (!ret)
-        shim_clean(ret);
-    else
-        DkThreadExit();
 
+    shim_clean(0);
     return 0;
 }
 

LibOS/shim/src/sys/shim_msgget.c

@@ -844,7 +844,7 @@ static int __store_msg_persist (struct shim_msg_handle * msgq)
                 struct sysv_client * c = &req->dest;
                 struct msg_req * next = req->next;
 
-                __response_ipc_message(c->port, c->vmid, -EIDRM, c->seq);
+                send_response_ipc_message(c->port, c->vmid, -EIDRM, c->seq);
 
                 put_ipc_port(c->port);
                 __free_msg_qobj(msgq, req);

LibOS/shim/src/sys/shim_sandbox.c

@@ -366,7 +366,7 @@ long shim_do_sandbox_create (int flags, const char * fs_sb,
     }
 
     if (flags & SANDBOX_RPC)
-        del_all_ipc_ports(0);
+        del_all_ipc_ports();
 
     if ((ret = free_config(root_config)) < 0)
         goto err;