[LibOS] Comprehensive cleanup of parent-child IPC (shim_ipc_child.c)

- Removed unused IPC_CLD_JOIN and corresponding functions
  ipc_cld_join_send/ipc_cld_join_callback.
- Renamed poorly named ipc_child_exit to ipc_port_with_child_fini.
- Added thread locking during child_sthread_exit/child_thread_exit.
- Added extensive comments.

LibOS/shim/include/shim_ipc.h

@@ -107,7 +107,7 @@ struct shim_ipc_port {
    succeed. */
 #define RESPONSE_CALLBACK   1
 
-typedef int (*ipc_callback) (IPC_CALLBACK_ARGS);
+typedef int (*ipc_callback)(struct shim_ipc_msg* msg, struct shim_ipc_port* port);
 
 /* Basic message codes */
 enum {
@@ -127,14 +127,13 @@ struct shim_ipc_checkpoint {
     char cpdir[1];
 } __attribute__((packed));
 
-int ipc_checkpoint_send (const char * cpdir, IDTYPE cpsession);
-int ipc_checkpoint_callback (IPC_CALLBACK_ARGS);
+int ipc_checkpoint_send(const char* cpdir, IDTYPE cpsession);
+int ipc_checkpoint_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port);
 
 /* Message code from child to parent */
 #define IPC_CLD_BASE       IPC_BASE_BOUND
 enum {
     IPC_CLD_EXIT = IPC_CLD_BASE,
-    IPC_CLD_JOIN,
 #ifdef PROFILE
     IPC_CLD_PROFILE,
 #endif
@@ -151,12 +150,8 @@ struct shim_ipc_cld_exit {
 #endif
 } __attribute__((packed));
 
-int ipc_cld_exit_send (IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned int term_signal);
-int ipc_cld_exit_callback (IPC_CALLBACK_ARGS);
-
-/* CLD_JOIN: child join the parent group */
-int ipc_cld_join_send (IDTYPE dest);
-int ipc_cld_join_callback (IPC_CALLBACK_ARGS);
+int ipc_cld_exit_send(IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned int term_signal);
+int ipc_cld_exit_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port);
 
 #ifdef PROFILE
 # include <shim_profile.h>
@@ -167,8 +162,8 @@ struct shim_ipc_cld_profile {
     struct profile_val profile[];
 } __attribute__((packed));
 
-int ipc_cld_profile_send (void);
-int ipc_cld_profile_callback (IPC_CALLBACK_ARGS);
+int ipc_cld_profile_send(void);
+int ipc_cld_profile_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port);
 #endif
 
 /* Message code to namespace manager */
@@ -450,8 +445,8 @@ int ipc_sysv_semreply_callback (IPC_CALLBACK_ARGS);
 #define IPC_CODE_NUM     IPC_SYSV_BOUND
 
 /* functions and routines */
-int init_ipc (void);
-int init_ipc_helper (void);
+int init_ipc(void);
+int init_ipc_helper(void);
 
 struct shim_process* create_process(void);
 void free_process(struct shim_process* process);
@@ -480,16 +475,14 @@ enum {
 };
 
 /* general-purpose routines */
-void add_ipc_port_by_id (IDTYPE vmid, PAL_HANDLE hdl, IDTYPE type,
-                         port_fini fini,
-                         struct shim_ipc_port ** portptr);
-void add_ipc_port (struct shim_ipc_port * port, IDTYPE vmid, IDTYPE type,
-                   port_fini fini);
-void del_ipc_port_fini (struct shim_ipc_port * port, unsigned int exitcode);
-struct shim_ipc_port * lookup_ipc_port (IDTYPE vmid, IDTYPE type);
-void get_ipc_port (struct shim_ipc_port * port);
-void put_ipc_port (struct shim_ipc_port * port);
-void del_all_ipc_ports (void);
+void add_ipc_port_by_id(IDTYPE vmid, PAL_HANDLE hdl, IDTYPE type,
+                        port_fini fini, struct shim_ipc_port** portptr);
+void add_ipc_port(struct shim_ipc_port * port, IDTYPE vmid, IDTYPE type, port_fini fini);
+void del_ipc_port_fini(struct shim_ipc_port * port, unsigned int exitcode);
+struct shim_ipc_port* lookup_ipc_port(IDTYPE vmid, IDTYPE type);
+void get_ipc_port(struct shim_ipc_port* port);
+void put_ipc_port(struct shim_ipc_port* port);
+void del_all_ipc_ports(void);
 
 struct shim_ipc_info* create_ipc_info(IDTYPE vmid, const char* uri, size_t len);
 void get_ipc_info(struct shim_ipc_info* port);
@@ -520,13 +513,12 @@ int send_ipc_message_duplex(struct shim_ipc_msg_duplex* msg, struct shim_ipc_por
                             unsigned long* seq, void* private_data);
 int send_response_ipc_message(struct shim_ipc_port* port, IDTYPE dest, int ret, unsigned long seq);
 
-void ipc_child_exit   (struct shim_ipc_port * port, IDTYPE vmid,
-                       unsigned int exitcode);
+void ipc_port_with_child_fini(struct shim_ipc_port* port, IDTYPE vmid, unsigned int exitcode);
 
 struct shim_thread* terminate_ipc_helper(void);
 
 #define IPC_FORCE_RECONNECT     ((void*)-1)
 
-int prepare_ns_leaders (void);
+int prepare_ns_leaders(void);
 
 #endif /* _SHIM_IPC_H_ */

LibOS/shim/src/ipc/shim_ipc_child.c

@@ -15,7 +15,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
 
 /*
- * shim_ipc_helper.c
+ * shim_ipc_child.c
  *
  * This file contains functions and callbacks to handle IPC between parent
  * processes and their children.
@@ -30,249 +30,236 @@
 
 #include <pal.h>
 #include <pal_error.h>
-
 #include <errno.h>
 
-static int ipc_thread_exit (IDTYPE vmid, IDTYPE tid, unsigned int exitcode,
-                            unsigned int term_signal, unsigned long exit_time)
-{
-    assert(vmid != cur_process.vmid);
-
-#ifdef PROFILE
-    if (!exit_time)
-        exit_time = GET_PROFILE_INTERVAL();
-#else
-    __UNUSED(exit_time);
-#endif
-
-    struct shim_thread * thread = __lookup_thread(tid);
-
-    if (thread) {
-        int ret = 0;
-        //assert(thread->vmid == vmid && !thread->in_vm);
-        thread->exit_code = -exitcode;
-        thread->term_signal = term_signal;
-#ifdef PROFILE
-        thread->exit_time = exit_time;
-#endif
-        ret = thread_exit(thread, false);
-        put_thread(thread);
-        return ret;
-    }
-
-    struct shim_simple_thread * sthread = __lookup_simple_thread(tid);
-
-    if (!sthread) {
-        sthread = get_new_simple_thread();
-        sthread->vmid = vmid;
-        sthread->tid = tid;
-        add_simple_thread(sthread);
-    }
-
-    sthread->is_alive = 0;
-    sthread->exit_code = -exitcode;
-    sthread->term_signal = term_signal;
-#ifdef PROFILE
-    sthread->exit_time = exit_time;
-#endif
-    DkEventSet(sthread->exit_event);
-    put_simple_thread(sthread);
-    return 0;
-}
-
 struct thread_info {
     IDTYPE vmid;
     unsigned int exitcode;
     unsigned int term_signal;
 };
 
-static int child_sthread_exit (struct shim_simple_thread * thread, void * arg,
-                               bool * unlocked)
-{
-    __UNUSED(unlocked); // Used by other callbacks
-    struct thread_info * info = (struct thread_info *) arg;
+/* walk_simple_thread_list callback; exit each simple thread of child process vmid. */
+static int child_sthread_exit(struct shim_simple_thread* thread, void* arg, bool* unlocked) {
+    __UNUSED(unlocked); /* FYI: notifies about unlocked thread_list_lock */
+
+    struct thread_info* info = (struct thread_info *) arg;
+    int found_exiting_thread = 0;
+
+    lock(&thread->lock);
     if (thread->vmid == info->vmid) {
+        found_exiting_thread = 1;
+
         if (thread->is_alive) {
             thread->exit_code = -info->exitcode;
             thread->term_signal = info->term_signal;
             thread->is_alive = false;
+
+            /* arrange exit event for subsequent wait4(thread->tid) */
             DkEventSet(thread->exit_event);
         }
-        return 1;
     }
-    return 0;
+    unlock(&thread->lock);
+    return found_exiting_thread;
 }
 
-static int child_thread_exit (struct shim_thread * thread, void * arg,
-                              bool * unlocked)
-{
-    __UNUSED(unlocked); // Used by other callbacks
-    struct thread_info * info = (struct thread_info *) arg;
+/* walk_thread_list callback; exit each thread of child process vmid. */
+static int child_thread_exit(struct shim_thread* thread, void* arg, bool* unlocked) {
+    __UNUSED(unlocked); /* FYI: notifies about unlocked thread_list_lock */
+
+    struct thread_info* info = (struct thread_info *) arg;
+    int found_exiting_thread = 0;
+
+    lock(&thread->lock);
     if (thread->vmid == info->vmid) {
+        found_exiting_thread = 1;
+
         if (thread->is_alive) {
             thread->exit_code = -info->exitcode;
             thread->term_signal = info->term_signal;
+            unlock(&thread->lock);
+
+            /* remote thread is "virtually" exited: SIGCHLD is generated for
+             * the parent thread and exit events are arranged for subsequent
+             * wait4(). */
             thread_exit(thread, false);
+            goto out;
         }
-        return 1;
     }
-    return 0;
+    unlock(&thread->lock);
+
+out:
+    return found_exiting_thread;
 }
 
-int remove_child_thread (IDTYPE vmid, unsigned int exitcode, unsigned int term_signal)
-{
-    struct thread_info info = { .vmid = vmid, .exitcode = exitcode, .term_signal = term_signal };
-    int nkilled = 0, ret;
+/* IPC helper thread invokes this fini function when main IPC port for
+ * communication with child process is disconnected/removed by host OS.
+ *
+ * Similarly to benign case of receiving an explicit IPC_CLD_EXIT message
+ * from exiting remote thread (see ipc_cld_exit_callback()), we want to
+ * delete all remote threads associated with disconnected child process.
+ */
+void ipc_port_with_child_fini(struct shim_ipc_port* port, IDTYPE vmid, unsigned int exitcode) {
+    __UNUSED(port);
 
+    /* NOTE: IPC port may be closed by host OS because the child process
+     *       exited on host OS (and so host OS closed all its sockets).
+     *       This may happen before arrival of the "expected" IPC_CLD_EXIT
+     *       message from child process. Ideally, we would inspect whether
+     *       we previously sent SIGINT/SIGTERM/SIGKILL to this child and
+     *       use the corresponding termination signal. For now, we simply
+     *       report that child process was killed by SIGKILL. */
+    struct thread_info info = { .vmid = vmid, .exitcode = exitcode, .term_signal = SIGKILL };
+
+    /* message cannot come from our own threads (from ourselves as process) */
     assert(vmid != cur_process.vmid);
 
-    if ((ret = walk_thread_list(&child_thread_exit, &info)) > 0)
-        nkilled += ret;
+    int ret;
+    int exited_threads_cnt = 0;
 
+    if ((ret = walk_thread_list(&child_thread_exit, &info)) > 0)
+        exited_threads_cnt += ret;
     if ((ret = walk_simple_thread_list(&child_sthread_exit, &info)) > 0)
-        nkilled += ret;
-
-    if (!nkilled)
-        debug("child port closed, no thread exited\n");
-
-    return 0;
-}
+        exited_threads_cnt += ret;
 
-void ipc_child_exit (struct shim_ipc_port * port, IDTYPE vmid,
-                     unsigned int exitcode)
-{
-    debug("ipc port %p of process %u closed suggests child exiting\n",
-          port, vmid);
-
-    /*
-     * Chia-Che 12/12/2017:
-     * Can't assume there is a termination signal. this callback
-     * is only called when the child process is not responding, and
-     * under this circumstance can only assume the child process
-     * has encountered severe failure, hence SIGKILL.
-     */
-    remove_child_thread(vmid, exitcode, SIGKILL);
-}
-
-static struct shim_ipc_port * get_parent_port (IDTYPE * dest)
-{
-    struct shim_ipc_port * port = NULL;
-    lock(&cur_process.lock);
-    if (cur_process.parent && (port = cur_process.parent->port)) {
-        get_ipc_port(port);
-        *dest = cur_process.parent->vmid;
-    }
-    unlock(&cur_process.lock);
-    return port;
+    debug("Child process %u got disconnected: assume that child exited and "
+          "force %d of its threads to exit\n", vmid & 0xFFFF, exited_threads_cnt);
 }
 
 DEFINE_PROFILE_INTERVAL(ipc_cld_exit_turnaround, ipc);
 DEFINE_PROFILE_INTERVAL(ipc_cld_exit_send, ipc);
 DEFINE_PROFILE_INTERVAL(ipc_cld_exit_callback, ipc);
 
-int ipc_cld_exit_send (IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned int term_signal)
-{
+/* The exiting thread of this process calls this function to broadcast
+ * IPC_CLD_EXIT notification to its parent process (technically, to all
+ * processes of type DIRPRT or DIRCLD but the only interesting case is
+ * the notification of parent). */
+int ipc_cld_exit_send(IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned int term_signal) {
     __attribute__((unused)) unsigned long send_time = GET_PROFILE_INTERVAL();
     BEGIN_PROFILE_INTERVAL_SET(send_time);
-    int ret = 0;
 
     size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_cld_exit));
     struct shim_ipc_msg* msg = __alloca(total_msg_size);
     init_ipc_msg(msg, IPC_CLD_EXIT, total_msg_size, 0);
 
-    struct shim_ipc_cld_exit * msgin =
-                (struct shim_ipc_cld_exit *) &msg->msg;
-    msgin->ppid = ppid;
-    msgin->tid = tid;
-    msgin->exitcode = exitcode;
+    struct shim_ipc_cld_exit* msgin = (struct shim_ipc_cld_exit *) &msg->msg;
+    msgin->ppid        = ppid;
+    msgin->tid         = tid;
+    msgin->exitcode    = exitcode;
     msgin->term_signal = term_signal;
 #ifdef PROFILE
-    msgin->time = send_time;
+    msgin->time        = send_time;
 #endif
 
-    debug("ipc broadcast: IPC_CLD_EXIT(%u, %u, %d)\n", ppid, tid, exitcode);
+    debug("IPC broadcast: IPC_CLD_EXIT(%u, %u, %d, %u)\n",
+          ppid, tid, exitcode, term_signal);
 
-    ret = broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, /*exclude_port*/ NULL);
+    int ret = broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, /*exclude_port*/ NULL);
     SAVE_PROFILE_INTERVAL(ipc_cld_exit_send);
     return ret;
 }
 
-int ipc_cld_exit_callback (IPC_CALLBACK_ARGS)
-{
-    // XXX: Should we close/free port?
+/* IPC helper thread invokes this callback on an IPC_CLD_EXIT mesage received
+ * from a specific thread msgin->tid of the exiting child process with vmid
+ * msg->src. The thread of the exiting child process informs about its exit
+ * code in msgin->exit_code and its terminating signal in msgin->term_signal.
+ *
+ * The callback finds this remote thread of the child process among our
+ * process's threads/simple threads (recall that parent process maintains
+ * remote child threads in its thread list, marking them as in_vm == false).
+ * The remote thread is "virtually" exited: SIGCHLD is generated for the
+ * parent thread and exit events are arranged for subsequent wait4().
+ */
+int ipc_cld_exit_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port) {
     __UNUSED(port);
+    int ret = 0;
 
-    struct shim_ipc_cld_exit * msgin =
-                (struct shim_ipc_cld_exit *) &msg->msg;
+    struct shim_ipc_cld_exit* msgin = (struct shim_ipc_cld_exit *) &msg->msg;
 
 #ifdef PROFILE
     unsigned long time = msgin->time;
-#else
-    unsigned long time = 0;
+    if (!time)
+        time = GET_PROFILE_INTERVAL();
 #endif
     BEGIN_PROFILE_INTERVAL_SET(time);
     SAVE_PROFILE_INTERVAL(ipc_cld_exit_turnaround);
 
-    debug("ipc callback from %u: IPC_CLD_EXIT(%u, %u, %d)\n",
-          msg->src, msgin->ppid, msgin->tid, msgin->exitcode);
+    debug("IPC callback from %u: IPC_CLD_EXIT(%u, %u, %d, %u)\n",
+          msg->src & 0xFFFF, msgin->ppid, msgin->tid, msgin->exitcode, msgin->term_signal);
 
-    // XXX: Assert that we are the msgin->ppid?
+    /* message cannot come from our own threads (from ourselves as process) */
+    assert(msg->src != cur_process.vmid);
 
-    int ret = ipc_thread_exit(msg->src, msgin->tid,
-                              msgin->exitcode, msgin->term_signal,
-                              time);
-    SAVE_PROFILE_INTERVAL(ipc_cld_exit_callback);
-    return ret;
-}
-
-DEFINE_PROFILE_INTERVAL(ipc_cld_join_send, ipc);
-DEFINE_PROFILE_INTERVAL(ipc_cld_join_callback, ipc);
-
-int ipc_cld_join_send (IDTYPE dest)
-{
-    BEGIN_PROFILE_INTERVAL();
-    struct shim_ipc_port * port = dest ?
-                                  lookup_ipc_port(dest, IPC_PORT_DIRPRT) :
-                                  get_parent_port(&dest);
-    if (!port)
-        return -ESRCH;
+    /* First try to find remote thread who sent this message among normal
+     * threads. In the common case, we (as parent process) keep remote child
+     * threads in the thread list. But sometimes the message can arrive twice
+     * or very late, such that the corresponding remote thread was already
+     * exited and deleted; in such cases, we fall back to simple threads. */
+    struct shim_thread* thread = lookup_thread(msgin->tid);
+    if (thread) {
+        lock(&thread->lock);
+        thread->exit_code   = -msgin->exitcode;
+        thread->term_signal = msgin->term_signal;
+#ifdef PROFILE
+        thread->exit_time = time;
+#endif
+        unlock(&thread->lock);
 
-    size_t total_msg_size = get_ipc_msg_size(0);
-    struct shim_ipc_msg* msg = __alloca(total_msg_size);
-    init_ipc_msg(msg, IPC_CLD_JOIN, total_msg_size, dest);
+        /* Remote thread is "virtually" exited: SIGCHLD is generated for the
+         * parent thread and exit events are arranged for subsequent wait4(). */
+        ret = thread_exit(thread, /*send_ipc*/ false);
+        put_thread(thread);
+    } else {
+        /* Uncommon case: remote child thread was already exited and deleted
+         * (probably because the same message was already received earlier).
+         * Find or create a simple thread for a sole purpose of arranging
+         * exit events for subsequent wait4(). */
+        struct shim_simple_thread* sthread = lookup_simple_thread(msgin->tid);
+
+        if (!sthread) {
+            sthread = get_new_simple_thread();
+            sthread->vmid = msg->src;
+            sthread->tid  = msgin->tid;
+            add_simple_thread(sthread);
+        }
 
-    debug("ipc send to %u: IPC_CLD_JOIN\n", dest);
+        lock(&sthread->lock);
+        sthread->is_alive    = false;
+        sthread->exit_code   = -msgin->exitcode;
+        sthread->term_signal = msgin->term_signal;
+#ifdef PROFILE
+        sthread->exit_time = time;
+#endif
+        unlock(&sthread->lock);
 
-    int ret = send_ipc_message(msg, port);
+        DkEventSet(sthread->exit_event); /* for wait4(msgin->tid) */
+        put_simple_thread(sthread);
+    }
 
-    add_ipc_port(port, dest, IPC_PORT_DIRPRT, NULL);
-    put_ipc_port(port);
-    SAVE_PROFILE_INTERVAL(ipc_cld_join_send);
+    SAVE_PROFILE_INTERVAL(ipc_cld_exit_callback);
     return ret;
 }
 
-int ipc_cld_join_callback (IPC_CALLBACK_ARGS)
-{
-    BEGIN_PROFILE_INTERVAL();
-    debug("ipc callback from %u: IPC_CLD_JOIN\n", msg->src);
-    add_ipc_port(port, msg->src, IPC_PORT_DIRCLD, NULL);
-    SAVE_PROFILE_INTERVAL(ipc_cld_join_callback);
-    return 0;
-}
-
 DEFINE_PROFILE_INTERVAL(ipc_send_profile, ipc);
 
 #ifdef PROFILE
-int ipc_cld_profile_send (void)
-{
-    IDTYPE dest;
-    struct shim_ipc_port * port = get_parent_port(&dest);
-    if (!port)
+int ipc_cld_profile_send(void) {
+    struct shim_ipc_port* port = NULL;
+    IDTYPE dest = (IDTYPE) -1;
+
+    /* port and dest are initialized to parent process */
+    lock(&cur_process.lock);
+    if (cur_process.parent && (port = cur_process.parent->port)) {
+        get_ipc_port(port);
+        dest = cur_process.parent->vmid;
+    }
+    unlock(&cur_process.lock);
+
+    if (!port || (dest == (IDTYPE) -1))
         return -ESRCH;
 
     unsigned long time = GET_PROFILE_INTERVAL();
     size_t nsending = 0;
-    for (size_t i = 0 ; i < N_PROFILE ; i++)
+    for (size_t i = 0; i < N_PROFILE; i++)
         switch (PROFILES[i].type) {
             case OCCURENCE:
                 if (atomic_read(&PROFILES[i].val.occurence.count))
@@ -286,38 +273,34 @@ int ipc_cld_profile_send (void)
                 break;
         }
 
-
     size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_cld_profile) +
                                              sizeof(struct profile_val) * nsending);
     struct shim_ipc_msg* msg = __alloca(total_msg_size);
     init_ipc_msg(msg, IPC_CLD_PROFILE, total_msg_size, dest);
 
-    struct shim_ipc_cld_profile * msgin =
-                (struct shim_ipc_cld_profile *) &msg->msg;
+    struct shim_ipc_cld_profile* msgin = (struct shim_ipc_cld_profile *) &msg->msg;
 
     size_t nsent = 0;
-    for (size_t i = 0 ; i < N_PROFILE && nsent < nsending ; i++)
+    for (size_t i = 0; i < N_PROFILE && nsent < nsending; i++)
         switch (PROFILES[i].type) {
             case OCCURENCE: {
-                unsigned long count =
-                    atomic_read(&PROFILES[i].val.occurence.count);
+                unsigned long count = atomic_read(&PROFILES[i].val.occurence.count);
                 if (count) {
                     msgin->profile[nsent].idx = i + 1;
                     msgin->profile[nsent].val.occurence.count = count;
-                    debug("send %s: %lu times\n", PROFILES[i].name, count);
+                    debug("Send %s: %lu times\n", PROFILES[i].name, count);
                     nsent++;
                 }
                 break;
             }
             case INTERVAL: {
-                unsigned long count =
-                    atomic_read(&PROFILES[i].val.interval.count);
+                unsigned long count = atomic_read(&PROFILES[i].val.interval.count);
                 if (count) {
                     msgin->profile[nsent].idx = i + 1;
                     msgin->profile[nsent].val.interval.count = count;
                     msgin->profile[nsent].val.interval.time =
                         atomic_read(&PROFILES[i].val.interval.time);
-                    debug("send %s: %lu times, %lu msec\n", PROFILES[i].name,
+                    debug("Send %s: %lu times, %lu msec\n", PROFILES[i].name,
                           count, msgin->profile[nsent].val.interval.time);
                     nsent++;
                 }
@@ -330,34 +313,32 @@ int ipc_cld_profile_send (void)
     msgin->time = time;
     msgin->nprofile = nsent;
 
-    debug("ipc send to %u: IPC_CLD_PROFILE\n", dest);
-
+    debug("IPC send to %u: IPC_CLD_PROFILE\n", dest & 0xFFFF);
     int ret = send_ipc_message(msg, port);
+
     put_ipc_port(port);
     return ret;
 }
 
-int ipc_cld_profile_callback (IPC_CALLBACK_ARGS)
-{
-    struct shim_ipc_cld_profile * msgin =
-                (struct shim_ipc_cld_profile *) &msg->msg;
+int ipc_cld_profile_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port) {
+    debug("IPC callback from %u: IPC_CLD_PROFILE\n", msg->src & 0xFFFF);
 
-    debug("ipc callback from %u: IPC_CLD_PROFILE\n", msg->src);
+    struct shim_ipc_cld_profile* msgin = (struct shim_ipc_cld_profile *) &msg->msg;
 
-    for (int i = 0 ; i < msgin->nprofile ; i++) {
+    for (int i = 0; i < msgin->nprofile; i++) {
         int idx = msgin->profile[i].idx;
         if (idx == 0)
             break;
         idx--;
         switch (PROFILES[idx].type) {
             case OCCURENCE:
-                debug("receive %s: %u times\n", PROFILES[idx].name,
+                debug("Receive %s: %u times\n", PROFILES[idx].name,
                       msgin->profile[i].val.occurence.count);
                 atomic_add(msgin->profile[i].val.occurence.count,
                            &PROFILES[idx].val.occurence.count);
                 break;
             case INTERVAL:
-                debug("receive %s: %u times, %lu msec\n", PROFILES[idx].name,
+                debug("Receive %s: %u times, %lu msec\n", PROFILES[idx].name,
                       msgin->profile[i].val.interval.count,
                       msgin->profile[i].val.interval.time);
                 atomic_add(msgin->profile[i].val.interval.count,

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -61,7 +61,7 @@ static ipc_callback ipc_callbacks[IPC_CODE_NUM] = {
 
     /* parents and children */
     /* CLD_EXIT         */  &ipc_cld_exit_callback,
-    /* CLD_JOIN         */  &ipc_cld_join_callback,
+
 #ifdef PROFILE
     /* CLD_PROFILE      */  &ipc_cld_profile_callback,
 #endif

LibOS/shim/src/shim_checkpoint.c

@@ -1145,7 +1145,7 @@ int do_migrate_process (int (*migrate) (struct shim_cp_store *,
     /* Listen on the RPC stream to the new process */
     add_ipc_port_by_id(res.child_vmid, proc,
                        IPC_PORT_DIRCLD|IPC_PORT_LISTEN|IPC_PORT_KEEPALIVE,
-                       &ipc_child_exit,
+                       &ipc_port_with_child_fini,
                        NULL);
 
     free_process(new_process);