[LibOS] Comprehensive cleanup of basic IPC (shim_ipc.c)

- Changed type of shim_ipc_msg.size from int to size_t.
- Renamed shim_ipc_msg_obj to shim_ipc_msg_duplex for readability.
- Simplified function signatures and changed to better names.
- Removed unused IPC_FINDURI & IPC_TELLURI and corresponding functions.
- Replaced macros IPC_MSG_SIZE & IPC_MSGOBJ_SIZE with inline functions.
- Removed dangerous create_xxx_on_stack() functions and changed all
  invocations to have explicit __alloca's. Those functions relied on
  being inlined in the callers otherwise their created objects would
  become corrupted. Explicit __alloca's avoid this brittle
  implementation and make object ownership clear.
- Removed unnecessary wrapper function do_ipc_duplex(), replacing it
  with send_ipc_message_duplex().

LibOS/shim/include/shim_ipc.h

@@ -58,7 +58,7 @@ extern struct shim_process cur_process;
 
 struct shim_ipc_msg {
     unsigned char       code;
-    unsigned int        size;
+    size_t              size;
     IDTYPE              src, dst;
     unsigned long       seq;
 #ifdef PROFILE
@@ -70,10 +70,10 @@ struct shim_ipc_msg {
 struct shim_ipc_port;
 struct shim_thread;
 
-DEFINE_LIST(shim_ipc_msg_obj);
-struct shim_ipc_msg_obj {
+DEFINE_LIST(shim_ipc_msg_duplex);
+struct shim_ipc_msg_duplex {
     struct shim_thread *    thread;
-    LIST_TYPE(shim_ipc_msg_obj) list;
+    LIST_TYPE(shim_ipc_msg_duplex) list;
     int                     retval;
     void *                  private;
     struct shim_ipc_msg     msg;
@@ -85,13 +85,13 @@ typedef void (*port_fini) (struct shim_ipc_port *, IDTYPE vmid,
 #define MAX_IPC_PORT_FINI_CB        3
 
 DEFINE_LIST(shim_ipc_port);
-DEFINE_LISTP(shim_ipc_msg_obj);
+DEFINE_LISTP(shim_ipc_msg_duplex);
 struct shim_ipc_port {
     PAL_HANDLE          pal_handle;
 
     REFTYPE             ref_count;
     LIST_TYPE(shim_ipc_port) list;
-    LISTP_TYPE(shim_ipc_msg_obj) msgs;
+    LISTP_TYPE(shim_ipc_msg_duplex) msgs;
     struct shim_lock    msgs_lock;
 
     port_fini           fini[MAX_IPC_PORT_FINI_CB];
@@ -109,11 +109,9 @@ struct shim_ipc_port {
 
 typedef int (*ipc_callback) (IPC_CALLBACK_ARGS);
 
-/* Messagge code to response the connection */
+/* Basic message codes */
 enum {
     IPC_RESP = 0,
-    IPC_FINDURI,
-    IPC_TELLURI,
     IPC_CHECKPOINT,
     IPC_BASE_BOUND,
 };
@@ -123,20 +121,6 @@ struct shim_ipc_resp {
     int retval;
 } __attribute__((packed));
 
-/* IPC_FINDURI: request a URI from a connect process */
-int ipc_finduri_send (struct shim_ipc_port * port, IDTYPE dest,
-                      struct shim_ipc_info ** info);
-int ipc_finduri_callback (IPC_CALLBACK_ARGS);
-
-/* IPC_TELLURI: replying with a connectable URI */
-struct shim_ipc_telluri {
-    char uri[1];
-} __attribute__((packed));
-
-int ipc_telluri_send (struct shim_ipc_port * port, IDTYPE dest,
-                      struct shim_ipc_info * info);
-int ipc_telluri_callback (IPC_CALLBACK_ARGS);
-
 /* PID_CHECKPOINT: broadcast checkpointing */
 struct shim_ipc_checkpoint {
     IDTYPE cpsession;
@@ -469,11 +453,11 @@ int ipc_sysv_semreply_callback (IPC_CALLBACK_ARGS);
 int init_ipc (void);
 int init_ipc_helper (void);
 
-struct shim_process * create_new_process (bool inherit_parent);
-void destroy_process (struct shim_process * proc);
+struct shim_process* create_process(void);
+void free_process(struct shim_process* process);
 
-struct shim_ipc_info * create_ipc_port (IDTYPE vmid, bool listen);
-int create_ipc_location (struct shim_ipc_info ** pinfo);
+struct shim_ipc_info* create_ipc_info_cur_process(void);
+int get_ipc_info_cur_process(struct shim_ipc_info** pinfo);
 
 enum {
     LISTEN=0,   /* listening */
@@ -507,82 +491,34 @@ void get_ipc_port (struct shim_ipc_port * port);
 void put_ipc_port (struct shim_ipc_port * port);
 void del_all_ipc_ports (void);
 
-struct shim_ipc_info * get_new_ipc_info (IDTYPE vmid, const char * uri,
-                                         size_t len);
-void get_ipc_info(struct shim_ipc_info * port);
-void put_ipc_info(struct shim_ipc_info * port);
-
-struct shim_ipc_info * lookup_and_alloc_client (IDTYPE vmid, const char * uri);
-void put_client (struct shim_ipc_info * info);
-struct shim_ipc_info * discover_client (struct shim_ipc_port * port,
-                                        IDTYPE vmid);
-
-#define IPC_MSG_SIZE(extra)                                             \
-    ({  int _size = (extra) + sizeof(struct shim_ipc_msg);              \
-        _size > IPC_MSG_MINIMAL_SIZE ? _size : IPC_MSG_MINIMAL_SIZE; })
-#define IPC_MSGOBJ_SIZE(extra)                                          \
-    ({  int _size = (extra) + sizeof(struct shim_ipc_msg);              \
-        (_size > IPC_MSG_MINIMAL_SIZE ? _size : IPC_MSG_MINIMAL_SIZE) + \
-        (sizeof(struct shim_ipc_msg_obj) - sizeof(struct shim_ipc_msg)); })
-
-int __init_ipc_msg (struct shim_ipc_msg * msg, int code, int size, IDTYPE dest);
-struct shim_ipc_msg * create_ipc_msg (int code, int size, IDTYPE dest);
-
-static_always_inline
-struct shim_ipc_msg * create_ipc_msg_on_stack (int code, int size, IDTYPE dest)
-{
-    struct shim_ipc_msg * msg = __alloca(IPC_MSG_SIZE(size));
-
-    return (!__init_ipc_msg(msg, code, size, dest)) ? msg : NULL;
-}
-
-int __init_ipc_msg_duplex (struct shim_ipc_msg_obj * msg, int code, int size,
-                           IDTYPE dest);
-struct shim_ipc_msg_obj *
-create_ipc_msg_duplex (int code, int size, IDTYPE dest);
+struct shim_ipc_info* create_ipc_info(IDTYPE vmid, const char* uri, size_t len);
+void get_ipc_info(struct shim_ipc_info* port);
+void put_ipc_info(struct shim_ipc_info* port);
 
-static_always_inline
-struct shim_ipc_msg_obj *
-create_ipc_msg_duplex_on_stack (int code, int size, IDTYPE dest)
-{
-    struct shim_ipc_msg_obj * msg = __alloca(IPC_MSGOBJ_SIZE(size));
+struct shim_ipc_info* create_ipc_info_in_list(IDTYPE vmid, const char* uri);
+void put_ipc_info_in_list(struct shim_ipc_info* info);
+struct shim_ipc_info* lookup_ipc_info(IDTYPE vmid);
 
-    return (!__init_ipc_msg_duplex(msg, code, size, dest)) ?
-           msg : NULL;
+static_always_inline size_t get_ipc_msg_size(size_t payload) {
+    size_t size = sizeof(struct shim_ipc_msg) + payload;
+    return (size > IPC_MSG_MINIMAL_SIZE) ? size : IPC_MSG_MINIMAL_SIZE;
 }
 
-int __init_ipc_resp_msg (struct shim_ipc_msg * resp, int ret,
-                         unsigned long seq);
-struct shim_ipc_msg *
-create_ipc_resp_msg (int ret, IDTYPE dest, unsigned long seq);
+static_always_inline size_t get_ipc_msg_duplex_size(size_t payload) {
+    return get_ipc_msg_size(payload) +
+        (sizeof(struct shim_ipc_msg_duplex) - sizeof(struct shim_ipc_msg));
+}
 
-static_always_inline
-struct shim_ipc_msg *
-create_ipc_resp_msg_on_stack (int ret, IDTYPE dest, unsigned long seq)
-{
-    struct shim_ipc_msg * resp = create_ipc_msg_on_stack(IPC_RESP,
-                                        sizeof(struct shim_ipc_resp), dest);
+void init_ipc_msg(struct shim_ipc_msg* msg, int code, size_t size, IDTYPE dest);
+void init_ipc_msg_duplex(struct shim_ipc_msg_duplex* msg, int code, size_t size, IDTYPE dest);
 
-    return (resp && !__init_ipc_resp_msg(resp, ret, seq)) ? resp : NULL;
-}
+struct shim_ipc_msg_duplex* pop_ipc_msg_duplex(struct shim_ipc_port* port, unsigned long seq);
 
-int send_ipc_message (struct shim_ipc_msg * msg, struct shim_ipc_port * port);
-int send_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
-                             struct shim_ipc_port * port, bool save,
-                             void * private_data);
-int close_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
-                              struct shim_ipc_port * port);
-int broadcast_ipc (struct shim_ipc_msg * msg, int target_type,
-                   struct shim_ipc_port * exclude_port);
-struct shim_ipc_msg_obj * find_ipc_msg_duplex (struct shim_ipc_port * port,
-                                               unsigned long seq);
-
-int send_response_ipc_message (struct shim_ipc_port * port, IDTYPE dest,
-                            int ret, unsigned long seq);
-
-int do_ipc_duplex (struct shim_ipc_msg_obj * msg,
-                   struct shim_ipc_port * port, unsigned long * seq,
-                   void * private_data);
+int broadcast_ipc(struct shim_ipc_msg* msg, int target_type, struct shim_ipc_port* exclude_port);
+int send_ipc_message(struct shim_ipc_msg* msg, struct shim_ipc_port* port);
+int send_ipc_message_duplex(struct shim_ipc_msg_duplex* msg, struct shim_ipc_port* port,
+                            unsigned long* seq, void* private_data);
+int send_response_ipc_message(struct shim_ipc_port* port, IDTYPE dest, int ret, unsigned long seq);
 
 void ipc_child_exit   (struct shim_ipc_port * port, IDTYPE vmid,
                        unsigned int exitcode);

LibOS/shim/src/ipc/shim_ipc.c

@@ -17,7 +17,9 @@
 /*
  * shim_ipc.c
  *
- * This file contains codes to maintain generic bookkeeping of IPC.
+ * This file contains codes to maintain generic bookkeeping of IPC: operations
+ * on shim_ipc_msg (one-way IPC messages), shim_ipc_msg_duplex (IPC messages
+ * with acknowledgement), shim_ipc_info (IPC ports of process), shim_process.
  */
 
 #include <shim_internal.h>
@@ -35,25 +37,29 @@
 
 #define IPC_INFO_MGR_ALLOC  32
 #define PAGE_SIZE           allocsize
-
 #define OBJ_TYPE struct shim_ipc_info
 #include "memmgr.h"
-
 static MEM_MGR ipc_info_mgr;
 
 struct shim_lock ipc_info_lock;
 
 struct shim_process cur_process;
 
+#define CLIENT_HASH_LEN     6
+#define CLIENT_HASH_NUM     (1 << CLIENT_HASH_LEN)
+#define CLIENT_HASH_MASK    (CLIENT_HASH_NUM - 1)
+#define CLIENT_HASH(vmid)   ((vmid) & CLIENT_HASH_MASK)
+DEFINE_LISTP(shim_ipc_info);
+static LISTP_TYPE(shim_ipc_info) info_hlist[CLIENT_HASH_NUM];
+
 DEFINE_PROFILE_CATEGORY(ipc, );
 DEFINE_PROFILE_OCCURENCE(syscall_use_ipc, ipc);
 
-int init_ipc_ports (void);
-int init_ns_pid    (void);
-int init_ns_sysv   (void);
+int init_ipc_ports(void);
+int init_ns_pid(void);
+int init_ns_sysv(void);
 
-int init_ipc (void)
-{
+int init_ipc(void) {
     int ret = 0;
 
     create_lock(&ipc_info_lock);
@@ -63,18 +69,15 @@ int init_ipc (void)
 
     if ((ret = init_ipc_ports()) < 0)
         return ret;
-
     if ((ret = init_ns_pid()) < 0)
         return ret;
-
     if ((ret = init_ns_sysv()) < 0)
         return ret;
 
     return 0;
 }
 
-int prepare_ns_leaders (void)
-{
+int prepare_ns_leaders(void) {
     int ret = 0;
     if ((ret = prepare_pid_leader()) < 0)
         return ret;
@@ -83,18 +86,14 @@ int prepare_ns_leaders (void)
     return 0;
 }
 
-static struct shim_ipc_info * __get_new_ipc_info (IDTYPE vmid, const char * uri,
-                                                  size_t len)
-{
-    struct shim_ipc_info * info =
-                get_mem_obj_from_mgr_enlarge(ipc_info_mgr,
-                                             size_align_up(IPC_INFO_MGR_ALLOC));
+static struct shim_ipc_info* __create_ipc_info(IDTYPE vmid, const char* uri, size_t len) {
+    struct shim_ipc_info* info =
+        get_mem_obj_from_mgr_enlarge(ipc_info_mgr, size_align_up(IPC_INFO_MGR_ALLOC));
     if (!info)
         return NULL;
 
     memset(info, 0, sizeof(struct shim_ipc_info));
-    if (vmid)
-        info->vmid = vmid;
+    info->vmid = vmid;
     if (uri)
         qstrsetstr(&info->uri, uri, len);
     REF_SET(info->ref_count, 1);
@@ -102,257 +101,168 @@ static struct shim_ipc_info * __get_new_ipc_info (IDTYPE vmid, const char * uri,
     return info;
 }
 
-struct shim_ipc_info * get_new_ipc_info (IDTYPE vmid, const char * uri,
-                                         size_t len)
-{
-    lock(&ipc_info_lock);
-    struct shim_ipc_info * info = __get_new_ipc_info(vmid, uri, len);
-    unlock(&ipc_info_lock);
-    return info;
+static void __free_ipc_info(struct shim_ipc_info* info) {
+    if (info->pal_handle) {
+        DkObjectClose(info->pal_handle);
+        info->pal_handle = NULL;
+    }
+    if (info->port)
+        put_ipc_port(info->port);
+    qstrfree(&info->uri);
+    free_mem_obj_to_mgr(ipc_info_mgr, info);
 }
 
-static void __get_ipc_info (struct shim_ipc_info * info)
-{
+static void __get_ipc_info(struct shim_ipc_info* info) {
     REF_INC(info->ref_count);
 }
 
-void get_ipc_info (struct shim_ipc_info * info)
-{
-    __get_ipc_info(info);
+static void __put_ipc_info(struct shim_ipc_info* info) {
+    int ref_count = REF_DEC(info->ref_count);
+    if (!ref_count)
+        __free_ipc_info(info);
 }
 
-static void unset_ipc_info (struct shim_ipc_info * info)
-{
-    qstrfree(&info->uri);
-
-    if (info->port)
-        put_ipc_port(info->port);
-
-    if (info->pal_handle)
-        DkObjectClose(info->pal_handle);
+void get_ipc_info(struct shim_ipc_info* info) {
+    /* no need to grab ipc_info_lock because __get_ipc_info() is atomic */
+    __get_ipc_info(info);
 }
 
-static void __put_ipc_info (struct shim_ipc_info * info)
-{
+void put_ipc_info(struct shim_ipc_info* info) {
+    /* this is atomic so we don't grab lock in common case of ref_count > 0 */
     int ref_count = REF_DEC(info->ref_count);
 
-    if (ref_count)
-        return;
-
-    unset_ipc_info(info);
-    free_mem_obj_to_mgr(ipc_info_mgr, info);
+    if (!ref_count) {
+        lock(&ipc_info_lock);
+        __free_ipc_info(info);
+        unlock(&ipc_info_lock);
+    }
 }
 
-void put_ipc_info (struct shim_ipc_info * info)
-{
-    int ref_count = REF_DEC(info->ref_count);
-
-    if (ref_count)
-        return;
-
-    unset_ipc_info(info);
+struct shim_ipc_info* create_ipc_info(IDTYPE vmid, const char* uri, size_t len) {
     lock(&ipc_info_lock);
-    free_mem_obj_to_mgr(ipc_info_mgr, info);
+    struct shim_ipc_info* info = __create_ipc_info(vmid, uri, len);
     unlock(&ipc_info_lock);
+    return info;
 }
 
-#define CLIENT_HASH_LEN     6
-#define CLIENT_HASH_NUM     (1 << CLIENT_HASH_LEN)
-#define CLIENT_HASH_MASK    (CLIENT_HASH_NUM - 1)
-#define CLIENT_HASH(vmid)   ((vmid) & CLIENT_HASH_MASK)
-
-/* Links to shim_ipc_info->hlist */
-DEFINE_LISTP(shim_ipc_info);
-static LISTP_TYPE(shim_ipc_info) client_table [CLIENT_HASH_NUM];
+struct shim_ipc_info* create_ipc_info_in_list(IDTYPE vmid, const char* uri) {
+    assert(vmid);
 
-struct shim_ipc_info *
-lookup_and_alloc_client (IDTYPE vmid, const char * uri)
-{
-    struct shim_ipc_info * p;
-    LISTP_TYPE(shim_ipc_info) *head = client_table + CLIENT_HASH(vmid);
+    struct shim_ipc_info* info;
     size_t len = strlen(uri);
 
-    assert(vmid);
-
     lock(&ipc_info_lock);
-    LISTP_FOR_EACH_ENTRY(p, head, hlist)
-        if (p->vmid == vmid && !qstrcmpstr(&p->uri, uri, len)) {
-            get_ipc_info(p);
+
+    /* check if info with this vmid and uri already exists and return it */
+    LISTP_TYPE(shim_ipc_info)* info_bucket = &info_hlist[CLIENT_HASH(vmid)];
+    LISTP_FOR_EACH_ENTRY(info, info_bucket, hlist)
+        if (info->vmid == vmid && !qstrcmpstr(&info->uri, uri, len)) {
+            get_ipc_info(info);
             unlock(&ipc_info_lock);
-            return p;
+            return info;
         }
 
-    p = __get_new_ipc_info(vmid, uri, len);
-    if (p) {
-        LISTP_ADD(p, head, hlist);
-        get_ipc_info(p);
+    /* otherwise create new info and return it */
+    info = __create_ipc_info(vmid, uri, len);
+    if (info) {
+        LISTP_ADD(info, info_bucket, hlist);
+        get_ipc_info(info);
     }
+
     unlock(&ipc_info_lock);
-    return p;
+    return info;
 }
 
-void put_client (struct shim_ipc_info * info)
-{
+void put_ipc_info_in_list(struct shim_ipc_info* info) {
+    LISTP_TYPE(shim_ipc_info)* info_bucket = &info_hlist[CLIENT_HASH(info->vmid)];
+
     lock(&ipc_info_lock);
-    /* Look up the hash */
-    LISTP_TYPE(shim_ipc_info) *head = client_table + CLIENT_HASH(info->vmid);
     __put_ipc_info(info);
     if (REF_GET(info->ref_count) == 1) {
-        LISTP_DEL_INIT(info, head, hlist);
+        LISTP_DEL_INIT(info, info_bucket, hlist);
         __put_ipc_info(info);
     }
     unlock(&ipc_info_lock);
 }
 
-struct shim_ipc_info * discover_client (struct shim_ipc_port * port,
-                                        IDTYPE vmid)
-{
-    struct shim_ipc_info * p;
-    LISTP_TYPE(shim_ipc_info) * head = client_table + CLIENT_HASH(vmid);
-
+struct shim_ipc_info* lookup_ipc_info(IDTYPE vmid) {
     assert(vmid);
-
     lock(&ipc_info_lock);
-    LISTP_FOR_EACH_ENTRY(p, head, hlist)
-        if (p->vmid == vmid && !qstrempty(&p->uri)) {
-            __get_ipc_info(p);
+
+    struct shim_ipc_info* info;
+    LISTP_TYPE(shim_ipc_info)* info_bucket = &info_hlist[CLIENT_HASH(vmid)];
+    LISTP_FOR_EACH_ENTRY(info, info_bucket, hlist)
+        if (info->vmid == vmid && !qstrempty(&info->uri)) {
+            __get_ipc_info(info);
             unlock(&ipc_info_lock);
-            return p;
+            return info;
         }
-    unlock(&ipc_info_lock);
-    return NULL;
-
-    if (!ipc_finduri_send(port, vmid, &p))
-        return p;
 
+    unlock(&ipc_info_lock);
     return NULL;
 }
 
-struct shim_process * create_new_process (bool inherit_parent)
-{
-    struct shim_process * new_process = calloc(1, sizeof(struct shim_process));
+struct shim_process* create_process(void) {
+    struct shim_process* new_process = calloc(1, sizeof(struct shim_process));
     if (!new_process)
         return NULL;
 
-    new_process->parent = get_new_ipc_info(cur_process.vmid, NULL, 0);
-
-    if (!inherit_parent)
-        return new_process;
+    new_process->parent = create_ipc_info(cur_process.vmid, NULL, 0);
 
     lock(&cur_process.lock);
 
     if (cur_process.self)
         qstrcopy(&new_process->parent->uri, &cur_process.self->uri);
-
-    for (int i = 0 ; i < TOTAL_NS ; i++)
+    for (int i = 0; i < TOTAL_NS; i++) {
         if (cur_process.ns[i])
-            new_process->ns[i] =
-                get_new_ipc_info(cur_process.ns[i]->vmid,
-                                 qstrgetstr(&cur_process.ns[i]->uri),
-                                 cur_process.ns[i]->uri.len);
+            new_process->ns[i] = create_ipc_info(cur_process.ns[i]->vmid,
+                                                 qstrgetstr(&cur_process.ns[i]->uri),
+                                                 cur_process.ns[i]->uri.len);
+    }
 
     unlock(&cur_process.lock);
     return new_process;
 }
 
-void destroy_process (struct shim_process * proc)
-{
-    if (proc->self)
-        put_ipc_info(proc->self);
-
-    if (proc->parent)
-        put_ipc_info(proc->parent);
-
-    for (int i = 0 ; i < TOTAL_NS ; i++)
-        if (proc->ns[i])
-            put_ipc_info(proc->ns[i]);
-
-    free(proc);
+void free_process(struct shim_process* process) {
+    if (process->self)
+        put_ipc_info(process->self);
+    if (process->parent)
+        put_ipc_info(process->parent);
+    for (int i = 0; i < TOTAL_NS; i++)
+        if (process->ns[i])
+            put_ipc_info(process->ns[i]);
+    free(process);
 }
 
-int __init_ipc_msg (struct shim_ipc_msg * msg, int code, int size, IDTYPE dest)
-{
+void init_ipc_msg(struct shim_ipc_msg* msg, int code, size_t size, IDTYPE dest) {
     msg->code = code;
-    msg->size = IPC_MSG_SIZE(size);
+    msg->size = get_ipc_msg_size(size);
     msg->src = cur_process.vmid;
     msg->dst = dest;
     msg->seq = 0;
-    return 0;
 }
 
-struct shim_ipc_msg * create_ipc_msg (int code, int size, IDTYPE dest)
-{
-    struct shim_ipc_msg * msg = malloc(IPC_MSG_SIZE(size));
-
-    if (msg && __init_ipc_msg(msg, code, size, dest)) {
-        free(msg);
-        msg = NULL;
-    }
-
-    return msg;
-}
-
-int __init_ipc_msg_duplex (struct shim_ipc_msg_obj * msg, int code, int size,
-                           IDTYPE dest)
-{
-    __init_ipc_msg(&msg->msg, code, size, dest);
+void init_ipc_msg_duplex(struct shim_ipc_msg_duplex* msg, int code, size_t size, IDTYPE dest) {
+    init_ipc_msg(&msg->msg, code, size, dest);
     msg->thread = NULL;
     INIT_LIST_HEAD(msg, list);
     msg->retval = 0;
     msg->private = NULL;
-    return 0;
-}
-
-struct shim_ipc_msg_obj *
-create_ipc_msg_duplex (int code, int size, IDTYPE dest)
-{
-    struct shim_ipc_msg_obj * msg = malloc(IPC_MSGOBJ_SIZE(size));
-
-    if (msg && __init_ipc_msg_duplex(msg, code, size, dest)) {
-        free(msg);
-        msg = NULL;
-    }
-
-    return msg;
-}
-
-int __init_ipc_resp_msg (struct shim_ipc_msg * resp, int ret,
-                         unsigned long seq)
-{
-    struct shim_ipc_resp * resp_in = (struct shim_ipc_resp *) resp->msg;
-    resp->seq = seq;
-    resp_in->retval = ret;
-    return 0;
-}
-
-struct shim_ipc_msg *
-create_ipc_resp_msg (int ret, IDTYPE dest, unsigned long seq)
-{
-    struct shim_ipc_msg * resp =
-            create_ipc_msg(IPC_RESP, sizeof(struct shim_ipc_resp), dest);
-
-    if (resp && __init_ipc_resp_msg(resp, ret, seq)) {
-        free(resp);
-        resp = NULL;
-    }
-
-    return resp;
 }
 
-int send_ipc_message (struct shim_ipc_msg * msg, struct shim_ipc_port * port)
-{
+int send_ipc_message(struct shim_ipc_msg* msg, struct shim_ipc_port* port) {
     assert(msg->size >= IPC_MSG_MINIMAL_SIZE);
-    msg->src = cur_process.vmid;
 
-    debug("send ipc message to port %p (handle %p)\n", port,
-          port->pal_handle);
+    msg->src = cur_process.vmid;
+    debug("Sending ipc message to port %p (handle %p)\n", port, port->pal_handle);
 
+    /* TODO: Handle benign EINTR case? */
+    /* TODO: Add while-loop to send all msg */
     int ret = DkStreamWrite(port->pal_handle, 0, msg->size, msg, NULL);
 
     if (ret == 0 && PAL_NATIVE_ERRNO) {
-        debug("port %p (handle %p) is removed at sending\n", port,
-              port->pal_handle);
-
+        debug("Port %p (handle %p) was removed during sending\n", port, port->pal_handle);
         del_ipc_port_fini(port, -ECHILD);
         return -PAL_ERRNO;
     }
@@ -360,57 +270,11 @@ int send_ipc_message (struct shim_ipc_msg * msg, struct shim_ipc_port * port)
     return 0;
 }
 
-int close_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
-                              struct shim_ipc_port * port)
-{
-    if (port) {
-        // Check if the message is pending on the port for response. If so,
-        // remove the message from the list.
-        lock(&port->msgs_lock);
-        if (!LIST_EMPTY(msg, list))
-            LISTP_DEL_INIT(msg, &port->msgs, list);
-        unlock(&port->msgs_lock);
-    }
-
-    if (msg->thread) {
-        put_thread(msg->thread);
-        msg->thread = NULL;
-    }
-
-    return 0;
-}
-
-static struct atomic_int ipc_seq_counter;
-
-int send_ipc_message_duplex (struct shim_ipc_msg_obj * msg,
-                             struct shim_ipc_port * port, bool save,
-                             void * private_data)
-{
-    msg->msg.seq = atomic_inc_return(&ipc_seq_counter);
-
-    if (save) {
-        lock(&port->msgs_lock);
-        msg->private = private_data;
-        LISTP_ADD_TAIL(msg, &port->msgs, list);
-        unlock(&port->msgs_lock);
-    }
-
-    int ret = send_ipc_message(&msg->msg, port);
-
-    if (ret < 0) {
-        if (save)
-            close_ipc_message_duplex(msg, port);
-        return ret;
-    }
-
-    return 0;
-}
+struct shim_ipc_msg_duplex* pop_ipc_msg_duplex(struct shim_ipc_port* port, unsigned long seq) {
+    struct shim_ipc_msg_duplex* found = NULL;
 
-struct shim_ipc_msg_obj * find_ipc_msg_duplex (struct shim_ipc_port * port,
-                                               unsigned long seq)
-{
-    struct shim_ipc_msg_obj * tmp, * found = NULL;
     lock(&port->msgs_lock);
+    struct shim_ipc_msg_duplex* tmp;
     LISTP_FOR_EACH_ENTRY(tmp, &port->msgs, list)
         if (tmp->msg.seq == seq) {
             found = tmp;
@@ -418,262 +282,197 @@ struct shim_ipc_msg_obj * find_ipc_msg_duplex (struct shim_ipc_port * port,
             break;
         }
     unlock(&port->msgs_lock);
+
     return found;
 }
 
-/* for convenience */
-int do_ipc_duplex (struct shim_ipc_msg_obj * msg,
-                   struct shim_ipc_port * port, unsigned long * seq,
-                   void * private_data)
-{
+int send_ipc_message_duplex(struct shim_ipc_msg_duplex* msg, struct shim_ipc_port* port,
+                  unsigned long* seq, void* private_data) {
     int ret = 0;
+
     struct shim_thread * thread = get_cur_thread();
     assert(thread);
 
+    /* prepare thread which sends the message for waiting for response
+     * (this also acquires reference to the thread) */
     if (!msg->thread)
         thread_setwait(&msg->thread, thread);
 
-    ret = send_ipc_message_duplex(msg, port, true, private_data);
+    static struct atomic_int ipc_seq_counter;
+    msg->msg.seq = atomic_inc_return(&ipc_seq_counter);
 
-    if (seq)
-        *seq = (ret < 0) ? 0 : msg->msg.seq;
+    /* save the message to list of port msgs together with its private data */
+    lock(&port->msgs_lock);
+    msg->private = private_data;
+    LISTP_ADD_TAIL(msg, &port->msgs, list);
+    unlock(&port->msgs_lock);
 
+    ret = send_ipc_message(&msg->msg, port);
     if (ret < 0)
         goto out;
 
-    debug("wait for response (seq = %lu)\n", msg->msg.seq);
-    thread_sleep(NO_TIMEOUT);
+    if (seq)
+        *seq = msg->msg.seq;
+
+    debug("Start waiting for response (seq = %lu)\n", msg->msg.seq);
 
+    /* force thread which sends the message to wait for response;
+     * ignore unrelated interrupts but fail on actual errors */
+    do {
+        ret = thread_sleep(NO_TIMEOUT);
+        if (ret < 0 && ret != -EINTR && ret != -EAGAIN)
+            goto out;
+    } while (ret != 0);
+
+    debug("Finished waiting for response (seq = %lu, ret = %d)\n",
+          msg->msg.seq, msg->retval);
     ret = msg->retval;
 out:
-    close_ipc_message_duplex(msg, port);
+    lock(&port->msgs_lock);
+    if (!LIST_EMPTY(msg, list))
+        LISTP_DEL_INIT(msg, &port->msgs, list);
+    unlock(&port->msgs_lock);
+
+    if (msg->thread) {
+        /* put reference to the thread acquired earlier */
+        put_thread(msg->thread);
+        msg->thread = NULL;
+    }
+
     return ret;
 }
 
-struct shim_ipc_info * create_ipc_port (IDTYPE vmid, bool listen)
-{
-    struct shim_ipc_info * proc = get_new_ipc_info(vmid, NULL, 0);
-    if (!proc)
+/* must be called with cur_process.lock taken */
+struct shim_ipc_info* create_ipc_info_cur_process(void) {
+    struct shim_ipc_info* info = create_ipc_info(cur_process.vmid, NULL, 0);
+    if (!info)
         return NULL;
 
     char uri[PIPE_URI_SIZE];
-    if (create_pipe(NULL, uri, PIPE_URI_SIZE, &proc->pal_handle,
-                    &proc->uri) < 0) {
-        put_ipc_info(proc);
+    if (create_pipe(NULL, uri, PIPE_URI_SIZE, &info->pal_handle,
+                    &info->uri) < 0) {
+        put_ipc_info(info);
         return NULL;
     }
 
-    if (listen)
-        add_ipc_port_by_id(0, proc->pal_handle, IPC_PORT_SERVER,
-                           NULL, &proc->port);
-    return proc;
+    add_ipc_port_by_id(0, info->pal_handle, IPC_PORT_SERVER,
+            NULL, &info->port);
+
+    return info;
 }
 
-int create_ipc_location (struct shim_ipc_info ** info)
-{
+int get_ipc_info_cur_process(struct shim_ipc_info** info) {
     lock(&cur_process.lock);
-    int ret = -EACCES;
 
-    if (cur_process.self)
-        goto success;
-
-    cur_process.self = create_ipc_port(cur_process.vmid, true);
-    if (!cur_process.self)
-        goto out;
+    if (!cur_process.self) {
+        cur_process.self = create_ipc_info_cur_process();
+        if (!cur_process.self) {
+            unlock(&cur_process.lock);
+            return -EACCES;
+        }
+    }
 
-success:
     get_ipc_info(cur_process.self);
     *info = cur_process.self;
-    ret = 0;
-out:
-    unlock(&cur_process.lock);
-    return ret;
-}
-
-DEFINE_PROFILE_INTERVAL(ipc_finduri_send, ipc);
-DEFINE_PROFILE_INTERVAL(ipc_finduri_callback, ipc);
-
-int ipc_finduri_send (struct shim_ipc_port * port, IDTYPE dest,
-                      struct shim_ipc_info ** info)
-{
-    BEGIN_PROFILE_INTERVAL();
-    int ret;
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_FINDURI, 0, dest);
-
-    debug("ipc send to %u: IPC_FINDURI\n", dest);
 
-    ret = do_ipc_duplex(msg, port, NULL, info);
-    SAVE_PROFILE_INTERVAL(ipc_finduri_send);
-    return ret;
-}
-
-int ipc_finduri_callback (IPC_CALLBACK_ARGS)
-{
-    BEGIN_PROFILE_INTERVAL();
-    int ret = 0;
-
-    debug("ipc callback from %u: IPC_FINDURI\n", msg->src);
-
-    struct shim_ipc_info * info;
-
-    if ((ret = create_ipc_location(&info)) < 0)
-        goto out;
-
-    ret = ipc_telluri_send(port, msg->src, info);
-out:
-    SAVE_PROFILE_INTERVAL(ipc_finduri_callback);
-    return ret;
-}
-
-DEFINE_PROFILE_INTERVAL(ipc_telluri_send, ipc);
-DEFINE_PROFILE_INTERVAL(ipc_telluri_callback, ipc);
-
-int ipc_telluri_send (struct shim_ipc_port * port, IDTYPE dest,
-                      struct shim_ipc_info * info)
-{
-    BEGIN_PROFILE_INTERVAL();
-    int ret;
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_TELLURI,
-                                        info->uri.len, dest);
-    struct shim_ipc_telluri * msgin =
-                (struct shim_ipc_telluri *) &msg->msg;
-
-    if (qstrempty(&info->uri)) {
-        ret = -ENOENT;
-        return ret;
-    }
-
-    memcpy(msgin->uri, qstrgetstr(&info->uri), info->uri.len + 1);
-
-    debug("ipc send to %u: IPC_TELLURI(%s)\n", dest,
-          qstrgetstr(&info->uri));
-
-    ret = send_ipc_message(msg, port);
-    SAVE_PROFILE_INTERVAL(ipc_telluri_send);
-    return ret;
-}
-
-int ipc_telluri_callback (IPC_CALLBACK_ARGS)
-{
-    BEGIN_PROFILE_INTERVAL();
-    int ret = 0;
-    struct shim_ipc_telluri * msgin =
-                (struct shim_ipc_telluri *) &msg->msg;
-
-    debug("ipc callback from %u: IPC_TELLURI(%s)\n", msg->src, msgin->uri);
-
-    struct shim_ipc_info * info =
-            lookup_and_alloc_client(msg->src, msgin->uri);
-
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
-
-    if (obj) {
-        if (info) {
-            if (obj->private)
-                *(struct shim_ipc_info **) obj->private = info;
-            obj->retval = 0;
-        } else {
-            obj->retval = -ENOMEM;
-        }
-
-        if (obj->thread)
-            thread_wakeup(obj->thread);
-    }
-
-    SAVE_PROFILE_INTERVAL(ipc_telluri_callback);
-    return ret;
+    unlock(&cur_process.lock);
+    return 0;
 }
 
 DEFINE_PROFILE_INTERVAL(ipc_checkpoint_send, ipc);
 DEFINE_PROFILE_INTERVAL(ipc_checkpoint_callback, ipc);
 
-int ipc_checkpoint_send (const char * cpdir, IDTYPE cpsession)
-{
+/* Graphene's checkpoint() syscall broadcasts a msg to all processes
+ * asking to checkpoint their state and save in process-unique file in
+ * directory cpdir under session cpsession. */
+int ipc_checkpoint_send(const char* cpdir, IDTYPE cpsession) {
     BEGIN_PROFILE_INTERVAL();
     int ret;
-    int len = strlen(cpdir);
+    size_t len = strlen(cpdir);
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_CHECKPOINT,
-                                        sizeof(struct shim_ipc_checkpoint)
-                                        + len, 0);
-    struct shim_ipc_checkpoint * msgin =
-                    (struct shim_ipc_checkpoint *) &msg->msg;
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_checkpoint) + len);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_CHECKPOINT, total_msg_size, 0);
 
+    struct shim_ipc_checkpoint* msgin = (struct shim_ipc_checkpoint *) &msg->msg;
     msgin->cpsession = cpsession;
     memcpy(&msgin->cpdir, cpdir, len + 1);
 
-    debug("ipc broadcast to all: IPC_CHECKPOINT(%u, %s)\n",
-          cpsession, cpdir);
+    debug("IPC broadcast to all: IPC_CHECKPOINT(%u, %s)\n", cpsession, cpdir);
 
+    /* broadcast to all including myself (so I can also checkpoint) */
     ret = broadcast_ipc(msg, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT, /*exclude_port*/ NULL);
     SAVE_PROFILE_INTERVAL(ipc_checkpoint_send);
     return ret;
 }
 
-int ipc_checkpoint_callback (IPC_CALLBACK_ARGS)
-{
+/* This process is asked to create a checkpoint, so it:
+ * - sends a Graphene-specific SIGCP signal to all its threads (for
+ *   all to stop and join the checkpoint for consistent state),
+ * - broadcasts checkpoint msg further to other processes. */
+int ipc_checkpoint_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* port) {
     BEGIN_PROFILE_INTERVAL();
     int ret = 0;
-    struct shim_ipc_checkpoint * msgin =
-                (struct shim_ipc_checkpoint *) msg->msg;
+    struct shim_ipc_checkpoint* msgin = (struct shim_ipc_checkpoint *) msg->msg;
 
-    debug("ipc callback form %u: IPC_CHECKPOINT(%u, %s)\n", msg->src,
-          msgin->cpsession, msgin->cpdir);
+    debug("IPC callback from %u: IPC_CHECKPOINT(%u, %s)\n",
+          msg->src, msgin->cpsession, msgin->cpdir);
 
     ret = create_checkpoint(msgin->cpdir, &msgin->cpsession);
     if (ret < 0)
         goto out;
 
     kill_all_threads(NULL, msgin->cpsession, SIGCP);
-    broadcast_ipc(msg, IPC_PORT_DIRPRT|IPC_PORT_DIRCLD, port);
+    broadcast_ipc(msg, IPC_PORT_DIRCLD|IPC_PORT_DIRPRT, port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_checkpoint_callback);
     return ret;
 }
 
-BEGIN_CP_FUNC(ipc_info)
-{
+BEGIN_CP_FUNC(ipc_info) {
     assert(size == sizeof(struct shim_ipc_info));
 
-    struct shim_ipc_info * port = (struct shim_ipc_info *) obj;
-    struct shim_ipc_info * new_port = NULL;
+    struct shim_ipc_info* info = (struct shim_ipc_info *) obj;
+    struct shim_ipc_info* new_info = NULL;
 
     ptr_t off = GET_FROM_CP_MAP(obj);
 
     if (!off) {
         off = ADD_CP_OFFSET(sizeof(struct shim_ipc_info));
+        ADD_TO_CP_MAP(obj, off);
 
-        new_port = (struct shim_ipc_info *) (base + off);
-        memcpy(new_port, port, sizeof(struct shim_ipc_info));
-        REF_SET(new_port->ref_count, 0);
-
-        DO_CP_IN_MEMBER(qstr, new_port, uri);
-
-        if (port->pal_handle &&
-            port->pal_handle != IPC_FORCE_RECONNECT) {
-            struct shim_palhdl_entry * entry;
-            DO_CP(palhdl, port->pal_handle, &entry);
-            entry->uri = &new_port->uri;
-            entry->phandle = &new_port->pal_handle;
+        new_info = (struct shim_ipc_info *) (base + off);
+        memcpy(new_info, info, sizeof(struct shim_ipc_info));
+        REF_SET(new_info->ref_count, 0);
+
+        /* call qstr-specific checkpointing function for new_info->uri */
+        DO_CP_IN_MEMBER(qstr, new_info, uri);
+
+        if (info->pal_handle && info->pal_handle != IPC_FORCE_RECONNECT) {
+            struct shim_palhdl_entry* entry;
+            /* call palhdl-specific checkpointing function to checkpoint
+             * info->pal_handle and return created object in entry */
+            DO_CP(palhdl, info->pal_handle, &entry);
+            /* info's PAL handle will be re-opened with new URI during
+             * palhdl restore (see checkpoint.c) */
+            entry->uri = &new_info->uri;
+            entry->phandle = &new_info->pal_handle;
         }
     } else {
-        new_port = (struct shim_ipc_info *) (base + off);
+        /* already checkpointed */
+        new_info = (struct shim_ipc_info *) (base + off);
     }
 
-    if (new_port && objp)
-        *objp = (void *) new_port;
+    if (new_info && objp)
+        *objp = (void *) new_info;
 }
 END_CP_FUNC_NO_RS(ipc_info)
 
-BEGIN_CP_FUNC(process)
-{
+BEGIN_CP_FUNC(process) {
     assert(size == sizeof(struct shim_process));
 
-    struct shim_process * proc = (struct shim_process *) obj;
-    struct shim_process * new_proc = NULL;
+    struct shim_process* process = (struct shim_process *) obj;
+    struct shim_process* new_process = NULL;
 
     ptr_t off = GET_FROM_CP_MAP(obj);
 
@@ -681,57 +480,55 @@ BEGIN_CP_FUNC(process)
         off = ADD_CP_OFFSET(sizeof(struct shim_process));
         ADD_TO_CP_MAP(obj, off);
 
-        new_proc = (struct shim_process *) (base + off);
-        memcpy(new_proc, proc, sizeof(struct shim_process));
+        new_process = (struct shim_process *) (base + off);
+        memcpy(new_process, process, sizeof(struct shim_process));
 
-        if (proc->self)
-            DO_CP_MEMBER(ipc_info, proc, new_proc, self);
-
-        if (proc->parent)
-            DO_CP_MEMBER(ipc_info, proc, new_proc, parent);
-
-        for (int i = 0 ; i < TOTAL_NS ; i++)
-            if (proc->ns[i])
-                DO_CP_MEMBER(ipc_info, proc, new_proc, ns[i]);
+        /* call ipc_info-specific checkpointing functions
+         * for new_process's self, parent, and ns infos */
+        if (process->self)
+            DO_CP_MEMBER(ipc_info, process, new_process, self);
+        if (process->parent)
+            DO_CP_MEMBER(ipc_info, process, new_process, parent);
+        for (int i = 0; i < TOTAL_NS; i++)
+            if (process->ns[i])
+                DO_CP_MEMBER(ipc_info, process, new_process, ns[i]);
 
         ADD_CP_FUNC_ENTRY(off);
     } else {
-        new_proc = (struct shim_process *) (base + off);
+        /* already checkpointed */
+        new_process = (struct shim_process *) (base + off);
     }
 
     if (objp)
-        *objp = (void *) new_proc;
+        *objp = (void *) new_process;
 }
 END_CP_FUNC(process)
 
-BEGIN_RS_FUNC(process)
-{
+BEGIN_RS_FUNC(process) {
     __UNUSED(offset);
-    struct shim_process * proc = (void *) (base + GET_CP_FUNC_ENTRY());
+    struct shim_process* process = (void *) (base + GET_CP_FUNC_ENTRY());
 
-    CP_REBASE(proc->self);
-    CP_REBASE(proc->parent);
-    CP_REBASE(proc->ns);
+    CP_REBASE(process->self);
+    CP_REBASE(process->parent);
+    CP_REBASE(process->ns);
 
-    if (proc->self) {
-        proc->self->vmid = cur_process.vmid;
-        get_ipc_info(proc->self);
+    if (process->self) {
+        process->self->vmid = cur_process.vmid;
+        get_ipc_info(process->self);
     }
-
-    if (proc->parent)
-        get_ipc_info(proc->parent);
-
-    for (int i = 0 ; i < TOTAL_NS ; i++)
-        if (proc->ns[i])
-            get_ipc_info(proc->ns[i]);
-
-    proc->vmid = cur_process.vmid;
-    memcpy(&cur_process, proc, sizeof(struct shim_process));
+    if (process->parent)
+        get_ipc_info(process->parent);
+    for (int i = 0; i < TOTAL_NS; i++)
+        if (process->ns[i])
+            get_ipc_info(process->ns[i]);
+
+    process->vmid = cur_process.vmid;
+    memcpy(&cur_process, process, sizeof(struct shim_process));
     create_lock(&cur_process.lock);
 
-    DEBUG_RS("vmid=%u,uri=%s,parent=%u(%s)", proc->vmid,
-             proc->self ? qstrgetstr(&proc->self->uri) : "",
-             proc->parent ? proc->parent->vmid : 0,
-             proc->parent ? qstrgetstr(&proc->parent->uri) : "");
+    DEBUG_RS("vmid=%u,uri=%s,parent=%u(%s)", process->vmid,
+             process->self ? qstrgetstr(&process->self->uri) : "",
+             process->parent ? process->parent->vmid : 0,
+             process->parent ? qstrgetstr(&process->parent->uri) : "");
 }
 END_RS_FUNC(process)

LibOS/shim/src/ipc/shim_ipc_child.c

@@ -176,9 +176,10 @@ int ipc_cld_exit_send (IDTYPE ppid, IDTYPE tid, unsigned int exitcode, unsigned
     BEGIN_PROFILE_INTERVAL_SET(send_time);
     int ret = 0;
 
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(IPC_CLD_EXIT,
-                                    sizeof(struct shim_ipc_cld_exit), 0);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_cld_exit));
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_CLD_EXIT, total_msg_size, 0);
+
     struct shim_ipc_cld_exit * msgin =
                 (struct shim_ipc_cld_exit *) &msg->msg;
     msgin->ppid = ppid;
@@ -236,8 +237,9 @@ int ipc_cld_join_send (IDTYPE dest)
     if (!port)
         return -ESRCH;
 
-    struct shim_ipc_msg * msg =
-                create_ipc_msg_on_stack(IPC_CLD_JOIN, 0, dest);
+    size_t total_msg_size = get_ipc_msg_size(0);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_CLD_JOIN, total_msg_size, dest);
 
     debug("ipc send to %u: IPC_CLD_JOIN\n", dest);
 
@@ -285,11 +287,11 @@ int ipc_cld_profile_send (void)
         }
 
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_CLD_PROFILE,
-                                        sizeof(struct shim_ipc_cld_profile) +
-                                        sizeof(struct profile_val) *
-                                        nsending, dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_cld_profile) +
+                                             sizeof(struct profile_val) * nsending);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_CLD_PROFILE, total_msg_size, dest);
+
     struct shim_ipc_cld_profile * msgin =
                 (struct shim_ipc_cld_profile *) &msg->msg;
 

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -57,8 +57,6 @@ static int ipc_resp_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* por
 
 static ipc_callback ipc_callbacks[IPC_CODE_NUM] = {
     /* RESP             */  &ipc_resp_callback,
-    /* FINDURI          */  &ipc_finduri_callback,
-    /* TELLURI          */  &ipc_telluri_callback,
     /* CHECKPOINT       */  &ipc_checkpoint_callback,
 
     /* parents and children */
@@ -244,8 +242,8 @@ static void __del_ipc_port(struct shim_ipc_port* port) {
 
     /* Check for pending messages on port (threads might be blocking for responses) */
     lock(&port->msgs_lock);
-    struct shim_ipc_msg_obj* msg;
-    struct shim_ipc_msg_obj* tmp;
+    struct shim_ipc_msg_duplex* msg;
+    struct shim_ipc_msg_duplex* tmp;
     LISTP_FOR_EACH_ENTRY_SAFE(msg, tmp, &port->msgs, list) {
         LISTP_DEL_INIT(msg, &port->msgs, list);
         msg->retval = -ECONNRESET;
@@ -447,7 +445,7 @@ static int ipc_resp_callback(struct shim_ipc_msg* msg, struct shim_ipc_port* por
         return resp->retval;
 
     /* find a corresponding request msg for this response msg */
-    struct shim_ipc_msg_obj* req_msg = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex* req_msg = pop_ipc_msg_duplex(port, msg->seq);
 
     /* if some thread waits for response, wake it up with response retval */
     if (req_msg) {
@@ -464,7 +462,14 @@ int send_response_ipc_message(struct shim_ipc_port* port, IDTYPE dest, int ret,
     ret = (ret == RESPONSE_CALLBACK) ? 0 : ret;
 
     /* create IPC_RESP msg to send to dest, with sequence number seq, and in-body retval ret */
-    struct shim_ipc_msg* resp_msg = create_ipc_resp_msg_on_stack(ret, dest, seq);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_resp));
+    struct shim_ipc_msg* resp_msg = __alloca(total_msg_size);
+    init_ipc_msg(resp_msg, IPC_RESP, total_msg_size, dest);
+    resp_msg->seq = seq;
+
+    struct shim_ipc_resp* resp = (struct shim_ipc_resp *) resp_msg->msg;
+    resp->retval = ret;
+
     debug("IPC send to %u: IPC_RESP(%d)\n", resp_msg->dst & 0xFFFF, ret);
     return send_ipc_message(resp_msg, port);
 }
@@ -490,6 +495,7 @@ static int receive_ipc_message(struct shim_ipc_port* port) {
                 msg = tmp_buf;
             }
 
+            /* TODO: Add while-loop to receive all msg */
             int read = DkStreamRead(port->pal_handle, /*offset*/ 0, expected_size - bytes + readahead,
                                      (void *) msg + bytes, NULL, 0);
 
@@ -508,7 +514,7 @@ static int receive_ipc_message(struct shim_ipc_port* port) {
                 expected_size = msg->size;
         }
 
-        debug("Received IPC message from port %p (handle %p): code=%d size=%d src=%u dst=%u seq=%lx\n",
+        debug("Received IPC message from port %p (handle %p): code=%d size=%lu src=%u dst=%u seq=%lx\n",
               port, port->pal_handle, msg->code, msg->size, msg->src & 0xFFFF, msg->dst & 0xFFFF, msg->seq);
 
         /* skip messages coming from myself (in case of broadcast) */

LibOS/shim/src/ipc/shim_ipc_nsimpl.h

@@ -266,7 +266,7 @@ static int __add_range (struct range * r, IDTYPE off, IDTYPE owner,
     r->subranges = NULL;
 
     if (owner) {
-        r->owner = lookup_and_alloc_client(owner, uri);
+        r->owner = create_ipc_info_in_list(owner, uri);
         if (!r->owner)
             return -ENOMEM;
     }
@@ -291,7 +291,7 @@ static int __add_range (struct range * r, IDTYPE off, IDTYPE owner,
                 }
 
                 if (tmp->owner)
-                    put_client(tmp->owner);
+                    put_ipc_info_in_list(tmp->owner);
 
                 r->used = tmp->used;
                 r->subranges = tmp->subranges;
@@ -367,7 +367,7 @@ int CONCAT3(add, NS, subrange) (IDTYPE idx, IDTYPE owner,
     assert(owner);
     lock(&range_map_lock);
 
-    s->owner = lookup_and_alloc_client(owner, uri);
+    s->owner = create_ipc_info_in_list(owner, uri);
     if (!s->owner) {
         err = -ENOMEM;
         goto failed;
@@ -770,8 +770,7 @@ static void __discover_ns (bool block, bool need_locate)
     if (NS_LEADER) {
         if (NS_LEADER->vmid == cur_process.vmid) {
             if (need_locate && qstrempty(&NS_LEADER->uri)) {
-                struct shim_ipc_info * info = create_ipc_port(cur_process.vmid,
-                                                              true);
+                struct shim_ipc_info * info = create_ipc_info_cur_process();
                 if (info) {
                     put_ipc_info(NS_LEADER);
                     NS_LEADER = info;
@@ -810,14 +809,14 @@ static void __discover_ns (bool block, bool need_locate)
 
     // If all other ways failed, the current process becomes the leader
     if (!need_locate) {
-        NS_LEADER = get_new_ipc_info(cur_process.vmid, NULL, 0);
+        NS_LEADER = create_ipc_info(cur_process.vmid, NULL, 0);
         goto out;
     }
 
     if (NS_LEADER)
         put_ipc_info(NS_LEADER);
 
-    if (!(NS_LEADER = create_ipc_port(cur_process.vmid, true)))
+    if (!(NS_LEADER = create_ipc_info_cur_process()))
         goto out;
 
     // Finally, set the IPC port as a leadership port
@@ -995,17 +994,19 @@ int NS_SEND(findns) (bool block)
     unlock(&cur_process.lock);
 
     if (block) {
-        struct shim_ipc_msg_obj * msg =
-            create_ipc_msg_duplex_on_stack(NS_CODE(FINDNS), 0, dest);
+        size_t total_msg_size = get_ipc_msg_duplex_size(0);
+        struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+        init_ipc_msg_duplex(msg, NS_CODE(FINDNS), total_msg_size, dest);
 
         debug("ipc send to %u: " NS_CODE_STR(FINDNS) "\n", dest);
 
-        ret = do_ipc_duplex(msg, port, NULL, NULL);
+        ret = send_ipc_message_duplex(msg, port, NULL, NULL);
         goto out_port;
     }
 
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(NS_CODE(FINDNS), 0, dest);
+    size_t total_msg_size = get_ipc_msg_size(0);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, NS_CODE(FINDNS), total_msg_size, dest);
 
     debug("ipc send to %u: " NS_CODE_STR(FINDNS) "\n", dest);
 
@@ -1057,10 +1058,10 @@ int NS_SEND(tellns) (struct shim_ipc_port * port, IDTYPE dest,
                      struct shim_ipc_info * leader, unsigned long seq)
 {
     BEGIN_PROFILE_INTERVAL();
-    struct shim_ipc_msg * msg =
-        create_ipc_msg_on_stack(NS_CODE(TELLNS),
-                                leader->uri.len + sizeof(NS_MSG_TYPE(tellns)),
-                                dest);
+    size_t total_msg_size = get_ipc_msg_size(leader->uri.len + sizeof(NS_MSG_TYPE(tellns)));
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, NS_CODE(TELLNS), total_msg_size, dest);
+
     NS_MSG_TYPE(tellns) * msgin = (void *) &msg->msg;
     msgin->vmid = leader->vmid;
     memcpy(msgin->uri, qstrgetstr(&leader->uri), leader->uri.len + 1);
@@ -1089,7 +1090,7 @@ int NS_CALLBACK(tellns) (IPC_CALLBACK_ARGS)
         NS_LEADER->vmid = msgin->vmid;
         qstrsetstr(&NS_LEADER->uri, msgin->uri, strlen(msgin->uri));
     } else {
-        NS_LEADER = get_new_ipc_info(msgin->vmid, msgin->uri,
+        NS_LEADER = create_ipc_info(msgin->vmid, msgin->uri,
                                       strlen(msgin->uri));
         if (!NS_LEADER) {
             ret = -ENOMEM;
@@ -1109,7 +1110,7 @@ int NS_CALLBACK(tellns) (IPC_CALLBACK_ARGS)
         free(query);
     }
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (obj && obj->thread)
         thread_wakeup(obj->thread);
 
@@ -1133,7 +1134,7 @@ int NS_SEND(lease) (LEASETYPE * lease)
     if ((ret = connect_ns(&leader, &port)) < 0)
         goto out;
 
-    if ((ret = create_ipc_location(&self)) < 0)
+    if ((ret = get_ipc_info_cur_process(&self)) < 0)
         goto out;
 
     if (leader == cur_process.vmid) {
@@ -1145,10 +1146,10 @@ int NS_SEND(lease) (LEASETYPE * lease)
     }
 
     int len = self->uri.len;
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        NS_CODE(LEASE),
-                                        len + sizeof(NS_MSG_TYPE(lease)),
-                                        leader);
+    size_t total_msg_size = get_ipc_msg_duplex_size(len + sizeof(NS_MSG_TYPE(lease)));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(LEASE), total_msg_size, leader);
+
     NS_MSG_TYPE(lease) * msgin = (void *) &msg->msg.msg;
     assert(!qstrempty(&self->uri));
     memcpy(msgin->uri, qstrgetstr(&self->uri), len + 1);
@@ -1157,7 +1158,7 @@ int NS_SEND(lease) (LEASETYPE * lease)
     debug("ipc send to %u: " NS_CODE_STR(LEASE) "(%s)\n", leader,
           msgin->uri);
 
-    ret = do_ipc_duplex(msg, port, NULL, lease);
+    ret = send_ipc_message_duplex(msg, port, NULL, lease);
 out:
     if (port)
         put_ipc_port(port);
@@ -1195,8 +1196,10 @@ int NS_SEND(offer) (struct shim_ipc_port * port, IDTYPE dest, IDTYPE base,
 {
     BEGIN_PROFILE_INTERVAL();
     int ret = 0;
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(NS_CODE(OFFER),
-                                        sizeof(NS_MSG_TYPE(offer)), dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(NS_MSG_TYPE(offer)));
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, NS_CODE(OFFER), total_msg_size, dest);
+
     NS_MSG_TYPE(offer) * msgin = (void *) &msg->msg;
     msgin->base  = base;
     msgin->size  = size;
@@ -1218,7 +1221,7 @@ int NS_CALLBACK(offer) (IPC_CALLBACK_ARGS)
     debug("ipc callback from %u: " NS_CODE_STR(OFFER) "(%u, %u, %lu)\n",
           msg->src, msgin->base, msgin->size, msgin->lease);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
 
     switch (msgin->size) {
         case RANGE_SIZE:
@@ -1265,9 +1268,10 @@ int NS_SEND(renew) (IDTYPE base, IDTYPE size)
     if ((ret = connect_ns(&leader, &port)) < 0)
         goto out;
 
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(NS_CODE(RENEW),
-                                    sizeof(NS_MSG_TYPE(renew)), leader);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(NS_MSG_TYPE(renew)));
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, NS_CODE(RENEW), total_msg_size, leader);
+
     NS_MSG_TYPE(renew) * msgin = (void *) &msg->msg;
     msgin->base = base;
     msgin->size = size;
@@ -1339,10 +1343,10 @@ int NS_SEND(sublease) (IDTYPE tenant, IDTYPE idx, const char * uri,
     }
 
     int len = strlen(uri);
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                            NS_CODE(SUBLEASE),
-                                            len + sizeof(NS_MSG_TYPE(sublease)),
-                                            leader);
+    size_t total_msg_size = get_ipc_msg_duplex_size(len + sizeof(NS_MSG_TYPE(sublease)));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(SUBLEASE), total_msg_size, leader);
+
     NS_MSG_TYPE(sublease) * msgin = (void *) &msg->msg.msg;
     msgin->tenant = tenant;
     msgin->idx = idx;
@@ -1351,7 +1355,7 @@ int NS_SEND(sublease) (IDTYPE tenant, IDTYPE idx, const char * uri,
     debug("ipc send to %u: " NS_CODE_STR(SUBLEASE) "(%u, %u, %s)\n",
           leader, tenant, idx, msgin->uri);
 
-    ret = do_ipc_duplex(msg, port, NULL, lease);
+    ret = send_ipc_message_duplex(msg, port, NULL, lease);
 out:
     if (port)
         put_ipc_port(port);
@@ -1399,17 +1403,16 @@ int NS_SEND(query) (IDTYPE idx)
         goto out;
     }
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                            NS_CODE(QUERY),
-                                            sizeof(NS_MSG_TYPE(query)),
-                                            leader);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(NS_MSG_TYPE(query)));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(QUERY), total_msg_size, leader);
 
     NS_MSG_TYPE(query) * msgin = (void *) &msg->msg.msg;
     msgin->idx = idx;
 
     debug("ipc send to %u: " NS_CODE_STR(QUERY) "(%u)\n", leader, idx);
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
 out:
     if (port)
         put_ipc_port(port);
@@ -1471,12 +1474,13 @@ int NS_SEND(queryall) (void)
     if (cur_process.vmid == leader)
         goto out;
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                            NS_CODE(QUERYALL), 0, leader);
+    size_t total_msg_size = get_ipc_msg_duplex_size(0);
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(QUERYALL), total_msg_size, leader);
 
     debug("ipc send to %u: " NS_CODE_STR(QUERYALL) "\n", leader);
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(NS_SEND(queryall));
@@ -1587,9 +1591,9 @@ int NS_SEND(answer) (struct shim_ipc_port * port, IDTYPE dest,
     for (int i = 0 ; i < nowners ; i++)
         total_ownerdatasz += ownerdatasz[i];
 
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(NS_CODE(ANSWER),
-                                    owner_offset + total_ownerdatasz, dest);
+    size_t total_msg_size = get_ipc_msg_size(owner_offset + total_ownerdatasz);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, NS_CODE(ANSWER), total_msg_size, dest);
 
     NS_MSG_TYPE(answer) * msgin = (void *) &msg->msg;
     msgin->nanswers = nanswers;
@@ -1646,7 +1650,7 @@ int NS_CALLBACK(answer) (IPC_CALLBACK_ARGS)
         }
     }
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (obj && obj->thread)
         thread_wakeup(obj->thread);
 
@@ -1746,17 +1750,17 @@ int NS_SEND(findkey) (NS_KEY * key)
         goto out;
     }
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        NS_CODE(FINDKEY),
-                                        sizeof(NS_MSG_TYPE(findkey)),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(NS_MSG_TYPE(findkey)));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(FINDKEY), total_msg_size, dest);
+
     NS_MSG_TYPE(findkey) * msgin = (void *) &msg->msg.msg;
     KEY_COPY(&msgin->key, key);
 
     debug("ipc send to %u: " NS_CODE_STR(FINDKEY) "(%lu)\n",
           dest, KEY_HASH(key));
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
     put_ipc_port(port);
 
     if (!ret)
@@ -1809,10 +1813,10 @@ int NS_SEND(tellkey) (struct shim_ipc_port * port, IDTYPE dest, NS_KEY * key,
     }
 
     if (owned) {
-        struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        NS_CODE(TELLKEY),
-                                        sizeof(NS_MSG_TYPE(tellkey)),
-                                        dest);
+        size_t total_msg_size = get_ipc_msg_size(sizeof(NS_MSG_TYPE(tellkey)));
+        struct shim_ipc_msg* msg = __alloca(total_msg_size);
+        init_ipc_msg(msg, NS_CODE(TELLKEY), total_msg_size, dest);
+
         NS_MSG_TYPE(tellkey) * msgin = (void *) &msg->msg;
         KEY_COPY(&msgin->key, key);
         msgin->id = id;
@@ -1825,10 +1829,10 @@ int NS_SEND(tellkey) (struct shim_ipc_port * port, IDTYPE dest, NS_KEY * key,
         goto out;
     }
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        NS_CODE(TELLKEY),
-                                        sizeof(NS_MSG_TYPE(tellkey)),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(NS_MSG_TYPE(tellkey)));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, NS_CODE(TELLKEY), total_msg_size, dest);
+
     NS_MSG_TYPE(tellkey) * msgin = (void *) &msg->msg.msg;
     KEY_COPY(&msgin->key, key);
     msgin->id = id;
@@ -1836,7 +1840,7 @@ int NS_SEND(tellkey) (struct shim_ipc_port * port, IDTYPE dest, NS_KEY * key,
     debug("ipc send to %u: IPC_SYSV_TELLKEY(%lu, %u)\n", dest,
           KEY_HASH(key), id);
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(NS_SEND(tellkey));
@@ -1854,7 +1858,7 @@ int NS_CALLBACK(tellkey) (IPC_CALLBACK_ARGS)
 
     ret = CONCAT2(NS, add_key)(&msgin->key, msgin->id);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (!obj) {
         ret = RESPONSE_CALLBACK;
         goto out;

LibOS/shim/src/ipc/shim_ipc_pid.c

@@ -61,7 +61,7 @@ int init_ns_pid (void)
 
     init_namespace();
 
-    if ((ret = create_ipc_location(&info)) < 0)
+    if ((ret = get_ipc_info_cur_process(&info)) < 0)
         return ret;
 
     walk_thread_list(&thread_add_subrange, info);
@@ -73,9 +73,9 @@ int broadcast_signal (IDTYPE sender, int signum)
     BEGIN_PROFILE_INTERVAL();
     int ret;
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_PID_KILL,
-                                        sizeof(struct shim_ipc_pid_kill), 0);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_pid_kill));
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_PID_KILL, total_msg_size, 0);
     struct shim_ipc_pid_kill * msgin =
                     (struct shim_ipc_pid_kill *) &msg->msg;
 
@@ -103,13 +103,12 @@ int ipc_pid_kill_send (IDTYPE sender, IDTYPE id, enum kill_type type,
     if ((ret = connect_owner(id, &port, &dest)) < 0)
         goto out;
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_PID_KILL,
-                                        sizeof(struct shim_ipc_pid_kill),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_pid_kill));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_PID_KILL, total_msg_size, dest);
+
     struct shim_ipc_pid_kill * msgin =
                     (struct shim_ipc_pid_kill *) &msg->msg.msg;
-
     msgin->sender = sender;
     msgin->id     = id;
     msgin->type   = type;
@@ -118,7 +117,7 @@ int ipc_pid_kill_send (IDTYPE sender, IDTYPE id, enum kill_type type,
     debug("ipc send to %u: IPC_PID_KILL(%u, %d, %u, %d)\n", dest,
           sender, type, id, signum);
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_pid_kill_send);
@@ -171,21 +170,20 @@ int ipc_pid_getstatus_send (struct shim_ipc_port * port, IDTYPE dest,
     BEGIN_PROFILE_INTERVAL();
     int ret;
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_PID_GETSTATUS,
-                                        sizeof(struct shim_ipc_pid_getstatus) +
-                                        sizeof(IDTYPE) * npids,
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_pid_getstatus) +
+                                                 sizeof(IDTYPE) * npids);
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_PID_GETSTATUS, total_msg_size, dest);
+
     struct shim_ipc_pid_getstatus * msgin =
                     (struct shim_ipc_pid_getstatus *) &msg->msg.msg;
-
     msgin->npids = npids;
     memcpy(msgin->pids, pids, sizeof(IDTYPE) * npids);
 
     debug("ipc send to %u: IPC_PID_GETSTATUS(%d, [%u, ...])\n", dest,
           npids, pids[0]);
 
-    ret = do_ipc_duplex(msg, port, NULL, status);
+    ret = send_ipc_message_duplex(msg, port, NULL, status);
 
     SAVE_PROFILE_INTERVAL(ipc_pid_getstatus_send);
     return ret;
@@ -254,11 +252,11 @@ int ipc_pid_retstatus_send (struct shim_ipc_port * port, IDTYPE dest,
     BEGIN_PROFILE_INTERVAL();
     int ret;
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_PID_RETSTATUS,
-                                        sizeof(struct shim_ipc_pid_retstatus) +
-                                        sizeof(struct pid_status) * nstatus,
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_pid_retstatus) +
+                                             sizeof(struct pid_status) * nstatus);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_PID_RETSTATUS, total_msg_size, dest);
+
     struct shim_ipc_pid_retstatus * msgin =
                     (struct shim_ipc_pid_retstatus *) &msg->msg;
 
@@ -291,7 +289,7 @@ int ipc_pid_retstatus_callback (IPC_CALLBACK_ARGS)
         debug("ipc callback from %u: IPC_PID_RETSTATUS(0, [])\n", msg->src);
 
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (obj) {
         struct pid_status ** status = (struct pid_status **) obj->private;
 
@@ -485,10 +483,10 @@ int ipc_pid_getmeta_send (IDTYPE pid, enum pid_meta_code code,
     if ((ret = connect_owner(pid, &port, &dest)) < 0)
         goto out;
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_PID_GETMETA,
-                                        sizeof(struct shim_ipc_pid_getmeta),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_pid_getmeta));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_PID_GETMETA, total_msg_size, dest);
+
     struct shim_ipc_pid_getmeta * msgin =
                     (struct shim_ipc_pid_getmeta *) &msg->msg.msg;
     msgin->pid  = pid;
@@ -497,7 +495,7 @@ int ipc_pid_getmeta_send (IDTYPE pid, enum pid_meta_code code,
     debug("ipc send to %u: IPC_PID_GETMETA(%u, %s)\n", dest,
           pid, pid_meta_code_str[code]);
 
-    ret = do_ipc_duplex(msg, port, NULL, data);
+    ret = send_ipc_message_duplex(msg, port, NULL, data);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_pid_getmeta_send);
@@ -582,10 +580,9 @@ int ipc_pid_retmeta_send (struct shim_ipc_port * port, IDTYPE dest,
     BEGIN_PROFILE_INTERVAL();
     int ret;
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_PID_RETMETA,
-                                        sizeof(struct shim_ipc_pid_retmeta) +
-                                        datasize, dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_pid_retmeta) + datasize);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_PID_RETMETA, total_msg_size, dest);
     struct shim_ipc_pid_retmeta * msgin =
                     (struct shim_ipc_pid_retmeta *) &msg->msg;
 
@@ -613,7 +610,7 @@ int ipc_pid_retmeta_callback (IPC_CALLBACK_ARGS)
     debug("ipc callback from %u: IPC_PID_RETMETA(%u, %s, %d)\n", msg->src,
           msgin->pid, pid_meta_code_str[msgin->code], msgin->datasize);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (obj) {
         void ** data = (void **) obj->private;
 
@@ -652,13 +649,12 @@ int ipc_pid_nop_send (struct shim_ipc_port * port, IDTYPE dest, int count,
                       const void * buf, int len)
 {
     BEGIN_PROFILE_INTERVAL();
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_PID_NOP,
-                                        sizeof(struct shim_ipc_pid_nop) +
-                                        len, dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_pid_nop) + len);
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_PID_NOP, total_msg_size, dest);
+
     struct shim_ipc_pid_nop * msgin =
                 (struct shim_ipc_pid_nop *) &msg->msg.msg;
-
     msgin->count = count * 2;
     memcpy(msgin->payload, buf, len);
 
@@ -666,7 +662,7 @@ int ipc_pid_nop_send (struct shim_ipc_port * port, IDTYPE dest, int count,
 
     SAVE_PROFILE_INTERVAL(ipc_pid_nop_send);
 
-    return do_ipc_duplex(msg, port, NULL, NULL);
+    return send_ipc_message_duplex(msg, port, NULL, NULL);
 }
 
 int ipc_pid_nop_callback (IPC_CALLBACK_ARGS)
@@ -679,7 +675,7 @@ int ipc_pid_nop_callback (IPC_CALLBACK_ARGS)
           msgin->count);
 
     if (!(--msgin->count)) {
-        struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+        struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
         if (obj && obj->thread)
             thread_wakeup(obj->thread);
 
@@ -711,10 +707,9 @@ int ipc_pid_sendrpc_send (IDTYPE pid, IDTYPE sender, const void * buf,
     if ((ret = get_pid_port(pid, &dest, &port)) < 0)
         return ret;
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                     IPC_PID_SENDRPC,
-                                     sizeof(struct shim_ipc_pid_sendrpc) +
-                                     len, dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_pid_sendrpc) + len);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_PID_SENDRPC, total_msg_size, dest);
     struct shim_ipc_pid_sendrpc * msgin =
                     (struct shim_ipc_pid_sendrpc *) &msg->msg;
 

LibOS/shim/src/ipc/shim_ipc_sysv.c

@@ -71,10 +71,10 @@ int ipc_sysv_delres_send (struct shim_ipc_port * port, IDTYPE dest,
     }
 
     if (!owned) {
-        struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                        IPC_SYSV_DELRES,
-                        sizeof(struct shim_ipc_sysv_delres),
-                        dest);
+        size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_delres));
+        struct shim_ipc_msg* msg = __alloca(total_msg_size);
+        init_ipc_msg(msg, IPC_SYSV_DELRES, total_msg_size, dest);
+
         struct shim_ipc_sysv_delres * msgin = (struct shim_ipc_sysv_delres *)
             &msg->msg;
         msgin->resid = resid;
@@ -87,10 +87,10 @@ int ipc_sysv_delres_send (struct shim_ipc_port * port, IDTYPE dest,
         goto out;
     }
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_SYSV_DELRES,
-                                        sizeof(struct shim_ipc_sysv_delres),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_sysv_delres));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_SYSV_DELRES, total_msg_size, dest);
+
     struct shim_ipc_sysv_delres * msgin = (struct shim_ipc_sysv_delres *)
                                           &msg->msg.msg;
     msgin->resid = resid;
@@ -99,7 +99,7 @@ int ipc_sysv_delres_send (struct shim_ipc_port * port, IDTYPE dest,
     debug("ipc send to %u: IPC_SYSV_DELRES(%u, %s)\n", dest, resid,
           SYSV_TYPE_STR(type));
 
-    ret = do_ipc_duplex(msg, port, NULL, NULL);
+    ret = send_ipc_message_duplex(msg, port, NULL, NULL);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_sysv_delres_send);
@@ -160,10 +160,9 @@ int ipc_sysv_movres_send (struct sysv_client * client, IDTYPE owner,
     int ret = 0;
     int len = strlen(uri);
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_SYSV_MOVRES,
-                                        sizeof(struct shim_ipc_sysv_movres) +
-                                        len, client->vmid);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_movres) + len);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_MOVRES, total_msg_size, client->vmid);
     struct shim_ipc_sysv_movres * msgin = (struct shim_ipc_sysv_movres *)
                                           &msg->msg;
     msgin->resid = resid;
@@ -191,7 +190,7 @@ int ipc_sysv_movres_callback (IPC_CALLBACK_ARGS)
     debug("ipc callback from %u: IPC_SYSV_MOVRES(%u, %s, %u, %s)\n", msg->src,
           msgin->resid, SYSV_TYPE_STR(msgin->type), msgin->owner, msgin->uri);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (!obj)
         goto out;
 
@@ -233,10 +232,9 @@ int ipc_sysv_msgsnd_send (struct shim_ipc_port * port, IDTYPE dest,
         owned = false;
     }
 
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                    IPC_SYSV_MSGSND,
-                                    sizeof(struct shim_ipc_sysv_msgsnd) +
-                                    size, dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_msgsnd) + size);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_MSGSND, total_msg_size, dest);
     struct shim_ipc_sysv_msgsnd * msgin =
                                (struct shim_ipc_sysv_msgsnd *) &msg->msg;
     msgin->msgid = msgid;
@@ -269,7 +267,7 @@ int ipc_sysv_msgsnd_callback (IPC_CALLBACK_ARGS)
     size_t size = msg->size - sizeof(*msg) - sizeof(*msgin);
 
     if (msg->seq) {
-        struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+        struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
         void * priv = obj ? obj->private : NULL;
 
         if (priv) {
@@ -329,10 +327,10 @@ int ipc_sysv_msgrcv_send (IDTYPE msgid, long msgtype, int flags, void * buf,
 
     assert(port);
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_SYSV_MSGRCV,
-                                        sizeof(struct shim_ipc_sysv_msgrcv),
-                                        true);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_sysv_msgrcv));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_SYSV_MSGRCV, total_msg_size, owner);
+
     struct shim_ipc_sysv_msgrcv * msgin =
                 (struct shim_ipc_sysv_msgrcv *) &msg->msg.msg;
     msgin->msgid = msgid;
@@ -343,7 +341,7 @@ int ipc_sysv_msgrcv_send (IDTYPE msgid, long msgtype, int flags, void * buf,
     debug("ipc send to %u: IPC_SYSV_MSGRCV(%u, %ld)\n", owner,
           msgid, msgtype);
 
-    ret = do_ipc_duplex(msg, port, NULL, buf);
+    ret = send_ipc_message_duplex(msg, port, NULL, buf);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_sysv_msgrcv_send);
@@ -396,11 +394,10 @@ int ipc_sysv_msgmov_send (struct shim_ipc_port * port, IDTYPE dest,
                           struct sysv_score * scores, int nscores)
 {
     BEGIN_PROFILE_INTERVAL();
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(IPC_SYSV_MSGMOV,
-                                    sizeof(struct shim_ipc_sysv_msgmov) +
-                                    sizeof(struct sysv_score) * nscores,
-                                    dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_msgmov) +
+                                             sizeof(struct sysv_score) * nscores);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_MSGMOV, total_msg_size, dest);
     struct shim_ipc_sysv_msgmov * msgin =
                             (struct shim_ipc_sysv_msgmov *) &msg->msg;
 
@@ -450,7 +447,7 @@ int ipc_sysv_msgmov_callback (IPC_CALLBACK_ARGS)
     ret = recover_msg_ownership(msgq);
 
     struct shim_ipc_info * info;
-    if (!create_ipc_location(&info)) {
+    if (!get_ipc_info_cur_process(&info)) {
         add_sysv_subrange(msgin->msgid, info->vmid, qstrgetstr(&info->uri),
                           &msgin->lease);
         put_ipc_info(info);
@@ -491,11 +488,10 @@ int ipc_sysv_semop_send (IDTYPE semid, struct sembuf * sops, int nsops,
     assert(port);
 
     if (!waitforreply) {
-        struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_SYSV_SEMOP,
-                                        sizeof(struct shim_ipc_sysv_semop) +
-                                        sizeof(struct sembuf) * nsops,
-                                        owner);
+        size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_semop) +
+                                                 sizeof(struct sembuf) * nsops);
+        struct shim_ipc_msg* msg = __alloca(total_msg_size);
+        init_ipc_msg(msg, IPC_SYSV_SEMOP, total_msg_size, owner);
         struct shim_ipc_sysv_semop * msgin =
                 (struct shim_ipc_sysv_semop *) &msg->msg;
 
@@ -514,11 +510,11 @@ int ipc_sysv_semop_send (IDTYPE semid, struct sembuf * sops, int nsops,
         goto out;
     }
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_SYSV_SEMOP,
-                                        sizeof(struct shim_ipc_sysv_semop) +
-                                        sizeof(struct sembuf) * nsops,
-                                        owner);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_sysv_semop) +
+                                                 sizeof(struct sembuf) * nsops);
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_SYSV_SEMOP, total_msg_size, owner);
+
     struct shim_ipc_sysv_semop * msgin =
             (struct shim_ipc_sysv_semop *) &msg->msg.msg;
     msgin->semid   = semid;
@@ -530,7 +526,7 @@ int ipc_sysv_semop_send (IDTYPE semid, struct sembuf * sops, int nsops,
     debug("ipc send to %u: IPC_SYSV_SEMOP(%u, %ld, %u)\n", owner, semid,
           timeout, nsops);
 
-    ret = do_ipc_duplex(msg, port, seq, NULL);
+    ret = send_ipc_message_duplex(msg, port, seq, NULL);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_sysv_semop_send);
@@ -581,14 +577,12 @@ int ipc_sysv_semctl_send (IDTYPE semid, int semnum, int cmd, void * vals,
 
     int ctlvalsize = (cmd == SETALL || cmd == SETVAL) ? valsize : 0;
 
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_SYSV_SEMCTL,
-                                        sizeof(struct shim_ipc_sysv_semctl) +
-                                        ctlvalsize,
-                                        owner);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_sysv_semctl) + ctlvalsize);
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_SYSV_SEMCTL, total_msg_size, owner);
+
     struct shim_ipc_sysv_semctl * msgin =
                 (struct shim_ipc_sysv_semctl *) &msg->msg.msg;
-
     msgin->semid   = semid;
     msgin->semnum  = semnum;
     msgin->cmd     = cmd;
@@ -599,7 +593,7 @@ int ipc_sysv_semctl_send (IDTYPE semid, int semnum, int cmd, void * vals,
     debug("ipc send to %u: IPC_SYSV_SEMCTL(%u, %d, %d)\n", owner, semid,
           semnum, cmd);
 
-    ret = do_ipc_duplex(msg, port, NULL, vals);
+    ret = send_ipc_message_duplex(msg, port, NULL, vals);
     put_ipc_port(port);
 out:
     SAVE_PROFILE_INTERVAL(ipc_sysv_semctl_send);
@@ -706,11 +700,10 @@ int ipc_sysv_semret_send (struct shim_ipc_port * port, IDTYPE dest, void * vals,
 {
     BEGIN_PROFILE_INTERVAL();
     int ret = 0;
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_SYSV_SEMRET,
-                                        sizeof(struct shim_ipc_sysv_semret) +
-                                        valsize,
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_semret) + valsize);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_SEMRET, total_msg_size, dest);
+
     struct shim_ipc_sysv_semret * msgin =
                 (struct shim_ipc_sysv_semret *) &msg->msg;
     msgin->valsize = valsize;
@@ -732,7 +725,7 @@ int ipc_sysv_semret_callback (IPC_CALLBACK_ARGS)
 
     debug("ipc callback from %u: IPC_SYSV_SEMRET\n", msg->src);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (obj) {
         struct shim_ipc_sysv_semctl * semctl =
                                 (struct shim_ipc_sysv_semctl *) &obj->msg.msg;
@@ -773,13 +766,13 @@ int ipc_sysv_semmov_send (struct shim_ipc_port * port, IDTYPE dest,
                           struct sysv_score * scores, int nscores)
 {
     BEGIN_PROFILE_INTERVAL();
-    struct shim_ipc_msg * msg =
-            create_ipc_msg_on_stack(IPC_SYSV_SEMMOV,
-                                    sizeof(struct shim_ipc_sysv_semmov) +
-                                    sizeof(struct sem_backup) * nsems +
-                                    sizeof(struct sem_client_backup) * nsrcs +
-                                    sizeof(struct sysv_score) * nscores,
-                                    dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_semmov) +
+                                             sizeof(struct sem_backup) * nsems +
+                                             sizeof(struct sem_client_backup) * nsrcs +
+                                             sizeof(struct sysv_score) * nscores);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_SEMMOV, total_msg_size, dest);
+
     struct shim_ipc_sysv_semmov * msgin =
                             (struct shim_ipc_sysv_semmov *) &msg->msg;
     msgin->semid   = semid;
@@ -843,7 +836,7 @@ int ipc_sysv_semmov_callback (IPC_CALLBACK_ARGS)
                                 msgin->nsrcs);
 
     struct shim_ipc_info * info;
-    if (!create_ipc_location(&info)) {
+    if (!get_ipc_info_cur_process(&info)) {
         add_sysv_subrange(msgin->semid, info->vmid, qstrgetstr(&info->uri),
                           &msgin->lease);
         put_ipc_info(info);
@@ -877,17 +870,17 @@ int ipc_sysv_semquery_send (IDTYPE semid, int * nsems,
     }
 
     assert(port);
-    struct shim_ipc_msg_obj * msg = create_ipc_msg_duplex_on_stack(
-                                        IPC_SYSV_SEMQUERY,
-                                        sizeof(struct shim_ipc_sysv_semquery),
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_duplex_size(sizeof(struct shim_ipc_sysv_semquery));
+    struct shim_ipc_msg_duplex* msg = __alloca(total_msg_size);
+    init_ipc_msg_duplex(msg, IPC_SYSV_SEMQUERY, total_msg_size, dest);
+
     struct shim_ipc_sysv_semquery * msgin =
                 (struct shim_ipc_sysv_semquery *) &msg->msg.msg;
     msgin->semid = semid;
 
     debug("ipc send to %u: IPC_SYSV_SEMQUERY(%u)\n", dest, semid);
 
-    ret = do_ipc_duplex(msg, port, NULL, host_sem_ids);
+    ret = send_ipc_message_duplex(msg, port, NULL, host_sem_ids);
     put_ipc_port(port);
     if (ret >= 0) {
         *nsems = ret;
@@ -930,11 +923,11 @@ int ipc_sysv_semreply_send (struct shim_ipc_port * port, IDTYPE dest,
 {
     BEGIN_PROFILE_INTERVAL();
     int ret = 0;
-    struct shim_ipc_msg * msg = create_ipc_msg_on_stack(
-                                        IPC_SYSV_SEMREPLY,
-                                        sizeof(struct shim_ipc_sysv_semreply)
-                                        + sizeof(PAL_NUM) * nsems,
-                                        dest);
+    size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_sysv_semreply) +
+                                             sizeof(PAL_NUM) * nsems);
+    struct shim_ipc_msg* msg = __alloca(total_msg_size);
+    init_ipc_msg(msg, IPC_SYSV_SEMREPLY, total_msg_size, dest);
+
     struct shim_ipc_sysv_semreply * msgin =
                     (struct shim_ipc_sysv_semreply *) &msg->msg;
     msgin->semid = semid;
@@ -960,7 +953,7 @@ int ipc_sysv_semreply_callback (IPC_CALLBACK_ARGS)
     debug("ipc callback from %u: IPC_SYSV_SEMREPLY(%u, %d)\n", msg->src,
           msgin->semid, msgin->nsems);
 
-    struct shim_ipc_msg_obj * obj = find_ipc_msg_duplex(port, msg->seq);
+    struct shim_ipc_msg_duplex * obj = pop_ipc_msg_duplex(port, msg->seq);
     if (!obj)
         goto out;
 

LibOS/shim/src/shim_checkpoint.c

@@ -976,12 +976,19 @@ int do_migrate_process (int (*migrate) (struct shim_cp_store *,
     }
 
     /* Create process and IPC bookkeepings */
-    if (!(new_process = create_new_process(true))) {
+    if (!(new_process = create_process())) {
         ret = -ENOMEM;
         goto err;
     }
 
-    if (!(new_process->self = create_ipc_port(0, false))) {
+    if (!(new_process->self = create_ipc_info(0, NULL, 0))) {
+        ret = -EACCES;
+        goto err;
+    }
+
+    char pipe_uri[PIPE_URI_SIZE];
+    if (create_pipe(NULL, pipe_uri, PIPE_URI_SIZE, &new_process->self->pal_handle,
+                    &new_process->self->uri) < 0) {
         ret = -EACCES;
         goto err;
     }
@@ -1141,7 +1148,7 @@ int do_migrate_process (int (*migrate) (struct shim_cp_store *,
                        &ipc_child_exit,
                        NULL);
 
-    destroy_process(new_process);
+    free_process(new_process);
     return 0;
 err:
     if (gipc_hdl)
@@ -1149,7 +1156,7 @@ err:
     if (proc)
         DkObjectClose(proc);
     if (new_process)
-        destroy_process(new_process);
+        free_process(new_process);
 
     SYS_PRINTF("process creation failed\n");
     return ret;

LibOS/shim/src/sys/shim_msgget.c

@@ -998,7 +998,7 @@ static int msg_balance_migrate (struct shim_handle * hdl,
     if ((ret = __store_msg_persist(msgq)) < 0)
         return 0;
 
-    struct shim_ipc_info * info = discover_client(src->port, src->vmid);
+    struct shim_ipc_info * info = lookup_ipc_info(src->vmid);
     if (!info)
         goto failed;
 

LibOS/shim/src/sys/shim_semget.c

@@ -569,10 +569,15 @@ send_result:
                 continue;
             }
 
-            send_ipc_message(create_ipc_resp_msg_on_stack(
-                             sops->stat.completed ? 0 : -EAGAIN,
-                             sops->client.vmid,
-                             sops->client.seq), sops->client.port);
+            size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_resp));
+            struct shim_ipc_msg* resp_msg = __alloca(total_msg_size);
+            init_ipc_msg(resp_msg, IPC_RESP, total_msg_size, sops->client.vmid);
+            resp_msg->seq = sops->client.seq;
+
+            struct shim_ipc_resp* resp = (struct shim_ipc_resp *) resp_msg->msg;
+            resp->retval = sops->stat.completed ? 0 : -EAGAIN;
+
+            send_ipc_message(resp_msg, sops->client.port);
 
             put_ipc_port(sops->client.port);
             sops->client.vmid = 0;
@@ -786,10 +791,17 @@ unowned:
 
     if (stat.completed || stat.failed) {
         ret = stat.completed ? 0 : -EAGAIN;
-        if (client && sendreply)
-            ret = send_ipc_message(create_ipc_resp_msg_on_stack(
-                                            ret, client->vmid,
-                                            client->seq), client->port);
+        if (client && sendreply) {
+            size_t total_msg_size = get_ipc_msg_size(sizeof(struct shim_ipc_resp));
+            struct shim_ipc_msg* resp_msg = __alloca(total_msg_size);
+            init_ipc_msg(resp_msg, IPC_RESP, total_msg_size, client->vmid);
+            resp_msg->seq = client->seq;
+
+            struct shim_ipc_resp* resp = (struct shim_ipc_resp *) resp_msg->msg;
+            resp->retval = ret;
+
+            ret = send_ipc_message(resp_msg, client->port);
+        }
 
         SAVE_PROFILE_INTERVAL(sem_send_ipc_response);
         goto out_locked;
@@ -910,7 +922,7 @@ static int sem_balance_migrate (struct shim_handle * hdl,
         }
     }
 
-    struct shim_ipc_info * info = discover_client(src->port, src->vmid);
+    struct shim_ipc_info * info = lookup_ipc_info(src->vmid);
     if (!info)
         goto out;