Fixing a bunch of memory leaking issues (#86)

* add memusg to pal_loader script

* Stop bleeding PAL handles.

Deprecating DkOjectReference and reference counting in PAL handle.
Deprecating DkSemaphoreDestroy and DkEventDestroy (replaced by DkObjectClose).
Cleaning unused PAL handles in the library OS.
Adding a heap tracing feature to profile usage of PAL handles.

* fix a bug in SGX mode that mapping untrusted files into memory never got free by DkVirtualMemoryFree()

* adding lighttpd SSL option

* fixing the freeing convention of PAL handles; On SGX, event and mutex handles need to be freed seperately.

* changing how mutexes and events are allocated on SGX

* fixing GCC regression tests (for both Linux and SGX)

* fix a double-free problem of the first thread handle

LibOS/shim/src/bookkeep/shim_handle.c

@@ -543,8 +543,13 @@ void put_handle (struct shim_handle * hdl)
         qstrfree(&hdl->path);
         qstrfree(&hdl->uri);
 
-        if (hdl->pal_handle)
+        if (hdl->pal_handle) {
+#ifdef DEBUG_REF
+            debug("handle %p closes PAL handle %p\n", hdl, hdl->pal_handle);
+#endif
             DkObjectClose(hdl->pal_handle);
+            hdl->pal_handle = NULL;
+        }
 
         if (hdl->dentry)
             put_dentry(hdl->dentry);

LibOS/shim/src/bookkeep/shim_thread.c

@@ -322,12 +322,24 @@ void put_thread (struct shim_thread * thread)
 #endif
 
     if (!ref_count) {
-       if (thread->exec)
+        if (thread->exec)
             put_handle(thread->exec);
 
         if (!IS_INTERNAL(thread))
             release_pid(thread->tid);
 
+        if (thread->pal_handle &&
+            thread->pal_handle != PAL_CB(first_thread))
+            DkObjectClose(thread->pal_handle);
+
+        if (thread->scheduler_event)
+            DkObjectClose(thread->scheduler_event);
+        if (thread->exit_event)
+            DkObjectClose(thread->exit_event);
+        if (thread->child_exit_event)
+            DkObjectClose(thread->child_exit_event);
+        destroy_lock(thread->lock);
+
         if (MEMORY_MIGRATED(thread))
             memset(thread, 0, sizeof(struct shim_thread));
         else
@@ -347,6 +359,9 @@ void put_simple_thread (struct shim_simple_thread * thread)
     if (!ref_count) {
         /* Simple threads always live on the simple thread list */
         listp_del(thread, &simple_thread_list, list);
+        if (thread->exit_event)
+            DkObjectClose(thread->exit_event);
+        destroy_lock(thread->lock);
         free(thread);
     }
 }
@@ -719,7 +734,7 @@ BEGIN_RS_FUNC(running_thread)
     struct shim_thread * thread = (void *) (base + GET_CP_FUNC_ENTRY());
     struct shim_thread * cur_thread = get_cur_thread();
     thread->in_vm = true;
-    
+
     if (!thread->user_tcb)
         CP_REBASE(thread->tcb);
 
@@ -734,7 +749,7 @@ BEGIN_RS_FUNC(running_thread)
         thread->pal_handle = handle;
     } else {
         __libc_tcb_t * libc_tcb = (__libc_tcb_t *) thread->tcb;
-        
+
         if (libc_tcb) {
             shim_tcb_t * tcb = &libc_tcb->shim_tcb;
             assert(tcb->context.sp);
@@ -745,7 +760,7 @@ BEGIN_RS_FUNC(running_thread)
         } else {
             set_cur_thread(thread);
         }
-        
+
         thread->in_vm = thread->is_alive = true;
         thread->pal_handle = PAL_CB(first_thread);
     }

LibOS/shim/src/ipc/shim_ipc_helper.c

@@ -200,6 +200,8 @@ static void __put_ipc_port (struct shim_ipc_port * pobj)
             pobj->pal_handle = NULL;
         }
 
+        destroy_lock(pobj->msgs_lock);
+
         free_mem_obj_to_mgr(port_mgr, pobj);
     }
 }

LibOS/shim/src/shim_init.c

@@ -941,8 +941,11 @@ static int open_pal_handle (const char * uri, void * obj)
             return -PAL_ERRNO;
     }
 
-    if (obj)
+    if (obj) {
         *((PAL_HANDLE *) obj) = hdl;
+    } else {
+        DkObjectClose(hdl);
+    }
 
     return 0;
 }
@@ -1122,6 +1125,7 @@ int shim_clean (void)
         }
 
         master_unlock();
+        DkObjectClose(hdl);
     }
 #endif
 
@@ -1178,6 +1182,7 @@ int message_confirm (const char * message, const char * options)
         goto out;
 
 out:
+    DkObjectClose(hdl);
     master_unlock();
     return (ret < 0) ? ret : answer;
 }

LibOS/shim/src/sys/shim_socket.c

@@ -884,6 +884,8 @@ int __do_accept (struct shim_handle * hdl, int flags, struct sockaddr * addr,
     cli->acc_mode = MAY_READ|MAY_WRITE;
     cli->flags      = O_RDWR|flags;
     cli->pal_handle = accepted;
+    accepted = NULL;
+
     cli_sock->domain     = sock->domain;
     cli_sock->sock_type  = sock->sock_type;
     cli_sock->protocol   = sock->protocol;
@@ -910,17 +912,15 @@ int __do_accept (struct shim_handle * hdl, int flags, struct sockaddr * addr,
         char uri[SOCK_URI_SIZE];
         int uri_len;
 
-        if (!(uri_len = DkStreamGetName(accepted, uri, SOCK_URI_SIZE))) {
+        if (!(uri_len = DkStreamGetName(cli->pal_handle, uri, SOCK_URI_SIZE))) {
             ret = -PAL_ERRNO;
-out_hdl:
-            DkObjectClose(accepted);
             goto out_cli;
         }
 
         if ((ret = inet_parse_addr(cli_sock->domain, cli_sock->sock_type, uri,
                                    &cli_sock->addr.in.bind,
                                    &cli_sock->addr.in.conn)) < 0)
-            goto out_hdl;
+            goto out_cli;
 
         qstrsetstr(&cli->uri, uri, uri_len);
 
@@ -943,7 +943,8 @@ out_cli:
 out:
     if (ret < 0)
         sock->error = -ret;
-
+    if (accepted)
+        DkObjectClose(accepted);
     unlock(hdl->lock);
     return ret;
 }

LibOS/shim/test/apps/apache/Makefile

@@ -100,12 +100,14 @@ build-conf:
 </IfModule>\n" >> $(INSTALL_DIR)/conf/httpd.conf.new
 	cd $(INSTALL_DIR)/conf && ln -sf httpd.conf.new httpd.conf
 
-start-native-server:
+clean-server:
+	rm -f $(INSTALL_DIR)/logs/httpd-$(HOST)-$(PORT).pid
+
+start-native-server: clean-server
 	@echo "Listen on $(HOST):$(PORT)"
 	$(PREFIX) $(INSTALL_DIR)/bin/httpd -D FOREGROUND -C "ServerName $(HOST)" -C "Listen $(HOST):$(PORT)" -C "PidFile logs/httpd-$(HOST)-$(PORT).pid"
 
-start-graphene-server:
-	rm -rf httpd-$(HOST)-$(PORT).pid
+start-graphene-server: clean-server
 	@echo "Listen on $(HOST):$(PORT)"
 	$(PREFIX) ./httpd.manifest -D FOREGROUND -C "ServerName $(HOST)" -C "Listen $(HOST):$(PORT)" -C "PidFile logs/httpd-$(HOST)-$(PORT).pid"
 

LibOS/shim/test/apps/apache/benchmark-ab.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+PROTOCOL=${PROTOCOL:=http}
 declare -A THROUGHPUTS
 declare -A LATENCIES
 LOOP=5
@@ -8,6 +9,7 @@ DOWNLOAD_FILE=random/10K.1.html
 REQUESTS=10000
 CONCURRENCY_LIST="1 2 4 8 16 32 64 128 256"
 RESULT=result-$(date +%y%m%d-%H%M%S)
+URL="$PROTOCOL://$DOWNLOAD_HOST/$DOWNLOAD_FILE"
 
 touch $RESULT
 
@@ -17,9 +19,8 @@ do
 	for CONCURRENCY in $CONCURRENCY_LIST
 	do
 		rm -f OUTPUT
-		echo "ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE"
-		ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE > OUTPUT
-
+		echo "ab -n $REQUESTS -c $CONCURRENCY $URL"
+		ab -n $REQUESTS -c $CONCURRENCY $URL >> OUTPUT
 		sleep 5
 
 		THROUGHPUT=$(grep -m1 "Requests per second:" OUTPUT | awk '{ print $4 }')

LibOS/shim/test/apps/gcc/Makefile

@@ -95,26 +95,29 @@ regression:
 	@$(MAKE) >> /dev/null 2>&1
 
 	@echo "\n\nCompile hello.c:"
-	-./gcc.manifest test_files/helloworld.c -o hello
-	@chmod 755 hello
-	-./hello
+	-./gcc.manifest test_files/helloworld.c -o test_files/hello
+	@chmod 755 test_files/hello
+	-./test_files/hello
+	@rm -f test_files/hello
 
 	@echo "\n\nCompile bzip2.c:"
-	-./gcc.manifest test_files/bzip2.c -o bzip2
-	@chmod 755 bzip2
+	-./gcc.manifest test_files/bzip2.c -o test_files/bzip2
+	@chmod 755 test_files/bzip2
 	@[ ! -f bzip2.tmp ] || rm -f bzip2.tmp
-	@cp -f bzip2 bzip2.copy
-	-./bzip2 -z bzip2.copy && ./bzip2 -d bzip2.copy.bz2
-	diff -q bzip2 bzip2.copy
-	@rm -f bzip2.copy
+	@cp -f test_files/bzip2 test_files/bzip2.copy
+	-./test_files/bzip2 -z test_files/bzip2.copy
+	-./test_files/bzip2 -d test_files/bzip2.copy.bz2
+	diff -q test_files/bzip2 test_files/bzip2.copy
+	@rm -f test_files/bzip2 test_file/bzip2.copy
 
 	@echo "\n\nCompile gzip.c:"
-	-./gcc.manifest test_files/gzip.c -o gzip
-	@chmod 755 gzip
-	@cp -f gzip gzip.copy
-	-./gzip gzip.copy && ./gzip -d gzip.copy.gz
-	diff -q gzip gzip.copy
-	@rm -f gzip.copy
+	-./gcc.manifest test_files/gzip.c -o test_files/gzip
+	@chmod 755 test_files/gzip
+	@cp -f test_files/gzip test_files/gzip.copy
+	-./test_files/gzip test_files/gzip.copy
+	-./test_files/gzip -d test_files/gzip.copy.gz
+	diff -q test_files/gzip test_files/gzip.copy
+	@rm -f test_files/gzip test_files/gzip.copy
 
 
 src:

LibOS/shim/test/apps/gcc/as.manifest.template

@@ -29,7 +29,7 @@ sgx.trusted_files.libdl = file:$(LIBCDIR)/libdl.so.2
 sgx.trusted_files.libm = file:$(LIBCDIR)/libm.so.6
 sgx.trusted_files.libpthread = file:$(LIBCDIR)/libpthread.so.0
 sgx.trusted_files.libz = file:/lib/x86_64-linux-gnu/libz.so.1
-sgx.trusted_files.libopcodes = file:/usr/lib/x86_64-linux-gnu/libopcodes-2.26-system.so
+sgx.trusted_files.libopcodes = file:/usr/lib/x86_64-linux-gnu/libopcodes-2.26.1-system.so
 sgx.trusted_files.libbfd = file:/usr/lib/x86_64-linux-gnu/libbfd-2.26.1-system.so
 
 sgx.allowed_files.tmp = file:/tmp

LibOS/shim/test/apps/gcc/cc1.manifest.template

@@ -31,8 +31,7 @@ sgx.trusted_files.libdl = file:$(LIBCDIR)/libdl.so.2
 sgx.trusted_files.libm = file:$(LIBCDIR)/libm.so.6
 sgx.trusted_files.libz = file:/lib/x86_64-linux-gnu/libz.so.1
 sgx.trusted_files.libpthread = file:$(LIBCDIR)/libpthread.so.0
-sgx.trusted_files.cloog = file:/usr/lib/x86_64-linux-gnu/libcloog-isl.so.4
-sgx.trusted_files.isl = file:/usr/lib/x86_64-linux-gnu/libisl.so
+sgx.trusted_files.isl = file:/usr/lib/x86_64-linux-gnu/libisl.so.15
 sgx.trusted_files.mpc = file:/usr/lib/x86_64-linux-gnu/libmpc.so.3
 sgx.trusted_files.mpfr = file:/usr/lib/x86_64-linux-gnu/libmpfr.so.4
 sgx.trusted_files.gmp = file:/usr/lib/x86_64-linux-gnu/libgmp.so.10

LibOS/shim/test/apps/gcc/ld.manifest.template

@@ -32,6 +32,7 @@ sgx.trusted_files.libm = file:$(LIBCDIR)/libm.so.6
 sgx.trusted_files.libpthread = file:$(LIBCDIR)/libpthread.so.0
 sgx.trusted_files.libz = file:/lib/x86_64-linux-gnu/libz.so.1
 sgx.trusted_files.libbfd = file:/usr/lib/x86_64-linux-gnu/libbfd-2.26.1-system.so
+sgx.trusted_files.liblto = file:$(GCCDIR)/liblto_plugin.so
 sgx.trusted_files.libgcc = file:$(GCCDIR)/libgcc_s.so
 sgx.trusted_files.libgcca = file:$(GCCDIR)/libgcc.a
 sgx.trusted_files.crtend = file:$(GCCDIR)/crtend.o

LibOS/shim/test/apps/lighttpd/Makefile

@@ -5,7 +5,7 @@ PORT = 8000
 CORES = 4
 THREADS = 25
 
-conf_files = lighttpd-server.conf lighttpd.conf lighttpd-multithreaded.conf
+conf_files = lighttpd-server.conf lighttpd.conf lighttpd-multithreaded.conf lighttpd-ssl.conf
 
 target = build-lighttpd $(conf_files) test-data
 exec_target = $(manifests)
@@ -20,7 +20,7 @@ build-lighttpd: build/sbin/lighttpd
 
 build/sbin/lighttpd: $(SRCDIR)
 	cd $(SRCDIR) && ./configure --prefix=$(PWD)/build \
-		--without-openssl --without-pcre --without-zlib --without-bzip2
+		--with-openssl --without-pcre --without-zlib --without-bzip2
 	$(MAKE) -C $(SRCDIR)
 	$(MAKE) -C $(SRCDIR) install
 
@@ -44,6 +44,16 @@ lighttpd-multithreaded.conf:
 	echo "server.max-worker          = $(THREADS)"         >> $@
 	echo "include \"lighttpd-generic.conf\""               >> $@
 
+lighttpd-ssl.conf: server.pem
+	rm -rf $@
+	echo "include \"lighttpd-server.conf\""                >> $@
+	echo "ssl.engine = \"enable\""                         >> $@
+	echo "ssl.pemfile = \"server.pem\""                    >> $@
+	echo "include \"lighttpd-generic.conf\""               >> $@
+
+server.pem:
+	openssl req -new -x509 -keyout server.pem -out server.pem -days 365 -nodes -subj "/cn=lighttpd/o=oscar/dc=cs/dc=stonybrook/dc=edu/"
+
 start-native-server:
 	$(PREFIX) build/sbin/lighttpd -D -m build/lib -f \
 		$(if $(CONF),$(CONF),lighttpd.conf)
@@ -52,6 +62,10 @@ start-multithreaded-native-server:
 	$(PREFIX) build/sbin/lighttpd -D -m build/lib -f \
                 $(if $(CONF),$(CONF),lighttpd-multithreaded.conf)
 
+start-ssl-native-server:
+	$(PREFIX) build/sbin/lighttpd -D -m build/lib -f \
+                $(if $(CONF),$(CONF),lighttpd-ssl.conf)
+
 start-graphene-server:
 	$(PREFIX) ./lighttpd.manifest -D -m /lighttpd -f \
 		$(if $(CONF),$(CONF),lighttpd.conf)
@@ -60,6 +74,10 @@ start-multithreaded-graphene-server:
 	$(PREFIX) ./lighttpd.manifest -D -m /lighttpd -f \
                 $(if $(CONF),$(CONF),lighttpd-multithreaded.conf)
 
+start-ssl-graphene-server:
+	$(PREFIX) ./lighttpd.manifest -D -m /lighttpd -f \
+                $(if $(CONF),$(CONF),lighttpd-ssl.conf)
+
 
 random-data = $(foreach n,1 2 3 4 5 6 7 8 9 10,2K.$n.html) \
 	      $(foreach n,1 2 3 4 5,10K.$n.html) \

LibOS/shim/test/apps/lighttpd/lighttpd.manifest.template

@@ -2,13 +2,18 @@
 
 loader.preload = file:$(SHIMPATH)
 loader.exec = file:build/sbin/lighttpd
-loader.env.LD_LIBRARY_PATH = /lib:/lighttpd
+loader.env.LD_LIBRARY_PATH = /lib:/lib/x86_64-linux-gnu:/lighttpd
 loader.debug_type = none
 
 fs.mount.lib.type = chroot
 fs.mount.lib.path = /lib
 fs.mount.lib.uri = file:$(LIBCDIR)
 
+fs.mount.lib64.type = chroot
+fs.mount.lib64.path = /lib/x86_64-linux-gnu
+fs.mount.lib64.uri = file:/lib/x86_64-linux-gnu
+
+
 fs.mount.build.type = chroot
 fs.mount.build.path = /lighttpd
 fs.mount.build.uri = file:build/lib
@@ -39,6 +44,8 @@ sgx.trusted_files.libc = file:$(LIBCDIR)/libc.so.6
 sgx.trusted_files.libdl = file:$(LIBCDIR)/libdl.so.2
 sgx.trusted_files.libm = file:$(LIBCDIR)/libm.so.6
 sgx.trusted_files.libpthread = file:$(LIBCDIR)/libpthread.so.0
+sgx.trusted_files.libssl = file:/lib/x86_64-linux-gnu/libssl.so.1.0.0
+sgx.trusted_files.libcrypto = file:/lib/x86_64-linux-gnu/libcrypto.so.1.0.0
 
 sgx.trusted_files.lib1 = file:build/lib/mod_indexfile.so
 sgx.trusted_files.lib2 = file:build/lib/mod_dirlisting.so
@@ -48,5 +55,8 @@ sgx.trusted_files.conf1 = file:lighttpd.conf
 sgx.trusted_files.conf2 = file:lighttpd-generic.conf
 sgx.trusted_files.conf3 = file:lighttpd-server.conf
 sgx.trusted_files.conf4 = file:lighttpd-multithreaded.conf
+sgx.trusted_files.conf5 = file:lighttpd-ssl.conf
+
+sgx.trusted_files.server_key = file:server.pem
 
 sgx.allowed_files.html = file:html

Pal/Makefile

@@ -18,7 +18,7 @@ DIRS = src test regression
 all clean:
 	for d in $(DIRS); \
 	do \
-		$(MAKE) -C $$d $@; \
+		$(MAKE) -C $$d $@ || exit $?; \
 	done
 
 .PHONY: linux-kernel

Pal/regression/00_Symbols.py

@@ -44,7 +44,6 @@ all_symbols = [
     'DkEventSet',
     'DkEventClear',
     'DkObjectsWaitAny',
-    'DkObjectReference',
     'DkObjectClose',
     'DkSystemTimeQuery',
     'DkRandomBitsRead',

Pal/regression/Symbols.c

@@ -56,7 +56,6 @@ int main (int argc, char ** argv, char ** envp)
     print_symbol(DkEventClear);
 
     print_symbol(DkObjectsWaitAny);
-    print_symbol(DkObjectReference);
     print_symbol(DkObjectClose);
 
     print_symbol(DkSystemTimeQuery);

Pal/src/db_events.c

@@ -41,6 +41,7 @@ PAL_HANDLE DkNotificationEventCreate (PAL_BOL initialState)
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 
@@ -56,21 +57,11 @@ PAL_HANDLE DkSynchronizationEventCreate (PAL_BOL initialState)
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 
-void DkEventDestroy (PAL_HANDLE handle)
-{
-    ENTER_PAL_CALL(DkEventDestroy);
-
-    if (!handle || !IS_HANDLE_TYPE(handle, event)) {
-        _DkRaiseFailure(PAL_ERROR_INVAL);
-        LEAVE_PAL_CALL();
-    }
-
-    _DkEventDestroy(handle);
-    LEAVE_PAL_CALL();
-}
+/* DkEventDestroy deprecated, replaced by DkObjectClose */
 
 void DkEventSet (PAL_HANDLE handle)
 {
@@ -122,19 +113,3 @@ void DkEventClear (PAL_HANDLE handle)
 
     LEAVE_PAL_CALL();
 }
-
-static int event_close (PAL_HANDLE handle)
-{
-    return _DkEventClear(handle);
-}
-
-static int event_wait (PAL_HANDLE handle, uint64_t timeout)
-{
-    return timeout == NO_TIMEOUT ?    _DkEventWait(handle) : 
-        _DkEventWaitTimeout(handle, timeout);
-}
-
-struct handle_ops event_ops = {
-        .close              = &event_close,
-        .wait               = &event_wait,
-    };

Pal/src/db_ipc.c

@@ -42,6 +42,7 @@ DkCreatePhysicalMemoryChannel (PAL_NUM * key)
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 

Pal/src/db_mutex.c

@@ -34,35 +34,18 @@ DkMutexCreate (PAL_NUM initialCount)
 {
     ENTER_PAL_CALL(DkMutexCreate);
 
-    PAL_HANDLE handle = (PAL_HANDLE) malloc(HANDLE_SIZE(mutex));
-    SET_HANDLE_TYPE(handle, mutex);
-    
-    int ret = _DkMutexCreate(handle, initialCount);
+    PAL_HANDLE handle = NULL;
+    int ret = _DkMutexCreate(&handle, initialCount);
 
     if (ret < 0) {
-        free(handle);
         _DkRaiseFailure(-ret);
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 
-void
-DkMutexDestroy (PAL_HANDLE handle)
-{
-    ENTER_PAL_CALL(DkMutexDestroy);
-
-    if (!handle) {
-        _DkRaiseFailure(PAL_ERROR_INVAL);
-        LEAVE_PAL_CALL();
-    }
-
-    _DkMutexDestroy(handle);
-    free(handle);
-    LEAVE_PAL_CALL();
-}
-
 void DkMutexRelease (PAL_HANDLE handle)
 {
     ENTER_PAL_CALL(DkMutexRelease);
@@ -76,12 +59,3 @@ void DkMutexRelease (PAL_HANDLE handle)
     _DkMutexRelease (handle);
     LEAVE_PAL_CALL();
 }
-
-static int mutex_wait (PAL_HANDLE handle, uint64_t timeout)
-{
-    return _DkMutexAcquireTimeout(handle, timeout);
-}
-
-struct handle_ops mutex_ops = {
-        .wait               = &mutex_wait,
-    };

Pal/src/db_object.c

@@ -31,53 +31,41 @@
 #include "api.h"
 #include "atomic.h"
 
-int _DkObjectReference (PAL_HANDLE objectHandle)
-{
-    if (!objectHandle || UNKNOWN_HANDLE(objectHandle))
-        return -PAL_ERROR_INVAL;
-
-    atomic_inc(&HANDLE_HDR(objectHandle)->ref);
-    return 0;
-}
-
-void DkObjectReference (PAL_HANDLE objectHandle)
-{
-    ENTER_PAL_CALL(DkObjectReference);
-
-    int ret = _DkObjectReference(objectHandle);
-
-    if (ret < 0)
-        _DkRaiseFailure(-ret);
-
-    LEAVE_PAL_CALL();
-}
+/* Deprecated DkObjectReference. */
 
 int _DkObjectClose (PAL_HANDLE objectHandle)
 {
-    if (!objectHandle || UNKNOWN_HANDLE(objectHandle))
-        return -PAL_ERROR_INVAL;
-
-    if (!atomic_dec_and_test(&HANDLE_HDR(objectHandle)->ref))
-        return 0;
-
     const struct handle_ops * ops = HANDLE_OPS(objectHandle);
+    int ret = 0;
 
     /* if the operation 'close' is defined, call the function. */
     if (ops && ops->close)
-        return ops->close(objectHandle);
-
-    free(objectHandle);
-    return 0;
+        ret = ops->close(objectHandle);
+
+    /*
+     * Chia-Che 12/7/2017:
+     *   _DkObjectClose will free the object, unless the handle has
+     *   a 'close' operation, and the operation returns a non-zero value
+     *   (e.g., 1 for skipping free() or -ERRNO).
+     */
+    if (!ret)
+        free(objectHandle);
+
+    return ret;
 }
 
-/* PAL call DkObjectClose: Close the given object handle. This function is
-   different from DkStreamDelete. It works on all kinds of handles, and it
-   simply close the reference to the object, the stream is not actually
-   deleted. */
+/* PAL call DkObjectClose: Close the given object handle. */
 void DkObjectClose (PAL_HANDLE objectHandle)
 {
     ENTER_PAL_CALL(DkObjectClose);
 
+    if (!objectHandle || UNKNOWN_HANDLE(objectHandle)) {
+        _DkRaiseFailure(PAL_ERROR_INVAL);
+        LEAVE_PAL_CALL();
+    }
+
+    UNTRACE_HEAP(objectHandle);
+
     int ret = _DkObjectClose(objectHandle);
 
     if (ret < 0)

Pal/src/db_process.c

@@ -46,7 +46,7 @@ DkProcessCreate (PAL_STR uri, PAL_FLG flags, PAL_STR * args)
      * for now, and make it consistent across hosts. */
     if (!uri)
         uri = pal_control.executable;
-    
+
     log_stream(uri);
 
     PAL_HANDLE handle = NULL;
@@ -57,6 +57,7 @@ DkProcessCreate (PAL_STR uri, PAL_FLG flags, PAL_STR * args)
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 

Pal/src/db_streams.c

@@ -175,6 +175,7 @@ DkStreamOpen (PAL_STR uri, PAL_FLG access, PAL_FLG share, PAL_FLG create,
     assert(handle);
     assert(PAL_GET_TYPE(handle));
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 
@@ -206,6 +207,7 @@ DkStreamWaitForClient (PAL_HANDLE handle)
         client = NULL;
     }
 
+    TRACE_HEAP(client);
     LEAVE_PAL_CALL_RETURN(client);
 }
 

Pal/src/db_threading.c

@@ -46,6 +46,7 @@ DkThreadCreate (PAL_PTR addr, PAL_PTR param, PAL_FLG flags)
         handle = NULL;
     }
 
+    TRACE_HEAP(handle);
     LEAVE_PAL_CALL_RETURN(handle);
 }
 

Pal/src/host/FreeBSD/db_process.c

@@ -159,7 +159,7 @@ failed:
 int _DkProcessCreate (PAL_HANDLE * handle,
                       const char * uri, int flags, const char ** args)
 {
-    PAL_HANDLE exec = NULL, exec_file = NULL;
+    PAL_HANDLE exec = NULL;
     PAL_HANDLE parent_handle = NULL, child_handle = NULL;
     int ret;
 
@@ -169,19 +169,7 @@ int _DkProcessCreate (PAL_HANDLE * handle,
         if ((ret = _DkStreamOpen(&exec, uri, PAL_ACCESS_RDONLY, 0, 0, 0)) < 0)
             return ret;
 
-        ret = _DkStreamFile(exec, &exec_file);
-        if (ret < 0)
-            goto out;
-
-        _DkObjectClose(exec);
-        exec = NULL;
-
-        if (check_elf_object(exec_file) < 0) {
-            ret = -PAL_ERROR_INVAL;
-            goto out;
-        }
-
-        handle_set_cloexec(exec_file, true);
+        handle_set_cloexec(exec, true);
     }
 
     /* step 2: create parant and child process handle */
@@ -194,7 +182,7 @@ int _DkProcessCreate (PAL_HANDLE * handle,
 
 
     param.parent = parent_handle;
-    param.exec = exec_file;
+    param.exec = exec;
     param.manifest = pal_state.manifest_handle;
 
     /* step 3: compose process parameter */
@@ -209,8 +197,8 @@ int _DkProcessCreate (PAL_HANDLE * handle,
         goto out;
     parent_datasz = ret;
 
-    if (exec_file) {
-        ret = handle_serialize(exec_file, &exec_data);
+    if (exec) {
+        ret = handle_serialize(exec, &exec_data);
         if (ret < 0) {
             free(parent_data);
             goto out;
@@ -308,8 +296,6 @@ out:
             _DkObjectClose(child_handle);
         if (exec)
             _DkObjectClose(exec);
-        if (exec_file)
-            _DkObjectClose(exec_file);
     }
     return ret;
 }

Pal/src/host/FreeBSD/db_streams.c

@@ -70,18 +70,6 @@ out:
     return (mode & acc);
 }
 
-int _DkStreamFile (PAL_HANDLE hdl, PAL_HANDLE * file)
-{
-    if (IS_HANDLE_TYPE(hdl, file)) {
-        _DkObjectReference(hdl);
-        *file = hdl;
-        return 0;
-    }
-
-    /* for other types of handles, try to save it to a local file */
-    return -PAL_ERROR_NOTIMPLEMENTED;
-}
-
 int handle_set_cloexec (PAL_HANDLE handle, bool enable)
 {
     for (int i = 0 ; i < MAX_FDS ; i++)

Pal/src/host/Linux-SGX/db_events.c

@@ -29,6 +29,7 @@
 #include "pal_internal.h"
 #include "pal_linux.h"
 #include "pal_error.h"
+#include "pal_debug.h"
 #include "api.h"
 
 #include <atomic.h>
@@ -37,42 +38,42 @@
 
 int _DkEventCreate (PAL_HANDLE * event, bool initialState, bool isnotification)
 {
-    PAL_HANDLE ev = malloc_untrusted(HANDLE_SIZE(event));
+    PAL_HANDLE ev = malloc(HANDLE_SIZE(event));
     SET_HANDLE_TYPE(ev, event);
     ev->event.isnotification = isnotification;
-    atomic_set(&ev->event.signaled, initialState ? 1 : 0);
+    ev->event.signaled = malloc_untrusted(sizeof(struct atomic_int));
+    if (!ev->event.signaled) {
+        free(ev);
+        return -PAL_ERROR_NOMEM;
+    }
+    atomic_set(ev->event.signaled, initialState ? 1 : 0);
     atomic_set(&ev->event.nwaiters, 0);
     *event = ev;
     return 0;
 }
 
-void _DkEventDestroy (PAL_HANDLE handle)
-{
-    free_untrusted(handle);
-}
-
 int _DkEventSet (PAL_HANDLE event, int wakeup)
 {
     int ret = 0;
 
     if (event->event.isnotification) {
         // Leave it signaled, wake all
-        if (atomic_cmpxchg(&event->event.signaled, 0, 1) == 0) {
+        if (atomic_cmpxchg(event->event.signaled, 0, 1) == 0) {
             int nwaiters = atomic_read(&event->event.nwaiters);
             if (nwaiters) {
                 if (wakeup != -1 && nwaiters > wakeup)
                     nwaiters = wakeup;
 
-                ret = ocall_futex((int *) &event->event.signaled.counter,
+                ret = ocall_futex((int *) &event->event.signaled->counter,
                                   FUTEX_WAKE, nwaiters, NULL);
 
                 if (ret < 0)
-                    atomic_set(&event->event.signaled, 0);
+                    atomic_set(event->event.signaled, 0);
             }
         }
     } else {
         // Only one thread wakes up, leave unsignaled
-        ret = ocall_futex((int *) &event->event.signaled.counter,
+        ret = ocall_futex((int *) &event->event.signaled->counter,
                           FUTEX_WAKE, 1, NULL);
         if (ret < 0)
              return ret;
@@ -85,13 +86,13 @@ int _DkEventWaitTimeout (PAL_HANDLE event, uint64_t timeout)
 {
     int ret = 0;
 
-    if (!event->event.isnotification || !atomic_read(&event->event.signaled)) {
+    if (!event->event.isnotification || !atomic_read(event->event.signaled)) {
         unsigned long waittime = timeout;
 
         atomic_inc(&event->event.nwaiters);
 
         do {
-            ret = ocall_futex((int *) &event->event.signaled.counter,
+            ret = ocall_futex((int *) &event->event.signaled->counter,
                               FUTEX_WAIT, 0, timeout ? &waittime : NULL);
             if (ret < 0) {
                 if (ret == -PAL_ERROR_TRYAGAIN)
@@ -100,7 +101,7 @@ int _DkEventWaitTimeout (PAL_HANDLE event, uint64_t timeout)
                     break;
             }
         } while (event->event.isnotification &&
-                 !atomic_read(&event->event.signaled));
+                 !atomic_read(event->event.signaled));
 
         atomic_dec(&event->event.nwaiters);
     }
@@ -112,11 +113,11 @@ int _DkEventWait (PAL_HANDLE event)
 {
     int ret = 0;
 
-    if (!event->event.isnotification || !atomic_read(&event->event.signaled)) {
+    if (!event->event.isnotification || !atomic_read(event->event.signaled)) {
         atomic_inc(&event->event.nwaiters);
 
         do {
-            ret = ocall_futex((int *) &event->event.signaled.counter,
+            ret = ocall_futex((int *) &event->event.signaled->counter,
                               FUTEX_WAIT, 0, NULL);
             if (ret < 0) {
                 if (ret == -PAL_ERROR_TRYAGAIN)
@@ -125,7 +126,7 @@ int _DkEventWait (PAL_HANDLE event)
                     break;
             }
         } while (event->event.isnotification &&
-                 !atomic_read(&event->event.signaled));
+                 !atomic_read(event->event.signaled));
 
         atomic_dec(&event->event.nwaiters);
     }
@@ -135,6 +136,24 @@ int _DkEventWait (PAL_HANDLE event)
 
 int _DkEventClear (PAL_HANDLE event)
 {
-    atomic_set(&event->event.signaled, 0);
+    atomic_set(event->event.signaled, 0);
+    return 0;
+}
+
+static int event_close (PAL_HANDLE handle)
+{
+    _DkEventSet(handle, -1);
+    free_untrusted(handle->event.signaled);
     return 0;
 }
+
+static int event_wait (PAL_HANDLE handle, uint64_t timeout)
+{
+    return timeout == NO_TIMEOUT ? _DkEventWait(handle) :
+           _DkEventWaitTimeout(handle, timeout);
+}
+
+struct handle_ops event_ops = {
+        .close              = &event_close,
+        .wait               = &event_wait,
+    };

Pal/src/host/Linux-SGX/db_memory.c

@@ -104,7 +104,14 @@ int _DkVirtualMemoryAlloc (void ** paddr, uint64_t size, int alloc_type, int pro
 
 int _DkVirtualMemoryFree (void * addr, uint64_t size)
 {
-    free_pages(addr, size);
+
+    if (sgx_is_within_enclave(addr, size)) {
+        free_pages(addr, size);
+    } else {
+        /* Possible to have untrusted mapping. Simply unmap
+           the memory outside the enclave */
+        ocall_unmap_untrusted(addr, size);
+    }
     return 0;
 }
 

Pal/src/host/Linux-SGX/db_mutex.c

@@ -48,30 +48,26 @@
 
 
 int
-_DkMutexCreate (PAL_HANDLE handle, int initialCount)
+_DkMutexCreate (PAL_HANDLE * handle, int initialCount)
 {
-    /*
-     * Allocation and free of the handle are done outside of host-specific code.
-     * This code initializes the mutex state that is host-specific,
-     * including how initialCount is encoded.
-     */
-    SET_HANDLE_TYPE(handle, mutex);
-    atomic_set(&handle->mutex.mut.nwaiters, 0);
-    handle->mutex.mut.locked = initialCount;
+    PAL_HANDLE mut = malloc(HANDLE_SIZE(mutex));
+    SET_HANDLE_TYPE(mut, mutex);
+    atomic_set(&mut->mutex.mut.nwaiters, 0);
+    mut->mutex.mut.locked = malloc_untrusted(sizeof(int64_t));
+    if (!mut->mutex.mut.locked) {
+        free(mut);
+        return -PAL_ERROR_NOMEM;
+    }
+    *(mut->mutex.mut.locked) = initialCount;
+    *handle = mut;
     return 0;
 }
 
-void _DkMutexDestroy (PAL_HANDLE handle)
-{
-    free(handle);
-}
-
-
 int _DkMutexLockTimeout (struct mutex_handle * m, uint64_t timeout)
 {
     int ret = 0;
 
-    if (MUTEX_UNLOCKED == cmpxchg(&m->locked, MUTEX_UNLOCKED, MUTEX_LOCKED))
+    if (MUTEX_UNLOCKED == cmpxchg(m->locked, MUTEX_UNLOCKED, MUTEX_LOCKED))
         goto success;
 
     if (timeout == 0) {
@@ -82,12 +78,13 @@ int _DkMutexLockTimeout (struct mutex_handle * m, uint64_t timeout)
     // Bump up the waiters count; we are probably going to block
     atomic_inc(&m->nwaiters);
 
-    while (MUTEX_LOCKED == cmpxchg(&m->locked, MUTEX_UNLOCKED, MUTEX_LOCKED)) {
-
-        // This is broken. The mutex is in enclave memory, the URTS can't
-        // do FUTEX_WAIT on it. This call will always fail and the next level
-        // up needs to retry.
-        ret = ocall_futex((int *) m, FUTEX_WAIT, MUTEX_LOCKED, timeout == -1 ? NULL : &timeout);
+    while (MUTEX_LOCKED == cmpxchg(m->locked, MUTEX_UNLOCKED, MUTEX_LOCKED)) {
+        /*
+         * Chia-Che 12/7/2017: m->locked points to untrusted memory, so
+         * can be used for futex. Potentially this design may allow
+         * attackers to change the mutex value and cause DoS.
+         */
+        ret = ocall_futex((int *) m->locked, FUTEX_WAIT, MUTEX_LOCKED, timeout == -1 ? NULL : &timeout);
 
         if (ret < 0) {
             if (-ret == EWOULDBLOCK) {
@@ -111,7 +108,6 @@ out:
 
 int _DkMutexLock (struct mutex_handle * m)
 {
-    int ret = 0, i;
     return _DkMutexLockTimeout(m, -1);
 }
 
@@ -127,7 +123,7 @@ int _DkMutexUnlock (struct mutex_handle * m)
     int need_wake;
 
     /* Unlock */
-    m->locked = 0;
+    *(m->locked) = 0;
     /* We need to make sure the write to locked is visible to lock-ers
      * before we read the waiter count. */
     mb();
@@ -136,7 +132,7 @@ int _DkMutexUnlock (struct mutex_handle * m)
 
     /* If we need to wake someone up... */
     if (need_wake)
-        ocall_futex((int *) m, FUTEX_WAKE, 1, NULL);
+        ocall_futex((int *) m->locked, FUTEX_WAKE, 1, NULL);
 
     return ret;
 }
@@ -150,3 +146,18 @@ void _DkMutexRelease (PAL_HANDLE handle)
     return;
 }
 
+static int mutex_wait (PAL_HANDLE handle, uint64_t timeout)
+{
+    return _DkMutexAcquireTimeout(handle, timeout);
+}
+
+static int mutex_close (PAL_HANDLE handle)
+{
+    free_untrusted(handle->mutex.mut.locked);
+    return 0;
+}
+
+struct handle_ops mutex_ops = {
+        .wait               = &mutex_wait,
+        .close              = &mutex_close,
+    };

Pal/src/host/Linux-SGX/db_process.c

@@ -321,12 +321,6 @@ void print_alloced_pages (void);
 
 void _DkProcessExit (int exitcode)
 {
-    if (__pal_control.parent_process)
-        _DkObjectClose(__pal_control.parent_process);
-
-    if (__pal_control.manifest_handle)
-        _DkObjectClose(__pal_control.manifest_handle);
-
 #if PRINT_ENCLAVE_STAT
     print_alloced_pages();
 #endif

Pal/src/host/Linux-SGX/db_streams.c

@@ -74,18 +74,6 @@ out:
     return (mode & acc);
 }
 
-int _DkStreamFile (PAL_HANDLE hdl, PAL_HANDLE * file)
-{
-    if (IS_HANDLE_TYPE(hdl, file)) {
-        _DkObjectReference(hdl);
-        *file = hdl;
-        return 0;
-    }
-
-    /* for other types of handles, try to save it to a local file */
-    return -PAL_ERROR_NOTIMPLEMENTED;
-}
-
 int handle_set_cloexec (PAL_HANDLE handle, bool enable)
 {
     return -PAL_ERROR_NOTIMPLEMENTED;

Pal/src/host/Linux-SGX/pal_host.h

@@ -58,7 +58,7 @@ void free_untrusted (void * mem);
  * If DEBUG_MUTEX is defined, mutex_handle will record the owner of
  * mutex locking. */
 struct mutex_handle {
-    volatile int64_t locked;
+    volatile int64_t * locked;
     struct atomic_int nwaiters;
 #ifdef DEBUG_MUTEX
     int owner;
@@ -170,7 +170,7 @@ typedef struct pal_handle
             } mutex;
 
             struct {
-                struct atomic_int signaled;
+                struct atomic_int * signaled;
                 struct atomic_int nwaiters;
                 PAL_BOL isnotification;
             } event;

Pal/src/host/Linux-SGX/sgx_graphene.c

@@ -26,6 +26,7 @@
 
 #include "sgx_internal.h"
 
+#if 0 /* this code is useless for now */
 int _DkEventSet (PAL_HANDLE event, int wakeup)
 {
     int ret = 0;
@@ -80,6 +81,7 @@ int _DkEventWait (PAL_HANDLE event)
 
     return ret;
 }
+#endif
 
 #define PRINTBUF_SIZE        256
 

Pal/src/host/Linux/db_events.c

@@ -29,6 +29,7 @@
 #include "pal_internal.h"
 #include "pal_linux.h"
 #include "pal_error.h"
+#include "pal_debug.h"
 #include "api.h"
 
 #include <atomic.h>
@@ -47,11 +48,6 @@ int _DkEventCreate (PAL_HANDLE * event, bool initialState, bool isnotification)
     return 0;
 }
 
-void _DkEventDestroy (PAL_HANDLE handle)
-{
-    free(handle);
-}
-
 int _DkEventSet (PAL_HANDLE event, int wakeup)
 {
     int ret = 0;
@@ -146,3 +142,19 @@ int _DkEventClear (PAL_HANDLE event)
     atomic_set(&event->event.signaled, 0);
     return 0;
 }
+
+static int event_close (PAL_HANDLE handle)
+{
+    _DkEventSet(handle, -1);
+    return 0;
+}
+
+static int event_wait (PAL_HANDLE handle, uint64_t timeout)
+{
+    return timeout == NO_TIMEOUT ? _DkEventWait(handle) :
+           _DkEventWaitTimeout(handle, timeout);
+}
+struct handle_ops event_ops = {
+        .close              = &event_close,
+        .wait               = &event_wait,
+    };

Pal/src/host/Linux/db_exception.c

@@ -231,21 +231,25 @@ static void return_frame (struct pal_frame * frame, int err)
                   "retq\r\n");
 }
 
+#if BLOCK_SIGFAULT == 1
+static char exception_msg[24] = "--- SIGSEGV --- [     ]\n";
+static volatile bool cont_exec = false;
+#endif
+
 static void _DkGenericSighandler (int signum, siginfo_t * info,
                                   struct ucontext * uc)
 {
-#if 0
+#if BLOCK_SIGFUALT == 1
     /* reseurrect this code if signal handler if giving segmentation fault */
     if (signum == SIGSEGV) {
         int pid = INLINE_SYSCALL(getpid, 0);
-        char msg[24] = "--- SIGSEGV --- [     ]\n";
-        msg[17] = '0' + pid / 10000;
-        msg[18] = '0' + (pid / 1000) % 10;
-        msg[19] = '0' + (pid / 100) % 10;
-        msg[20] = '0' + (pid / 10) % 10;
-        msg[21] = '0' + pid % 10;
-        INLINE_SYSCALL(write, 3, 1, msg, 24);
-        while(1);
+        exception_msg[17] = '0' + pid / 10000;
+        exception_msg[18] = '0' + (pid / 1000) % 10;
+        exception_msg[19] = '0' + (pid / 100) % 10;
+        exception_msg[20] = '0' + (pid / 10) % 10;
+        exception_msg[21] = '0' + pid % 10;
+        INLINE_SYSCALL(write, 3, 1, exception_msg, 24);
+        while(!cont_exec);
     }
 #endif
 

Pal/src/host/Linux/db_mutex.c

@@ -76,24 +76,16 @@
  */
 
 int
-_DkMutexCreate (PAL_HANDLE handle, int initialCount)
+_DkMutexCreate (PAL_HANDLE * handle, int initialCount)
 {
-    /*
-     * Allocation and free of the handle are done outside of host-specific code.
-     * This code initializes the mutex state that is host-specific,
-     * including how initialCount is encoded.
-     */
-    atomic_set(&handle->mutex.mut.nwaiters, 0);
-    handle->mutex.mut.locked = initialCount;
+    PAL_HANDLE mut = malloc(HANDLE_SIZE(mutex));
+    SET_HANDLE_TYPE(mut, mutex);
+    atomic_set(&mut->mutex.mut.nwaiters, 0);
+    mut->mutex.mut.locked = initialCount;
+    *handle = mut;
     return 0;
 }
 
-void _DkMutexDestroy (PAL_HANDLE handle)
-{
-    // Do nothing; handled in higher-level code
-}
-
-
 int _DkMutexLockTimeout (struct mutex_handle * m, uint64_t timeout)
 {
     int i, ret = 0;
@@ -204,3 +196,12 @@ void _DkMutexRelease (PAL_HANDLE handle)
     _DkMutexUnlock(&handle->mutex.mut);
     return;
 }
+
+static int mutex_wait (PAL_HANDLE handle, uint64_t timeout)
+{
+    return _DkMutexAcquireTimeout(handle, timeout);
+}
+
+struct handle_ops mutex_ops = {
+        .wait               = &mutex_wait,
+    };

Pal/src/host/Linux/db_object.c

@@ -260,3 +260,49 @@ int _DkObjectsWaitAny (int count, PAL_HANDLE * handleArray, uint64_t timeout,
     *polled = polled_hdl;
     return polled_hdl ? 0 : -PAL_ERROR_TRYAGAIN;
 }
+
+#if TRACE_HEAP_LEAK == 1
+
+PAL_HANDLE heap_alloc_head;
+PAL_LOCK   heap_alloc_trace_lock = LOCK_INIT;
+
+HEAP_ALLOC_RECORD * collect_heap_alloc_records (PAL_NUM max_records)
+{
+    HEAP_ALLOC_RECORD * records =
+            malloc(sizeof(HEAP_ALLOC_RECORD) * max_records);
+
+    if (!records)
+        return NULL;
+
+    memset(records, 0, sizeof(HEAP_ALLOC_RECORD) * max_records);
+
+    _DkInternalLock(&heap_alloc_trace_lock);
+
+    PAL_HANDLE ptr = heap_alloc_head;
+    int nrecords = 0, i;
+
+    for (; ptr ; ptr = ptr->hdr.heap_trace.next) {
+        assert(!ptr->hdr.heap_trace.next ||
+               ptr->hdr.heap_trace.next->hdr.heap_trace.pprev ==
+               &ptr->hdr.heap_trace.next);
+
+        for (i = 0 ; i < nrecords ; i++)
+            if (ptr->hdr.heap_trace.caller == records[i].caller) {
+                records[i].count++;
+                break;
+            }
+
+        if (i == nrecords) {
+            if (nrecords == max_records) break;
+            records[nrecords].caller = ptr->hdr.heap_trace.caller;
+            records[nrecords].count = 1;
+            nrecords++;
+        }
+    }
+
+    _DkInternalUnlock(&heap_alloc_trace_lock);
+
+    return records;
+}
+
+#endif /* TRACE_HEAP_LEAK == 0 */

Pal/src/host/Linux/db_process.c

@@ -167,7 +167,7 @@ failed:
 int _DkProcessCreate (PAL_HANDLE * handle,
                       const char * uri, int flags, const char ** args)
 {
-    PAL_HANDLE exec = NULL, exec_file = NULL;
+    PAL_HANDLE exec = NULL;
     PAL_HANDLE parent_handle = NULL, child_handle = NULL;
     int ret;
 #if PROFILING == 1
@@ -180,19 +180,10 @@ int _DkProcessCreate (PAL_HANDLE * handle,
         if ((ret = _DkStreamOpen(&exec, uri, PAL_ACCESS_RDONLY, 0, 0, 0)) < 0)
             return ret;
 
-        ret = _DkStreamFile(exec, &exec_file);
-        if (ret < 0)
-            goto out;
-
-        _DkObjectClose(exec);
-        exec = NULL;
-
-        if (check_elf_object(exec_file) < 0) {
+        if (check_elf_object(exec) < 0) {
             ret = -PAL_ERROR_INVAL;
             goto out;
         }
-
-        handle_set_cloexec(exec_file, true);
     }
 
     /* step 2: create parant and child process handle */
@@ -203,7 +194,7 @@ int _DkProcessCreate (PAL_HANDLE * handle,
         goto out;
 
     param.parent = parent_handle;
-    param.exec = exec_file;
+    param.exec = exec;
     param.manifest = pal_state.manifest_handle;
 
     /* step 3: compose process parameter */
@@ -218,8 +209,8 @@ int _DkProcessCreate (PAL_HANDLE * handle,
         goto out;
     parent_datasz = ret;
 
-    if (exec_file) {
-        ret = handle_serialize(exec_file, &exec_data);
+    if (exec) {
+        ret = handle_serialize(exec, &exec_data);
         if (ret < 0) {
             free(parent_data);
             goto out;
@@ -326,8 +317,6 @@ out:
         _DkObjectClose(parent_handle);
     if (exec)
         _DkObjectClose(exec);
-    if (exec_file)
-        _DkObjectClose(exec_file);
     if (ret < 0) {
         if (child_handle)
             _DkObjectClose(child_handle);

Pal/src/host/Linux/db_streams.c

@@ -74,18 +74,6 @@ out:
     return (mode & acc);
 }
 
-int _DkStreamFile (PAL_HANDLE hdl, PAL_HANDLE * file)
-{
-    if (IS_HANDLE_TYPE(hdl, file)) {
-        _DkObjectReference(hdl);
-        *file = hdl;
-        return 0;
-    }
-
-    /* for other types of handles, try to save it to a local file */
-    return -PAL_ERROR_NOTIMPLEMENTED;
-}
-
 int handle_set_cloexec (PAL_HANDLE handle, bool enable)
 {
     for (int i = 0 ; i < MAX_FDS ; i++)

Pal/src/host/Linux/pal.map

@@ -29,7 +29,7 @@ PAL {
 
         DkSystemTimeQuery; DkRandomBitsRead;
         DkInstructionCacheFlush;
-        DkObjectReference; DkObjectClose;
+        DkObjectClose;
         # objects checkpoint?
         # objects reload?
 

Pal/src/host/Linux/pal_host.h

@@ -56,6 +56,23 @@ typedef struct mutex_handle {
 #define _DkInternalLock _DkMutexLock
 #define _DkInternalUnlock _DkMutexUnlock
 
+/* Locking and unlocking of Mutexes */
+int _DkMutexLock (struct mutex_handle * mut);
+int _DkMutexLockTimeout (struct mutex_handle * mut, uint64_t timeout);
+int _DkMutexUnlock (struct mutex_handle * mut);
+
+typedef struct {
+    PAL_HDR hdr;
+#if TRACE_HEAP_LEAK == 1
+    struct heap_trace_info {
+        /* maintaining a list of handles */
+        struct pal_handle ** pprev, * next;
+        /* trace the PC where the handle is created */
+        PAL_PTR caller;
+    } heap_trace;
+#endif
+} PAL_RESERVED_HDR;
+
 typedef struct pal_handle
 {
     /* TSAI: Here we define the internal types of PAL_HANDLE
@@ -257,4 +274,63 @@ void __clear_frame (struct pal_frame * frame)
         return (retval);                    \
     } while (0)
 
+#if TRACE_HEAP_LEAK == 1
+
+/* The following code adds a piece of information
+   in each handle to trace heap leakage. */
+
+extern PAL_HANDLE heap_alloc_head;
+extern PAL_LOCK   heap_alloc_trace_lock;
+
+/* call the following function in GDB */
+typedef struct {
+    PAL_PTR caller;
+    PAL_NUM count;
+} HEAP_ALLOC_RECORD;
+
+extern HEAP_ALLOC_RECORD * collect_heap_alloc_records (PAL_NUM max_records);
+
+static inline
+void __trace_heap (PAL_HANDLE handle, struct pal_frame * frame)
+{
+    _DkInternalLock(&heap_alloc_trace_lock);
+
+    handle->hdr.heap_trace.caller = ((PAL_PTR *)frame->arch.rbp)[1];
+
+    /* Add the handle to the list */
+    if (heap_alloc_head)
+        heap_alloc_head->hdr.heap_trace.pprev
+                                    = &handle->hdr.heap_trace.next;
+    handle->hdr.heap_trace.next     = heap_alloc_head;
+    handle->hdr.heap_trace.pprev    = &heap_alloc_head;
+    heap_alloc_head                 = handle;
+
+    _DkInternalUnlock(&heap_alloc_trace_lock);
+}
+
+#define TRACE_HEAP(handle) \
+    do { if (handle) __trace_heap(handle, &frame); } while (0)
+
+static inline
+void __untrace_heap (PAL_HANDLE handle)
+{
+    _DkInternalLock(&heap_alloc_trace_lock);
+
+    /* remove the handle from the list */
+    *handle->hdr.heap_trace.pprev = handle->hdr.heap_trace.next;
+    if (handle->hdr.heap_trace.next)
+        handle->hdr.heap_trace.next->hdr.heap_trace.pprev
+            = handle->hdr.heap_trace.pprev;
+
+    handle->hdr.heap_trace.pprev = NULL;
+    handle->hdr.heap_trace.next  = NULL;
+
+    _DkInternalUnlock(&heap_alloc_trace_lock);
+}
+
+#define UNTRACE_HEAP(handle) \
+    do { if (handle) __untrace_heap(handle); } while (0)
+
+#endif /* TRACE_HEAP_LEAK == 1 */
+
 #endif /* PAL_HOST_H */

Pal/src/host/Linux/pal_linux_defs.h

@@ -16,4 +16,6 @@
 
 #define USE_ARCH_RDRAND         0
 
+#define BLOCK_SIGFAULT          0
+
 #endif /* PAL_LINUX_DEFS_H */

Pal/src/pal.h

@@ -43,7 +43,6 @@ typedef struct atomic_int PAL_REF;
 
 typedef struct {
     PAL_IDX type;
-    PAL_REF ref;
     PAL_FLG flags;
 } PAL_HDR;
 
@@ -53,9 +52,16 @@ typedef struct {
 #  define HANDLE_HDR(handle) (&((handle)->hdr))
 # endif
 
+# ifndef TRACE_HEAP
+#  define TRACE_HEAP(handle) do {} while (0)
+# endif
+
+# ifndef UNTRACE_HEAP
+#  define UNTRACE_HEAP(handle) do {} while (0)
+# endif
+
 static inline void init_handle_hdr(PAL_HDR *hdr, int pal_type) {
     hdr->type = pal_type;
-    hdr->ref.counter = 1;
     hdr->flags = 0;
 }
 
@@ -70,6 +76,7 @@ typedef union pal_handle
 {
     struct {
         PAL_IDX type;
+        /* the PAL-level reference counting is deprecated */
     } hdr;
 } * PAL_HANDLE;
 
@@ -459,7 +466,7 @@ DkEventClear (PAL_HANDLE eventHandle);
 PAL_HANDLE
 DkObjectsWaitAny (PAL_NUM count, PAL_HANDLE * handleArray, PAL_NUM timeout);
 
-void DkObjectReference (PAL_HANDLE objectHandle);
+/* Deprecate DkObjectReference */
 
 void DkObjectClose (PAL_HANDLE objectHandle);
 

Pal/src/pal_defs.h

@@ -17,4 +17,7 @@
 /* maximum length of URIs */
 #define URI_MAX                  256
 
+/* turn on the following option to trace heap memory leak */
+#define TRACE_HEAP_LEAK          0
+
 #endif /* PAL_DEFS_H */

Pal/src/pal_internal.h

@@ -283,7 +283,6 @@ int64_t _DkStreamSetLength (PAL_HANDLE handle, uint64_t length);
 int _DkStreamFlush (PAL_HANDLE handle);
 int _DkStreamGetName (PAL_HANDLE handle, char * buf, int size);
 const char * _DkStreamRealpath (PAL_HANDLE hdl);
-int _DkStreamFile (PAL_HANDLE hdl, PAL_HANDLE * file);
 int _DkSendHandle(PAL_HANDLE hdl, PAL_HANDLE cargo);
 int _DkReceiveHandle(PAL_HANDLE hdl, PAL_HANDLE * cargo);
 PAL_HANDLE _DkBroadcastStreamOpen (void);
@@ -301,8 +300,7 @@ void _DkProcessExit (int exitCode);
 int _DkProcessSandboxCreate (const char * manifest, int flags);
 
 /* DkMutex calls */
-int _DkMutexCreate (PAL_HANDLE handle, int initialCount);
-void _DkMutexDestroy (PAL_HANDLE semaphoreHandle);
+int _DkMutexCreate (PAL_HANDLE * handle, int initialCount);
 int _DkMutexAcquire (PAL_HANDLE sem);
 int _DkMutexAcquireTimeout (PAL_HANDLE sem, int timeout);
 void _DkMutexRelease (PAL_HANDLE sem);
@@ -311,7 +309,6 @@ int _DkMutexGetCurrentCount (PAL_HANDLE sem);
 /* DkEvent calls */
 int _DkEventCreate (PAL_HANDLE * event, bool initialState,
                     bool isnotification);
-void _DkEventDestroy (PAL_HANDLE handle);
 int _DkEventSet (PAL_HANDLE event, int wakeup);
 int _DkEventWaitTimeout (PAL_HANDLE event, uint64_t timeout);
 int _DkEventWait (PAL_HANDLE event);

Runtime/pal_loader

@@ -19,8 +19,7 @@ RUNTIME_DIR=$(/usr/bin/dirname $(readlink -f ${BASH_SOURCE[0]}))
 PAL_HOST=$(/usr/bin/make --quiet -f $RUNTIME_DIR/../Pal/src/Makefile.Host print_host 2>&1)
 
 MANIFEST=
-PERF_CMD=
-GDB_CMD=
+PREFIX=
 PAL_CMD=$RUNTIME_DIR/pal-$PAL_HOST
 
 if [ "$GDB" == "1" ]; then
@@ -31,11 +30,15 @@ if [ "$GDB" == "1" ]; then
 fi
 
 if [ "$GDB" != "" ] && [ "$GDB" != "0" ]; then
-	GDB_CMD="$GDB --args"
+	PREFIX="$GDB --args"
 fi
 
 if [ "$PERF" == "1" ]; then
-	GDB_CMD="perf stat"
+	PREFIX="perf stat"
+fi
+
+if [ "$MEMUSG" == "1" ]; then
+	PREFIX="$RUNTIME_DIR/../Scripts/memusg"
 fi
 
 if [ "$SEC" == "1" ]; then
@@ -59,4 +62,4 @@ if [ ! -f "$PAL_CMD" ]; then
 	exit 1
 fi
 
-exec $GDB_CMD $PAL_CMD $MANIFEST "$@"
+exec $PREFIX $PAL_CMD $MANIFEST "$@"

Scripts/memusg

@@ -30,6 +30,8 @@ case `uname` in
     *) echo "`uname`: unsupported operating system" >&2; exit 2 ;;
 esac
 
+rm -f "histogram.mem.usg"
+
 # monitor the memory usage in the background.
 (
 peak=0
@@ -37,6 +39,7 @@ while sizes=`sizes $pgid`
 do
     set -- $sizes
     sample=$((${@/#/+}))
+    echo "$sample" >> "histogram.mem.usg"
     let peak="sample > peak ? sample : peak"
     sleep 0.1
 done