merge internal patches

LibOS/shim/include/shim_fs.h

@@ -276,9 +276,6 @@ int init_mount (void);
 
 /* path utilities */
 const char * get_file_name (const char * path, size_t len);
-int get_abs_path (const char * cwd, const char * path, char * buf,
-                  int size);
-int get_norm_path (const char * path, char * buf, int size);
 
 /* file system operations */
 int mount_fs (const char * mount_type, const char * mount_uri,

LibOS/shim/src/fs/shim_fs.c

@@ -533,115 +533,3 @@ const char * get_file_name (const char * path, size_t len)
         c--;
     return *c == '/' ? c + 1 : c;
 }
-
-int get_abs_path (const char * cwd, const char * path, char * buf, int size)
-{
-    int cnt = 0;
-    char c, c1;
-    const char * p = path;
-
-    if (*p != '/') {
-        cnt = strlen(cwd);
-        while (cnt >= 0 && cwd[cnt - 1] == '/')
-            cnt--;
-        memcpy(buf, cwd, cnt);
-    }
-
-    for (c = '/' ; c ; c = c1, p++) {
-        c1 = *p;
-        if (c == '/') {
-            if (c1 == 0)
-                break;
-            if (c1 == '/')
-                continue;
-            if (c1 == '.') {
-                c1 = *(++p);
-                if (c1 == 0)
-                    break;
-                if (c1 == '/')
-                    continue;
-                if (c1 == '.') {
-                    c1 = *(++p);
-                    if (c1 == 0) {
-                        while (cnt > 0 && buf[--cnt] != '/');
-                        break;
-                    }
-                    if (c1 == '/') {
-                        while (cnt > 0 && buf[--cnt] != '/');
-                        continue;
-                    }
-                    return -EINVAL;
-                }
-                if (cnt >= size-1)
-                    return -ENAMETOOLONG;
-                buf[cnt++] = c;
-                c = '.';
-            }
-        }
-        if (cnt >= size-1)
-            return -ENAMETOOLONG;
-        buf[cnt++] = c;
-    }
-
-    if (cnt) {
-        buf[cnt] = 0;
-    } else {
-        buf[0] = '/';
-        buf[1] = 0;
-    }
-
-    return cnt;
-}
-
-int get_norm_path (const char * path, char * buf, int size)
-{
-    int cnt = 0;
-    char c, c1;
-    const char * p = path;
-
-    for (c = '/' ; c ; c = c1, p++) {
-        c1 = *p;
-        if (c == '/') {
-            if (c1 == 0)
-                break;
-            if (c1 == '/')
-                continue;
-            if (c1 == '.') {
-                c1 = *(++p);
-                if (c1 == 0)
-                    break;
-                if (c1 == '/')
-                    continue;
-                if (c1 == '.') {
-                    c1 = *(++p);
-                    if (c1 != 0 && c1 != '/')
-                        return -EINVAL;
-                    if (cnt) {
-                        while (cnt > 0 && buf[--cnt] != '/');
-                    } else {
-                        if (cnt >= size-2)
-                            return -ENAMETOOLONG;
-                        buf[cnt++] = '.';
-                        buf[cnt++] = '.';
-                    }
-                    c = c1;
-                    continue;
-                }
-                if (cnt || c != '/') {
-                    if (cnt >= size-1)
-                        return -ENAMETOOLONG;
-                    buf[cnt++] = c;
-                }
-                c = '.';
-            }
-        }
-        if (cnt || c != '/') {
-            if (cnt >= size-1)
-                return -ENAMETOOLONG;
-            buf[cnt++] = c;
-        }
-    }
-
-    buf[cnt] = 0;
-    return cnt;
-}

LibOS/shim/src/fs/shim_namei.c

@@ -468,7 +468,7 @@ static int path_lookup_dcache (struct shim_dentry * start, const char * path,
     char * name = fullpath + startpathlen;
     int namelen;
 
-    if ((namelen = get_norm_path(path, name, STR_SIZE - startpathlen)) < 0)
+    if ((namelen = get_norm_path(path, name, 0, STR_SIZE - startpathlen)) < 0)
         return namelen;
 
     struct shim_dentry * found =

Pal/Makefile

@@ -1,4 +1,4 @@
-import src/Makefile.Host
+include src/Makefile.Host
 
 ifeq ($(OS),Linux)
 	NPROCS ?= $(shell grep -c ^processor /proc/cpuinfo)

Pal/lib/api.h

@@ -61,6 +61,8 @@ extern const char * const * sys_errlist_internal;
 #define XSTRINGIFY(x) STRINGIFY(x)
 #define STRINGIFY(x) #x
 
+int get_norm_path (const char * path, char * buf, int offset, int size);
+
 #include <linux_list.h>
 
 struct config_store {

Pal/lib/graphene/path.c

@@ -0,0 +1,97 @@
+/* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
+/* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
+
+/* Copyright (C) 2014 OSCAR lab, Stony Brook University
+   This file is part of Graphene Library OS.
+
+   Graphene Library OS is free software: you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation, either version 3 of the
+   License, or (at your option) any later version.
+
+   Graphene Library OS is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/*
+ * path.c
+ *
+ * This file contains functions to read app config (manifest) file and create
+ * a tree to lookup / access config values.
+ */
+
+#include <api.h>
+#include <pal_error.h>
+
+int get_norm_path (const char * path, char * buf, int offset, int size)
+{
+    int head = offset;
+    char c, c1;
+    const char * p = path;
+
+    while (head) { /* find the real head, not interrupted by dot-dot */
+        if (head > 1 && buf[head - 1] == '.' && buf[head - 2] == '.')
+            break;
+        head--;
+    }
+
+    for (c = '/' ; c ; c = c1, p++) {
+        c1 = *p;
+        if (c == '/') {     /* find a slash, or the beginning of the path */
+            if (c1 == 0)    /* no more path */
+                break;
+            if (c1 == '/')  /* consequential slashes */
+                continue;
+            if (c1 == '.') {    /* find a dot, can be dot-dot or a file */
+                c1 = *(++p);
+                if (c1 == 0)    /* no more path */
+                    break;
+                if (c1 == '/')  /* a dot, skip it */
+                    continue;
+                if (c1 == '.') {    /* must be dot-dot */
+                    c1 = *(++p);
+                    if (c1 != 0 && c1 != '/')   /* must be the end or a slash */
+                        return -PAL_ERROR_INVAL;
+                    if (offset > head) {    /* remove the last token */
+                        while (offset > head && buf[--offset] != '/');
+                    } else {
+                        if (offset) {   /* add a slash */
+                            if (offset >= size - 1)
+                                return -PAL_ERROR_TOOLONG;
+                            buf[offset++] = '/';
+                        }               /* add a dot-dot */
+                        if (offset >= size - 2)
+                            return -PAL_ERROR_TOOLONG;
+                        buf[offset++] = '.';
+                        buf[offset++] = '.';
+                        head = offset;
+                    }
+                } else { /* it's a file */
+                    if (offset) {   /* add a slash */
+                        if (offset >= size - 1)
+                            return -PAL_ERROR_TOOLONG;
+                        buf[offset++] = '/';
+                    }
+                    if (offset >= size - 1)
+                        return -PAL_ERROR_TOOLONG;
+                    buf[offset++] = '.';
+                }
+                continue;
+            }
+        }
+        if (offset || c != '/') {
+            if (offset >= size - 1)
+                return -PAL_ERROR_TOOLONG;
+            buf[offset++] = c;
+        }
+    }
+
+    buf[offset] = 0;
+    return offset;
+}
+
+

Pal/regression/Memory.c

@@ -56,13 +56,17 @@ int main (int argc, char ** argv, char ** envp)
             pal_printf("Memory Deallocation OK\n");
     }
 
-    void * mem3 = (void *) DkVirtualMemoryAlloc(pal_control.user_address.start,
-                                                UNIT, 0,
-                                                PAL_PROT_READ|PAL_PROT_WRITE);
-    void * mem4 = (void *) DkVirtualMemoryAlloc(pal_control.user_address.end - UNIT,
-                                                UNIT, 0,
-                                                PAL_PROT_READ|PAL_PROT_WRITE);
+    void * mem3 = (void *) pal_control.user_address.start;
+    void * mem4 = (void *) pal_control.user_address.end - UNIT;
+
+    if (mem3 >= pal_control.executable_range.start &&
+        mem3 < pal_control.executable_range.end)
+        mem3 = (void *) (((PAL_NUM) pal_control.executable_range.end + UNIT - 1) & ~(UNIT - 1));
 
+    mem3 = (void *) DkVirtualMemoryAlloc(mem3, UNIT, 0,
+                                         PAL_PROT_READ|PAL_PROT_WRITE);
+    mem4 = (void *) DkVirtualMemoryAlloc(mem4, UNIT, 0,
+                                         PAL_PROT_READ|PAL_PROT_WRITE);
 
     if (mem3 && mem4)
         pal_printf("Memory Allocation with Address OK\n");

Pal/src/db_main.c

@@ -276,7 +276,7 @@ void pal_main (PAL_NUM pal_token, void * pal_addr,
                int argc, const char ** argv, const char ** envp,
                PAL_HANDLE parent_handle,
                PAL_HANDLE thread_handle,
-               PAL_HANDLE exec_handle,
+               PAL_HANDLE exec_handle, PAL_PTR_RANGE * exec_range,
                PAL_HANDLE manifest_handle)
 {
     int ret;
@@ -448,7 +448,11 @@ void pal_main (PAL_NUM pal_token, void * pal_addr,
         unsigned long before_load_exec = _DkSystemTimeQuery();
 #endif
 
-        ret = load_elf_object_by_handle(exec_handle, OBJECT_EXEC);
+        ret = exec_range ?
+              add_elf_object((void *) exec_range->start,
+                             exec_range->end - exec_range->start,
+                             exec_handle, OBJECT_EXEC) :
+              load_elf_object_by_handle(exec_handle, OBJECT_EXEC);
         if (ret < 0)
             init_fail(ret, PAL_STRERROR(ret));
 

Pal/src/db_rtld.c

@@ -519,23 +519,18 @@ int load_elf_object (const char * uri, enum object_type type)
     return ret;
 }
 
-int add_elf_object(void * addr, PAL_HANDLE handle, int type)
+int add_elf_object(void * addr, unsigned int size, PAL_HANDLE handle, int type)
 {
     struct link_map * map = new_elf_object(_DkStreamRealpath(handle), type);
     const ElfW(Ehdr) * header = (void *) addr;
-    int ret;
+    const ElfW(Phdr) * ph, * phdr = addr + header->e_phoff;
 
+    map->l_addr  = (header->e_type == ET_DYN) ? (ElfW(Addr)) addr : 0;
     map->l_entry = header->e_entry;
     map->l_phdr  = (void *) header->e_phoff;
     map->l_phnum = header->e_phnum;
-
-    const ElfW(Phdr) * ph, * phdr;
-    phdr = __alloca(header->e_phnum * sizeof(ElfW(Phdr)));
-
-    if ((ret = _DkStreamRead(handle, header->e_phoff,
-                             header->e_phnum * sizeof(ElfW(Phdr)),
-                             (void *) phdr, NULL, 0)) < 0)
-        return ret;
+    map->l_map_start = (ElfW(Addr)) addr;
+    map->l_map_end = (ElfW(Addr)) addr + size;
 
     for (ph = phdr; ph < &phdr[map->l_phnum]; ++ph)
         switch (ph->p_type) {
@@ -545,14 +540,12 @@ int add_elf_object(void * addr, PAL_HANDLE handle, int type)
                 break;
         }
 
-    map->l_real_ld = map->l_ld;
-    map->l_ld = remalloc(map->l_ld, sizeof(ElfW(Dyn)) * map->l_ldnum);
-
-    elf_get_dynamic_info(map->l_ld, map->l_info, 0);
+    map->l_real_ld = (void *) map->l_addr + (unsigned long) map->l_ld;
+    map->l_ld = remalloc(map->l_real_ld, sizeof(ElfW(Dyn)) * map->l_ldnum);
 
+    elf_get_dynamic_info(map->l_ld, map->l_info, map->l_addr);
     setup_elf_hash(map);
-
-    //ELF_DYNAMIC_SCAN(map->l_info, 0);
+    ELF_DYNAMIC_RELOCATE(map);
 
     struct link_map * prev = loaded_maps;
     while (prev->l_next)
@@ -560,7 +553,10 @@ int add_elf_object(void * addr, PAL_HANDLE handle, int type)
     map->l_prev = prev;
     map->l_next = NULL;
     prev->l_next = map;
+    if (type == OBJECT_EXEC)
+        exec_map = map;
 
+    _DkDebugAddMap(map);
     return 0;
 }
 

Pal/src/host/Linux/db_main.c

@@ -295,7 +295,7 @@ done_init:
     /* jump to main function */
     pal_main(linux_state.parent_process_id,
              (void *) pal_map.l_addr, pal_name, argc, argv, envp,
-             parent, first_thread, exec, manifest);
+             parent, first_thread, exec, NULL, manifest);
 }
 
 /* the following code is borrowed from CPUID */

Pal/src/pal_internal.h

@@ -365,11 +365,12 @@ typedef typeof(__pal_control.cpu_info) PAL_CPU_INFO;
 void _DkGetCPUInfo (PAL_CPU_INFO * info);
 
 /* Main function called by host-dependent bootstrapper */
-void pal_main (PAL_NUM pal_token, void * pal_addr, const char * pal_name,
+void pal_main (PAL_NUM pal_token, void * pal_addr,
+               const char * pal_name,
                int argc, const char ** argv, const char ** envp,
                PAL_HANDLE parent_handle,
                PAL_HANDLE thread_handle,
-               PAL_HANDLE exec_handle,
+               PAL_HANDLE exec_handle, PAL_PTR_RANGE * exec_range,
                PAL_HANDLE manifest_handle);
 
 void start_execution (const char * first_argv, int argc, const char ** argv,
@@ -479,6 +480,7 @@ enum object_type { OBJECT_RTLD, OBJECT_EXEC, OBJECT_PRELOAD, OBJECT_EXTERNAL };
 int check_elf_object (PAL_HANDLE handle);
 int load_elf_object (const char * uri, enum object_type type);
 int load_elf_object_by_handle (PAL_HANDLE handle, enum object_type type);
+int add_elf_object(void * addr, unsigned int size, PAL_HANDLE handle, int type);
 
 void init_slab_mgr (int alignment);
 void * malloc (int size);

Pal/test/Cpuid.c

@@ -0,0 +1,29 @@
+/* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
+/* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
+
+/* This Hello World simply print out "Hello World" */
+
+#include "pal.h"
+#include "pal_debug.h"
+
+int main (int argc, char ** argv, char ** envp)
+{
+    PAL_NUM values[4];
+    DkCpuIdRetrieve(0, values);
+
+    pal_printf("cpuid[0] = %08x %08x %08x %08x\n", values[0], values[1],
+               values[2], values[3]);
+
+    asm volatile("mov $0, %%rax\n"
+                 "cpuid\n"
+                 : "=a"(values[0]),
+                   "=b"(values[1]),
+                   "=c"(values[2]),
+                   "=d"(values[3])
+                :: "memory");
+
+    pal_printf("cpuid[0] = %08x %08x %08x %08x\n", values[0], values[1],
+               values[2], values[3]);
+
+    return 0;
+}

Pal/test/Makefile

@@ -6,7 +6,7 @@ CFLAGS	= -Wall -O2 -std=gnu99 -fgnu89-inline -fno-builtin -nostdlib \
 
 executables = HelloWorld File Failure Thread Fork Event Process Exception \
 	      Memory Pipe Tcp Udp Yield Broadcast Ipc Server Wait HandleSend \
-	      Select Segment
+	      Select Segment Cpuid Pie
 manifests = manifest
 
 target = $(executables) $(manifests)
@@ -21,12 +21,15 @@ debug:	CC=gcc -g
 debug:	$(target)
 
 manifest: manifest.template
-	cp -f manifest.template manifest
+	cp -f $< $@
+
+%.manifest: %.manifest.template
+	cp -f $< $@
 
 ifeq ($(SYS),x86_64-linux-gnu)
 $(executables): %: %.c $(graphene_lib) $(pal_lib) ../src/user_start.o
 	@echo [ $@ ]
-	@$(CC) $(CFLAGS) $^ -o $@
+	@$(CC) $(CFLAGS) $(if $(filter Pie,$@),-fPIC -pie,) $^ -o $@
 
 $(graphene_lib):
 	$(MAKE) -C ../lib target=$(shell pwd)/.lib/ $(debug)

Pal/test/Pie.c

@@ -0,0 +1,45 @@
+/* -*- mode:c; c-file-style:"k&r"; c-basic-offset: 4; tab-width:4; indent-tabs-mode:nil; mode:auto-fill; fill-column:78; -*- */
+/* vim: set ts=4 sw=4 et tw=78 fo=cqt wm=0: */
+
+/* This Hello World simply print out "Hello World" */
+
+#include "pal.h"
+#include "pal_debug.h"
+
+char str[13];
+
+int main (int argc, char ** argv, char ** envp)
+{
+    pal_printf("start program: %s\n", pal_control.executable);
+
+    PAL_HANDLE out = DkStreamOpen("dev:tty", PAL_ACCESS_WRONLY, 0, 0, 0);
+
+    if (out == NULL) {
+        pal_printf("DkStreamOpen failed\n");
+        return -1;
+    }
+
+    str[0] = 'H';
+    str[1] = 'e';
+    str[2] = 'l';
+    str[3] = 'l';
+    str[4] = 'o';
+    str[5] = ' ';
+    str[6] = 'W';
+    str[7] = 'o';
+    str[8] = 'r';
+    str[9] = 'l';
+    str[10] = 'd';
+    str[11] = '\n';
+    str[12] = 0;
+
+    int bytes = DkStreamWrite(out, 0, 12, str, NULL);
+
+    if (bytes < 0) {
+        pal_printf("DkStreamWrite failed\n");
+        return -1;
+    }
+
+    DkObjectClose(out);
+    return 0;
+}

Pal/test/Process.c

@@ -30,7 +30,7 @@ int main (int argc, char ** argv)
         PAL_HANDLE proc = DkProcessCreate ("file:Process", 0, newargs);
 
         if (!proc)
-            pal_printf("Can't creste process\n");
+            pal_printf("Can't create process\n");
 
         DkObjectClose(proc);
         DkThreadDelayExecution(30000000);

Scripts/regression.py

@@ -9,13 +9,17 @@ class Result:
         self.code = code
 
 class Regression:
-    def __init__(self, loader = None, executable = '', prepare = None, timeout = 1000, keep_log = False):
+    def __init__(self, loader = None, executable = '', prepare = None, timeout = 0):
         self.loader = loader
         self.executable = executable
         self.prepare = prepare
         self.runs = dict()
-        self.timeout = timeout
-        self.keep_log = keep_log
+        default_timeout = int(os.getenv('TIMEOUT', '1000'))
+        if default_timeout > timeout:
+            self.timeout = default_timeout
+        else:
+            self.timeout = timeout
+        self.keep_log = (os.getenv('KEEP_LOG', '0') == '1')
 
     def add_check(self, name, check, times = 1, args = []):
         combined_args = ' '.join(args)
@@ -44,6 +48,8 @@ class Regression:
                 if self.prepare:
                     self.prepare(args)
 
+                time.sleep(0.1)
+
                 p = subprocess.Popen(args,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE)
@@ -60,20 +66,15 @@ class Regression:
                 if not finish and p.poll() is None:
                     p.kill()
 
+                time.sleep(0.1)
+
                 out = p.stdout.read()
                 log = p.stderr.read()
 
-                if self.keep_log:
-                    sargs = [re.sub(r"\W", '_', a).strip('_') for a in args]
-                    filename = '_'.join(sargs) + '_' + time.strftime("%Y%m%d_%H%M%S")
-                    with open(filename + '.log', 'w') as f:
-                        f.write(log)
-                    with open(filename + '.out', 'w') as f:
-                        f.write(out)
-
                 outputs.append(Result(out, log, p.returncode))
 
                 run_times = run_times + 1
+                keep_log = False
                 for (name, check, times) in self.runs[combined_args]:
                     if run_times == times:
                         result = check(outputs)
@@ -81,3 +82,11 @@ class Regression:
                             print '\033[92m[Success]\033[0m', name
                         else:
                             print '\033[93m[Fail   ]\033[0m', name
+                            keep_log = True
+
+                if self.keep_log and keep_log:
+                    sargs = [re.sub(r"\W", '_', a).strip('_') for a in args]
+                    filename = 'log-' + '_'.join(sargs) + '_' + time.strftime("%Y%m%d_%H%M%S")
+                    with open(filename, 'w') as f:
+                        f.write(log + out)
+                    print 'keep log to %s' % (filename)