Merge branch 'master' of https://github.com/oscarlab/graphene

Jenkinsfiles/Linux

@@ -1,51 +1,59 @@
 pipeline {
-        agent any
+	agent {
+              dockerfile { filename 'Jenkinsfiles/ubuntu-16.04.dockerfile' }
+	}
         stages {
                 stage('Build') {
                     steps {
                         sh '''
-                            make clean && make
+                            make
                            '''
                     }
                 }
                 stage('Test') {
                     steps {
                         sh '''
-                            cd LibOS/shim/test/apps/gcc
+                            cd Pal/regression
                             make regression
-                           '''
+                            '''
                         sh '''
-                            cd LibOS/shim/test/apps/lmbench
+                            cd LibOS/shim/test/regression
                             make regression
                             '''
+                        sh '''
+                            cd LibOS/shim/test/apps/ltp
+                            make
+                            ./syscalls.sh
+                            '''
                         sh '''
                             cd LibOS/shim/test/apps/python
                             make regression
                             '''
+			timeout(time: 5, unit: 'MINUTES') {		    
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make regression
+                           '''
+			}
+			timeout(time: 15, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make regression
+                            '''
+			}
                         sh '''
                             cd LibOS/shim/test/apps/lighttpd
                             make
                             make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
+			    sleep 1
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8000			    
                             '''
                         sh '''
                             cd LibOS/shim/test/apps/apache
                             make
                             make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
-                            '''
-                        sh '''
-                            cd Pal/regression
-                            make regression
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/regression
-                            make regression
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/ltp
-                            make
-                            ./syscalls.sh
+			    sleep 1			    
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8001
                             '''
                     }
                 }

Jenkinsfiles/Linux-Debug

@@ -1,51 +1,52 @@
 pipeline {
-        agent any
+	agent {
+              dockerfile { filename 'Jenkinsfiles/ubuntu-16.04.dockerfile' }
+	}
         stages {
                 stage('Build') {
                     steps {
                         sh '''
-                            make clean && make DEBUG=1
+                            make DEBUG=1
                            '''
                     }
                 }
                 stage('Test') {
                     steps {
                         sh '''
-                            cd LibOS/shim/test/apps/gcc
-                            make regression
-                           '''
-                        sh '''
-                            cd LibOS/shim/test/apps/lmbench
+                            cd Pal/regression
                             make regression
                             '''
                         sh '''
-                            cd LibOS/shim/test/apps/python
+                            cd LibOS/shim/test/regression
                             make regression
                             '''
                         sh '''
-                            cd LibOS/shim/test/apps/lighttpd
-                            make
-                            make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/apache
+                            cd LibOS/shim/test/apps/ltp
                             make
-                            make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
+                            ./syscalls.sh
                             '''
                         sh '''
-                            cd Pal/regression
+                            cd LibOS/shim/test/apps/python
                             make regression
                             '''
-                        sh '''
-                            cd LibOS/shim/test/regression
-                            make regression
+			timeout(time: 5, unit: 'MINUTES') {		    
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make regression
+                           '''
+			}
+			timeout(time: 20, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make regression
                             '''
+			}
                         sh '''
-                            cd LibOS/shim/test/apps/ltp
+                            cd LibOS/shim/test/apps/lighttpd
                             make
-                            ./syscalls.sh
+                            make start-graphene-server &
+			    sleep 1
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8000
                             '''
                     }
                 }

Jenkinsfiles/ubuntu-16.04.dockerfile

@@ -0,0 +1,27 @@
+# Start with 16.04
+FROM ubuntu:16.04
+
+# Add steps here to set up dependencies
+RUN apt-get update && apt-get install -y \
+    apache2-utils \
+    autoconf \
+    build-essential \
+    gawk \
+    gettext \
+    git \
+    libpcre3-dev \
+    libxml2-dev \
+    net-tools \
+    python \
+    texinfo \
+    wget
+
+
+# Set environment variables.
+ENV HOME /root
+
+# Define working directory.
+WORKDIR /root
+
+# Define default command.
+CMD ["bash"]

LibOS/Makefile

@@ -18,7 +18,8 @@ ifeq ($(SYS),x86_64-linux-gnu)
 
 $(BUILD_DIR)/Build.success: $(BUILD_DIR)/Makefile
 	@echo "Building glibc, may take a while to finish. Warning messages may show up. If this process terminates with failures, see \"$(BUILD_DIR)/build.log\" for more information."
-	cd $(BUILD_DIR) && (($(MAKE) >> build.log) 2>&1 | tee -a build.log)
+	cd $(BUILD_DIR) && ($(MAKE) 2>&1 >> build.log)
+#  2>&1 | tee -a build.log)
 
 $(GLIBC_TARGET): $(BUILD_DIR)/Build.success
 

LibOS/shim/include/shim_utils.h

@@ -235,9 +235,9 @@ int create_handle (const char * prefix, char * path, size_t size,
 
 /* Asynchronous event support */
 int init_async (void);
-int install_async_event (PAL_HANDLE object, unsigned long time,
-                         void (*callback) (IDTYPE caller, void * arg),
-                         void * arg);
+uint64_t install_async_event (PAL_HANDLE object, unsigned long time,
+                              void (*callback) (IDTYPE caller, void * arg),
+                              void * arg);
 int create_async_helper (void);
 int terminate_async_helper (void);
 

LibOS/shim/src/bookkeep/shim_vma.c

@@ -903,8 +903,15 @@ static struct shim_vma * __lookup_supervma (const void * addr, uint64_t length,
         if (!(!prev || prev->addr + prev->length <= tmp->addr)) {
             struct shim_vma * tmp2;
             warn("Failure\n");
+            uint64_t last_addr = 0;
             listp_for_each_entry(tmp2, &vma_list, list) {
                 warn ("Entry: %llx..%llx (%llx)\n", tmp2->addr, tmp2->addr + tmp2->length, tmp2->length);
+                // Don't do an infinite dump if the list gets corrupted
+                if (tmp2->addr < last_addr) {
+                    warn("VMA list corruption detected.  Stopping debug print.\n");
+                    break;
+                }
+                last_addr = tmp2->addr;
             }
             warn("Prev is %p, tmp->addr = %llx, len is %llx\n", prev, tmp->addr, tmp->length);
             if (prev)

LibOS/shim/src/shim_async.c

@@ -53,15 +53,17 @@ static AEVENTTYPE           async_helper_event;
 
 static LOCKTYPE async_helper_lock;
 
-int install_async_event (PAL_HANDLE object, unsigned long time,
-                         void (*callback) (IDTYPE caller, void * arg),
-                         void * arg)
+/* Returns remaining usecs */
+uint64_t install_async_event (PAL_HANDLE object, unsigned long time,
+                              void (*callback) (IDTYPE caller, void * arg),
+                              void * arg)
 {
     struct async_event * event =
                     malloc(sizeof(struct async_event));
 
     unsigned long install_time = DkSystemTimeQuery();
-
+    uint64_t rv = 0;
+    
     debug("install async event at %llu\n", install_time);
 
     event->callback     = callback;
@@ -97,6 +99,7 @@ int install_async_event (PAL_HANDLE object, unsigned long time,
          */
 		listp_del(tmp, &async_list, list);
         free(tmp);
+        rv = tmp->expire_time - install_time;
     } else
 	   tmp = NULL;
     
@@ -114,7 +117,7 @@ int install_async_event (PAL_HANDLE object, unsigned long time,
     unlock(async_helper_lock);
     
     set_event(&async_helper_event, 1);
-    return 0;
+    return rv;
 }
 
 int init_async (void)

LibOS/shim/src/sys/shim_alarm.c

@@ -43,7 +43,11 @@ void signal_alarm (IDTYPE target, void * arg)
 int shim_do_alarm (unsigned int seconds)
 {
     uint64_t usecs = 1000000ULL * seconds;
-    return install_async_event(NULL, usecs, &signal_alarm, NULL);
+    uint64_t usecs_left = install_async_event(NULL, usecs, &signal_alarm, NULL);
+    // Alarm expects the number of seconds remaining.  Round up.
+    int secs = usecs_left / 1000000ULL;
+    if (usecs_left % 1000000ULL) secs++;
+    return secs;
 }
 
 static struct {
@@ -91,8 +95,8 @@ int shim_do_setitimer (int which, struct __kernel_itimerval * value,
                                     real_itimer.timeout - setup_time : 0;
     unsigned long current_reset = real_itimer.reset;
 
-    int ret = install_async_event(NULL, next_value, &signal_itimer,
-                                  (void *) (setup_time + next_value));
+    uint64_t ret = install_async_event(NULL, next_value, &signal_itimer,
+                                       (void *) (setup_time + next_value));
 
     if (ret < 0) {
         master_unlock();

LibOS/shim/src/sys/shim_socket.c

@@ -1676,7 +1676,7 @@ int shim_do_getsockopt (int fd, int level, int optname, char * optval,
                 }
                 goto out;
             case SO_TYPE:
-                *intval = sock->protocol;
+                *intval = sock->sock_type;
                 goto out;
             case SO_KEEPALIVE:
             case SO_LINGER:

LibOS/shim/test/apps/apache/.gitignore

@@ -6,3 +6,6 @@ OUTPUT
 apr-1.4.6/*
 apr-util-1.5.1/*
 php-5.6.6/*
+mod_auth_basic_sandbox.la
+mod_auth_basic_sandbox.lo
+mod_auth_basic_sandbox.slo

LibOS/shim/test/apps/apache/Makefile

@@ -1,5 +1,5 @@
 HOST ?= $(firstword $(shell ifconfig | grep 'inet addr:' | grep -v '127.0.0.1' -m 1 | cut -d: -f2))
-PORT ?= 8000
+PORT ?= 8001
 
 NPROCS := 1
 OS := $(shell uname -s)

LibOS/shim/test/apps/lighttpd/.gitignore

@@ -5,4 +5,4 @@ build
 html
 lighttpd-*
 lighttpd.conf
-
+server.pem

LibOS/shim/test/apps/lighttpd/benchmark-http.sh

@@ -22,7 +22,7 @@ do
 	do
 		rm -f OUTPUT
 		echo "ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE"
-		ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE > OUTPUT
+		ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE > OUTPUT || exit $?
 
 		sleep 5
 

LibOS/shim/test/apps/ltp/FLAKY

@@ -0,0 +1,20 @@
+Flaky tests
+-----------
+
+These should be debugged, or revisited after memory debugging PRs are merged
+
+waitpid03,1 - fails about 20% of the time
+preadv01 - fails intermittently in CI - perhaps an unrelated bug?
+preadv01,2
+preadv01,3
+preadv01,4
+
+waitpid02 - Gets a segfault in debug build fairly often
+waitpid02,1
+waitpid02,2
+waitpid02,3
+
+clock_nanosleep01,11 - Pretty prone to hanging, don't think it is a timeout
+
+sendfile05,1 - pretty prone to a segfault, perhaps an unrelated issue
+Internal memory fault at 0x8 (IP = +0x34f1a, VMID = 3902099696, TID = 1)

LibOS/shim/test/apps/ltp/PASSED

@@ -9,6 +9,9 @@ alarm01,1
 alarm02,1
 alarm02,2
 alarm02,3
+alarm03,2
+alarm05,1
+alarm06,1
 asyncio02,1
 asyncio02,2
 asyncio02,3
@@ -47,7 +50,6 @@ clock_getres01,9
 clock_getres01,10
 clock_getres01,11
 clock_getres01,12
-clock_nanosleep01,11
 clone01,1
 clone03,1
 clone04,1
@@ -340,9 +342,6 @@ pread01,1
 pread01_64,1
 pread02,1
 pread02_64,1
-preadv01,2
-preadv01,3
-preadv01,4
 preadv01_64,2
 preadv01_64,3
 preadv01_64,4
@@ -728,7 +727,6 @@ sendfile03,3
 sendfile03_64,1
 sendfile03_64,2
 sendfile03_64,3
-sendfile05,1
 sendfile05_64,1
 setgid01,1
 setitimer01,1
@@ -942,9 +940,6 @@ wait401,1
 wait401,2
 waitpid01,1
 waitpid01,2
-waitpid02,1
-waitpid02,2
-waitpid02,3
 waitpid05,1
 waitpid05,2
 waitpid05,3

LibOS/shim/test/apps/ltp/TIMEOUTS

@@ -1,40 +1,58 @@
 testcase,timeout
 alarm01,5
+alarm02,40
+alarm03,40
+alarm05,40
 alarm06,16
+brk01,40
+chmod02,40
+clock_nanosleep01,40
 clone03,40
 clone05,30
+clone06,40
+dup204,40
 exit01,40
 faccessat01,80
 fchdir02,40
 fchmod07,40
+fchmodat01,40
 fcntl02_64,40
 fcntl13_64,40
 fork02,40
 futex_wait_bitset02,40
 fsync03,40
 gethostname01,40
+getgid03,40
+getpid02,40
 gettid01,40
 msgsnd01,40
 mprotect04,40
 newuname01,40
-kill09,40
+kill09,80
 lseek02,40
+poll01,40
 pread01_64,40
 pread01,40
 pread02,40
 preadv01,40
+pwrite04_64,40
+pwritev01,40
 read01,40
 readdir01,40
 readv01,40
+semop04,40
 sendfile03_64,40
+setuid01,40
 setuid02,40
 sockioctl01,40
 sigaltstack02,40
 socket01,40
 socket02,40
 stat05_64,40
+sync01,40
 syscall01,80
 wait401,40
 waitpid03,80
-waitpid05,160
+waitpid05,240
 write03,40
+write05,40

LibOS/shim/test/apps/ltp/fetch.py

@@ -101,7 +101,7 @@ def finish(result):
 CRED = '\033[91m'
 CGREEN = '\033[92m'
 CEND = '\033[0m'
-DEFAULT_TIMEOUT = 20
+DEFAULT_TIMEOUT = 30
 
 resultfile = "run_output"
 stablePass = "PASSED"

LibOS/shim/test/regression/80_sockets.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python
+
+import os, sys, mmap
+from regression import Regression
+
+loader = sys.argv[1]
+
+# Running getsockopt
+regression = Regression(loader, "getsockopt", None)
+
+regression.add_check(name="getsockopt",
+    check=lambda res: "getsockopt: Got socket type OK" in res[0].out)
+
+rv = regression.run_checks()
+if rv: sys.exit(rv)

LibOS/shim/test/regression/Makefile

@@ -54,11 +54,13 @@ endif
 .PHONY: regression
 regression: $(target)
 	@echo "\n\nBasic Bootstrapping:"
-	@for f in $(wildcard 00_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 00_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 	@echo "\n\nSyscall Support:"
-	@for f in $(wildcard 30_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 30_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
+	@echo "\n\nSocket Support:"
+	@for f in $(wildcard 80_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 	@echo "\n\nLarge File Support:"
-	@for f in $(wildcard 90_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 90_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 
 .PHONY: clean-tmp
 clean-tmp:

LibOS/shim/test/regression/getsockopt.c

@@ -0,0 +1,50 @@
+/* Unit test for issue #92.  
+ * Example for use of getsockopt with SO_TYPE 
+ * taken from here: http://alas.matf.bg.ac.rs/manuals/lspe/snode=103.html
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <assert.h>
+
+int main(int argc,char **argv) {
+
+  int z;
+  int s = -1;                /* Socket */
+  int so_type = -1;    /* Socket type */
+  socklen_t optlen;  /* Option length */
+  int rv;
+  
+  /*
+   * Create a TCP/IP socket to use:
+   */
+  s = socket(PF_INET,SOCK_STREAM,0);
+  if ( s == -1 ) {
+    printf("socket(2) error %d", errno);
+    exit(-1);
+  }
+
+  /*
+   * Get socket option SO_SNDBUF:
+   */
+  optlen = sizeof so_type;
+  z = getsockopt(s,SOL_SOCKET,SO_TYPE,
+		 &so_type,&optlen);
+  if ( z ) {
+    printf("getsockopt(s,SOL_SOCKET,"
+	    "SO_TYPE) %d", errno);
+    exit(-1);
+  }
+  
+  assert(optlen == sizeof so_type);
+  if (so_type == SOCK_STREAM) {
+    printf("getsockopt: Got socket type OK\n");
+  } else {
+    printf("getsockopt: Got socket type failed\n");
+    rv = -1;
+  }
+    
+  return rv;
+}
+