CI Improvements (#141)

Run CI in a Docker image

* Wait a second for lighttp and apache to fully initialize the socket

* Try to add a timeout for lmbench, which is prone to hanging (vfork)

* Set a timeout on gcc; one of the PRs is causing gcc hello to hang.  Also, to avoid interference, run apache test on 8001 by default.

* Update lmbench to ignore bin dir

* Do the faster tests first; fail earlier

* Get the IP addr properly for web server tests

* Some notes on flaky LTP tests to debug later.  Move a few that tend to heisenbugs off of the required list for now 

* Try to detect vma list corruption during the debug dump, lest it be infinite.

* Temporarily disable the lmbench vfork test (Issue #142).

* Catch some failures faster.

* Catch failures in apachebench

* Take apache out of the Linux host/Debug build CI for now, as it hangs consistently.  Filed issue #144 to track.

Jenkinsfiles/Linux

@@ -1,51 +1,59 @@
 pipeline {
-        agent any
+	agent {
+              dockerfile { filename 'Jenkinsfiles/ubuntu-16.04.dockerfile' }
+	}
         stages {
                 stage('Build') {
                     steps {
                         sh '''
-                            make clean && make
+                            make
                            '''
                     }
                 }
                 stage('Test') {
                     steps {
                         sh '''
-                            cd LibOS/shim/test/apps/gcc
+                            cd Pal/regression
                             make regression
-                           '''
+                            '''
                         sh '''
-                            cd LibOS/shim/test/apps/lmbench
+                            cd LibOS/shim/test/regression
                             make regression
                             '''
+                        sh '''
+                            cd LibOS/shim/test/apps/ltp
+                            make
+                            ./syscalls.sh
+                            '''
                         sh '''
                             cd LibOS/shim/test/apps/python
                             make regression
                             '''
+			timeout(time: 5, unit: 'MINUTES') {		    
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make regression
+                           '''
+			}
+			timeout(time: 15, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make regression
+                            '''
+			}
                         sh '''
                             cd LibOS/shim/test/apps/lighttpd
                             make
                             make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
+			    sleep 1
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8000			    
                             '''
                         sh '''
                             cd LibOS/shim/test/apps/apache
                             make
                             make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
-                            '''
-                        sh '''
-                            cd Pal/regression
-                            make regression
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/regression
-                            make regression
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/ltp
-                            make
-                            ./syscalls.sh
+			    sleep 1			    
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8001
                             '''
                     }
                 }

Jenkinsfiles/Linux-Debug

@@ -1,51 +1,52 @@
 pipeline {
-        agent any
+	agent {
+              dockerfile { filename 'Jenkinsfiles/ubuntu-16.04.dockerfile' }
+	}
         stages {
                 stage('Build') {
                     steps {
                         sh '''
-                            make clean && make DEBUG=1
+                            make DEBUG=1
                            '''
                     }
                 }
                 stage('Test') {
                     steps {
                         sh '''
-                            cd LibOS/shim/test/apps/gcc
-                            make regression
-                           '''
-                        sh '''
-                            cd LibOS/shim/test/apps/lmbench
+                            cd Pal/regression
                             make regression
                             '''
                         sh '''
-                            cd LibOS/shim/test/apps/python
+                            cd LibOS/shim/test/regression
                             make regression
                             '''
                         sh '''
-                            cd LibOS/shim/test/apps/lighttpd
-                            make
-                            make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/apache
+                            cd LibOS/shim/test/apps/ltp
                             make
-                            make start-graphene-server &
-                            ./benchmark-http.sh 127.0.0.1:8000
+                            ./syscalls.sh
                             '''
                         sh '''
-                            cd Pal/regression
+                            cd LibOS/shim/test/apps/python
                             make regression
                             '''
-                        sh '''
-                            cd LibOS/shim/test/regression
-                            make regression
+			timeout(time: 5, unit: 'MINUTES') {		    
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make regression
+                           '''
+			}
+			timeout(time: 20, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make regression
                             '''
+			}
                         sh '''
-                            cd LibOS/shim/test/apps/ltp
+                            cd LibOS/shim/test/apps/lighttpd
                             make
-                            ./syscalls.sh
+                            make start-graphene-server &
+			    sleep 1
+                            ./benchmark-http.sh `hostname -I|tr -d '[:space:]'`:8000
                             '''
                     }
                 }

Jenkinsfiles/ubuntu-16.04.dockerfile

@@ -0,0 +1,27 @@
+# Start with 16.04
+FROM ubuntu:16.04
+
+# Add steps here to set up dependencies
+RUN apt-get update && apt-get install -y \
+    apache2-utils \
+    autoconf \
+    build-essential \
+    gawk \
+    gettext \
+    git \
+    libpcre3-dev \
+    libxml2-dev \
+    net-tools \
+    python \
+    texinfo \
+    wget
+
+
+# Set environment variables.
+ENV HOME /root
+
+# Define working directory.
+WORKDIR /root
+
+# Define default command.
+CMD ["bash"]

LibOS/Makefile

@@ -18,7 +18,8 @@ ifeq ($(SYS),x86_64-linux-gnu)
 
 $(BUILD_DIR)/Build.success: $(BUILD_DIR)/Makefile
 	@echo "Building glibc, may take a while to finish. Warning messages may show up. If this process terminates with failures, see \"$(BUILD_DIR)/build.log\" for more information."
-	cd $(BUILD_DIR) && (($(MAKE) >> build.log) 2>&1 | tee -a build.log)
+	cd $(BUILD_DIR) && ($(MAKE) 2>&1 >> build.log)
+#  2>&1 | tee -a build.log)
 
 $(GLIBC_TARGET): $(BUILD_DIR)/Build.success
 

LibOS/shim/src/bookkeep/shim_vma.c

@@ -903,8 +903,15 @@ static struct shim_vma * __lookup_supervma (const void * addr, uint64_t length,
         if (!(!prev || prev->addr + prev->length <= tmp->addr)) {
             struct shim_vma * tmp2;
             warn("Failure\n");
+            uint64_t last_addr = 0;
             listp_for_each_entry(tmp2, &vma_list, list) {
                 warn ("Entry: %llx..%llx (%llx)\n", tmp2->addr, tmp2->addr + tmp2->length, tmp2->length);
+                // Don't do an infinite dump if the list gets corrupted
+                if (tmp2->addr < last_addr) {
+                    warn("VMA list corruption detected.  Stopping debug print.\n");
+                    break;
+                }
+                last_addr = tmp2->addr;
             }
             warn("Prev is %p, tmp->addr = %llx, len is %llx\n", prev, tmp->addr, tmp->length);
             if (prev)

LibOS/shim/test/apps/apache/.gitignore

@@ -6,3 +6,6 @@ OUTPUT
 apr-1.4.6/*
 apr-util-1.5.1/*
 php-5.6.6/*
+mod_auth_basic_sandbox.la
+mod_auth_basic_sandbox.lo
+mod_auth_basic_sandbox.slo

LibOS/shim/test/apps/apache/Makefile

@@ -1,5 +1,5 @@
 HOST ?= $(firstword $(shell ifconfig | grep 'inet addr:' | grep -v '127.0.0.1' -m 1 | cut -d: -f2))
-PORT ?= 8000
+PORT ?= 8001
 
 NPROCS := 1
 OS := $(shell uname -s)

LibOS/shim/test/apps/lighttpd/benchmark-http.sh

@@ -22,7 +22,7 @@ do
 	do
 		rm -f OUTPUT
 		echo "ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE"
-		ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE > OUTPUT
+		ab -n $REQUESTS -c $CONCURRENCY http://$DOWNLOAD_HOST/$DOWNLOAD_FILE > OUTPUT || exit $?
 
 		sleep 5
 

LibOS/shim/test/apps/lmbench/lmbench-2.5

@@ -1 +1 @@
-Subproject commit 8feddfa6e4064c0591ed6c593e679f45b83c958f
+Subproject commit 274a0210fcb73a466a316648ac75c8d1938f715e

LibOS/shim/test/apps/ltp/FLAKY

@@ -0,0 +1,18 @@
+Flaky tests
+-----------
+
+These should be debugged, or revisited after memory debugging PRs are merged
+
+waitpid03,1 - fails about 20% of the time
+preadv01 - fails intermittently in CI - perhaps an unrelated bug?
+preadv01,2
+preadv01,3
+preadv01,4
+
+waitpid02 - Gets a segfault in debug build fairly often
+waitpid02,1
+waitpid02,2
+waitpid02,3
+
+clock_nanosleep01,11 - Pretty prone to hanging, don't think it is a timeout
+

LibOS/shim/test/apps/ltp/PASSED

@@ -47,7 +47,6 @@ clock_getres01,9
 clock_getres01,10
 clock_getres01,11
 clock_getres01,12
-clock_nanosleep01,11
 clone01,1
 clone03,1
 clone04,1
@@ -340,9 +339,6 @@ pread01,1
 pread01_64,1
 pread02,1
 pread02_64,1
-preadv01,2
-preadv01,3
-preadv01,4
 preadv01_64,2
 preadv01_64,3
 preadv01_64,4
@@ -942,9 +938,6 @@ wait401,1
 wait401,2
 waitpid01,1
 waitpid01,2
-waitpid02,1
-waitpid02,2
-waitpid02,3
 waitpid05,1
 waitpid05,2
 waitpid05,3

LibOS/shim/test/apps/ltp/TIMEOUTS

@@ -1,8 +1,10 @@
 testcase,timeout
 alarm01,5
 alarm06,16
+clock_nanosleep01,40
 clone03,40
 clone05,30
+clone06,40
 exit01,40
 faccessat01,80
 fchdir02,40
@@ -22,7 +24,6 @@ lseek02,40
 pread01_64,40
 pread01,40
 pread02,40
-preadv01,40
 read01,40
 readdir01,40
 readv01,40
@@ -36,5 +37,5 @@ stat05_64,40
 syscall01,80
 wait401,40
 waitpid03,80
-waitpid05,160
+waitpid05,240
 write03,40

LibOS/shim/test/apps/ltp/fetch.py

@@ -101,7 +101,7 @@ def finish(result):
 CRED = '\033[91m'
 CGREEN = '\033[92m'
 CEND = '\033[0m'
-DEFAULT_TIMEOUT = 20
+DEFAULT_TIMEOUT = 30
 
 resultfile = "run_output"
 stablePass = "PASSED"

LibOS/shim/test/regression/Makefile

@@ -54,11 +54,11 @@ endif
 .PHONY: regression
 regression: $(target)
 	@echo "\n\nBasic Bootstrapping:"
-	@for f in $(wildcard 00_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 00_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 	@echo "\n\nSyscall Support:"
-	@for f in $(wildcard 30_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 30_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 	@echo "\n\nLarge File Support:"
-	@for f in $(wildcard 90_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST); done
+	@for f in $(wildcard 90_*.py); do env $(PYTHONENV) python $$f $(RUNTIME)/pal-$(PAL_HOST) || exit $?; done
 
 .PHONY: clean-tmp
 clean-tmp: