[Jenkins] Split SGX pipelines into two: tests and apps

Previously, it took more than 30-45 minutes to run one SGX pipeline.
This commit splits that SGX pipeline into two: one runs LibOS/PAL
tests and LTP tests, the other one runs only application examples.
The times to run each of these pipelines are roughly the same.
Splitting SGX pipelines decreases wait time for Jenkins in half.

Jenkinsfiles/Linux-SGX

@@ -80,69 +80,6 @@ pipeline {
                             make SGX=1 ltp-sgx.xml || :
                             '''
 
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/python-simple
-                                make SGX=1
-                                make SGX=1 check
-                            '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/python-scipy-insecure
-                                make SGX=1
-                                make SGX=1 check
-                            '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/bash
-                                make SGX=1 all
-                                make SGX=1 regression
-                           '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/curl
-                                make SGX=1 all
-                                make SGX=1 check
-                           '''
-                        }
-                        timeout(time: 10, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/gcc
-                                make SGX=1 all
-                                make SGX=1 check
-                           '''
-                        }
-                        timeout(time: 15, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/lmbench
-                                make SGX=1 all
-                                make SGX=1 test N_RUNS=1 ENOUGH=100
-                            '''
-                        }
-                        sh '''
-                            cd LibOS/shim/test/apps/lighttpd
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 10
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8003
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/nginx
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 30
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8002
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/apache
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 30
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8001
-                            '''
                         sh '''
                            cd "$(./Scripts/clean-check-test-copy)"
 
@@ -156,16 +93,6 @@ pipeline {
                            rm -rf /tmp/graphene-sgx.clean-check.clean/LibOS/shim/test/apps/ltp/src
                            rm -rf LibOS/shim/test/apps/ltp/src
 
-                           make -C LibOS/shim/test/apps/python-simple SGX=1 clean
-                           make -C LibOS/shim/test/apps/python-scipy-insecure SGX=1 clean
-                           make -C LibOS/shim/test/apps/bash SGX=1 clean
-                           make -C LibOS/shim/test/apps/curl SGX=1 clean
-                           make -C LibOS/shim/test/apps/gcc SGX=1 clean
-                           make -C LibOS/shim/test/apps/lmbench SGX=1 distclean
-                           make -C LibOS/shim/test/apps/lighttpd SGX=1 distclean
-                           make -C LibOS/shim/test/apps/nginx SGX=1 distclean
-                           make -C LibOS/shim/test/apps/apache SGX=1 distclean
-
                            ./Scripts/clean-check
                         '''
                     }

Jenkinsfiles/Linux-SGX-18.04

@@ -79,72 +79,6 @@ pipeline {
                             make SGX=1 all sgx-tokens
                             make SGX=1 ltp-sgx.xml || :
                             '''
-
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/python-simple
-                                PYTHONVERSION=python3.6 make SGX=1
-                                PYTHONVERSION=python3.6 make SGX=1 check
-                            '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/python-scipy-insecure
-                                PYTHONVERSION=python3.6 make SGX=1
-                                PYTHONVERSION=python3.6 make SGX=1 check
-                            '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/bash
-                                make SGX=1 all
-                                make SGX=1 regression
-                           '''
-                        }
-                        timeout(time: 5, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/curl
-                                make SGX=1 all
-                                make SGX=1 check
-                           '''
-                        }
-                        timeout(time: 10, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/gcc
-                                make SGX=1 all
-                                make SGX=1 check
-                           '''
-                        }
-                        timeout(time: 15, unit: 'MINUTES') {
-                            sh '''
-                                cd LibOS/shim/test/apps/lmbench
-                                make SGX=1 all
-                                make SGX=1 test N_RUNS=1 ENOUGH=100
-                            '''
-                        }
-                        sh '''
-                            cd LibOS/shim/test/apps/lighttpd
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 10
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8003
-                            '''
-                        sh '''
-                            cd LibOS/shim/test/apps/nginx
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 30
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8002
-                            '''
-                            /*
-                        sh '''
-                            cd LibOS/shim/test/apps/apache
-                            make SGX=1
-                            make SGX=1 start-graphene-server &
-                            sleep 30
-                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8001
-                            '''
-                            */
                     }
                     post {
                         always {

Jenkinsfiles/Linux-SGX-18.04-apps

@@ -0,0 +1,125 @@
+pipeline {
+        agent {
+            dockerfile { filename 'Jenkinsfiles/ubuntu-18.04.dockerfile'
+                         label 'sgx_slave'
+                         args "-v /lib/modules:/lib/modules:ro -v /usr/src:/usr/src:ro -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket --device=/dev/gsgx:/dev/gsgx --device=/dev/isgx:/dev/isgx"
+                       }
+        }
+        stages {
+                stage('Lint') {
+                    steps {
+                        sh '''
+                            ./.ci/run-pylint -f text || :
+                            ./.ci/run-pylint -f json | ./.ci/prfilter
+
+                            ./.ci/run-shellcheck || :
+                            ./.ci/run-shellcheck -f json | ./.ci/prfilter
+                        '''
+                    }
+                }
+                stage('Build') {
+                    steps {
+                        sh '''
+                            cd Pal/src/host/Linux-SGX/signer/ && openssl genrsa -3 -out enclave-key.pem 3072
+                        '''
+                        sh '''
+                           cd /opt/intel
+                           git clone https://github.com/01org/linux-sgx-driver.git
+                           cd linux-sgx-driver
+                           git checkout sgx_driver_1.9
+                           make
+                        '''
+                        sh '''
+                            cd Pal/src/host/Linux-SGX/sgx-driver
+                            ISGX_DRIVER_PATH=/opt/intel/linux-sgx-driver ISGX_DRIVER_VERSION=1.9 make
+                        '''
+                        sh '''
+                            make -j 8 SGX=1 WERROR=1
+                        '''
+                    }
+                }
+                stage('Test') {
+                    steps {
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/python-simple
+                                PYTHONVERSION=python3.6 make SGX=1
+                                PYTHONVERSION=python3.6 make SGX=1 check
+                            '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/python-scipy-insecure
+                                PYTHONVERSION=python3.6 make SGX=1
+                                PYTHONVERSION=python3.6 make SGX=1 check
+                            '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/bash
+                                make SGX=1 all
+                                make SGX=1 regression
+                           '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/curl
+                                make SGX=1 all
+                                make SGX=1 check
+                           '''
+                        }
+                        timeout(time: 10, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make SGX=1 all
+                                make SGX=1 check
+                           '''
+                        }
+                        timeout(time: 15, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make SGX=1 all
+                                make SGX=1 test N_RUNS=1 ENOUGH=100
+                            '''
+                        }
+                        sh '''
+                            cd LibOS/shim/test/apps/lighttpd
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 10
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8003
+                            '''
+                        sh '''
+                            cd LibOS/shim/test/apps/nginx
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 30
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8002
+                            '''
+                        sh '''
+                            cd LibOS/shim/test/apps/apache
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 30
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8001
+                            '''
+                    }
+                }
+                stage('Deploy') {
+                    steps {
+                        sh 'echo Deploying code'
+                    }
+                }
+        }
+        post {
+                success {
+                        echo 'Deployment successful'
+                }
+                failure {
+                        echo 'Failure while on the pipeline'
+                }
+                unstable {
+                        echo 'Pipeline marked as "unstable"'
+                }
+        }
+}

Jenkinsfiles/Linux-SGX-apps

@@ -0,0 +1,152 @@
+pipeline {
+        agent {
+            dockerfile { filename 'Jenkinsfiles/ubuntu-16.04.dockerfile'
+                         label 'sgx_slave'
+                         args "-v /lib/modules:/lib/modules:ro -v /usr/src:/usr/src:ro -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket --device=/dev/gsgx:/dev/gsgx --device=/dev/isgx:/dev/isgx"
+                       }
+        }
+        stages {
+                stage('Lint') {
+                    steps {
+                        sh '''
+                            ./.ci/run-shellcheck || :
+                            ./.ci/run-shellcheck -f json | ./.ci/prfilter
+                        '''
+                    }
+                }
+                stage('Build') {
+                    steps {
+                        sh '''
+                           ./Scripts/clean-check-prepare
+                        '''
+                        sh '''
+                            cd Pal/src/host/Linux-SGX/signer/ && openssl genrsa -3 -out enclave-key.pem 3072
+                        '''
+                        sh '''
+                           cd /opt/intel
+                           git clone https://github.com/01org/linux-sgx-driver.git
+                           cd linux-sgx-driver
+                           git checkout sgx_driver_1.9
+                           make
+                        '''
+                        sh '''
+                            cd Pal/src/host/Linux-SGX/sgx-driver
+                            ISGX_DRIVER_PATH=/opt/intel/linux-sgx-driver ISGX_DRIVER_VERSION=1.9 make
+                        '''
+                        sh '''
+                            make -j 8 SGX=1 WERROR=1
+                        '''
+                    }
+                }
+                stage('Test') {
+                    steps {
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/python-simple
+                                make SGX=1
+                                make SGX=1 check
+                            '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/python-scipy-insecure
+                                make SGX=1
+                                make SGX=1 check
+                            '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/bash
+                                make SGX=1 all
+                                make SGX=1 regression
+                           '''
+                        }
+                        timeout(time: 5, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/curl
+                                make SGX=1 all
+                                make SGX=1 check
+                           '''
+                        }
+                        timeout(time: 10, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/gcc
+                                make SGX=1 all
+                                make SGX=1 check
+                           '''
+                        }
+                        timeout(time: 15, unit: 'MINUTES') {
+                            sh '''
+                                cd LibOS/shim/test/apps/lmbench
+                                make SGX=1 all
+                                make SGX=1 test N_RUNS=1 ENOUGH=100
+                            '''
+                        }
+                        sh '''
+                            cd LibOS/shim/test/apps/lighttpd
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 10
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8003
+                            '''
+                        sh '''
+                            cd LibOS/shim/test/apps/nginx
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 30
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8002
+                            '''
+                        sh '''
+                            cd LibOS/shim/test/apps/apache
+                            make SGX=1
+                            make SGX=1 start-graphene-server &
+                            sleep 30
+                            LOOP=1 CONCURRENCY_LIST="1 32" ./benchmark-http.sh 127.0.0.1:8001
+                            '''
+                        sh '''
+                           cd "$(./Scripts/clean-check-test-copy)"
+
+                           rm Pal/src/host/Linux-SGX/signer/enclave-key.pem
+                           make -C Pal/src/host/Linux-SGX/sgx-driver distclean
+                           make SGX=1 clean
+                           rm LibOS/glibc-*.tar.gz
+
+                           make -C LibOS/shim/test/apps/python-simple SGX=1 clean
+                           make -C LibOS/shim/test/apps/python-scipy-insecure SGX=1 clean
+                           make -C LibOS/shim/test/apps/bash SGX=1 clean
+                           make -C LibOS/shim/test/apps/curl SGX=1 clean
+                           make -C LibOS/shim/test/apps/gcc SGX=1 clean
+                           make -C LibOS/shim/test/apps/lmbench SGX=1 distclean
+                           make -C LibOS/shim/test/apps/lighttpd SGX=1 distclean
+                           make -C LibOS/shim/test/apps/nginx SGX=1 distclean
+                           make -C LibOS/shim/test/apps/apache SGX=1 distclean
+
+                           # Currently used release of LTP contains broken symlinks under
+                           # utils/ffsb-6.0-rc2/ (config.guess and config.sub); without explicit
+                           # rm of these symlinks, the clean-check utility fails. We should
+                           # update to newer release of LTP where this issue will be fixed.
+                           rm -rf /tmp/graphene-sgx-apps.clean-check.clean/LibOS/shim/test/apps/ltp/src
+                           rm -rf LibOS/shim/test/apps/ltp/src
+
+                           ./Scripts/clean-check
+                        '''
+                    }
+                }
+                stage('Deploy') {
+                    steps {
+                        sh 'echo Deploying code'
+                    }
+                }
+        }
+        post {
+                success {
+                        echo 'Deployment successful'
+                }
+                failure {
+                        echo 'Failure while on the pipeline'
+                }
+                unstable {
+                        echo 'Pipeline marked as "unstable"'
+                }
+        }
+}