Browse Source

[Make] Improve download handling

Instead of implementing downloading of external resources in every
Makefile again, use one script. This script adds the following
features:

 - Always check the download against a known SHA-256 hash.

 - Support caching of downloaded resources (set DL_CACHE=/some/dir).

 - Allow offline builds if all files are cached. If DL_OFFLINE=true the
   build will never attempt to download anything.
Simon Gaiser 4 years ago
parent
commit
9052837dfc
6 changed files with 84 additions and 12 deletions
  1. 1 0
      .ci/run-shellcheck
  2. 1 6
      LibOS/Makefile
  3. 1 1
      LibOS/shim/test/apps
  4. 1 4
      Pal/lib/Makefile
  5. 2 1
      Pal/src/host/Linux-SGX/Makefile
  6. 78 0
      Scripts/download

+ 1 - 0
.ci/run-shellcheck

@@ -15,6 +15,7 @@ shellcheck "$@" \
     Scripts/clean-check \
     Scripts/clean-check-prepare \
     Scripts/clean-check-test-copy \
+    Scripts/download \
     Scripts/list-all-graphene.sh \
     Scripts/memusg \
     .ci/run-pylint \

+ 1 - 6
LibOS/Makefile

@@ -101,8 +101,6 @@ endif
 GLIBC_PATCHES += $(GLIBC_PATCHES_$(GLIBC_VERSION))
 
 $(GLIBC_SRC)/configure: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz
-	[ "`sha256sum $(GLIBC_SRC).tar.gz`" = "$(GLIBC_CHECKSUM)  $(GLIBC_SRC).tar.gz" ] || \
-		(echo "*** $(GLIBC_SRC).tar.gz has a wrong checksum ***"; exit 255)
 	rm -rf $(GLIBC_SRC)
 	tar -xzf $(GLIBC_SRC).tar.gz
 	cd $(GLIBC_SRC) && \
@@ -113,10 +111,7 @@ $(GLIBC_SRC)/configure: $(GLIBC_PATCHES) $(GLIBC_SRC).tar.gz
 	touch $@
 
 $(GLIBC_SRC).tar.gz:
-	for MIRROR in $(GLIBC_MIRRORS); do \
-		wget --timeout=10 $${MIRROR}glibc/$(GLIBC_SRC).tar.gz \
-		&& break; \
-	done
+	../Scripts/download --output $@ --sha256 $(GLIBC_CHECKSUM) $(foreach mirror,$(GLIBC_MIRRORS),--url $(mirror)glibc/$(GLIBC_SRC).tar.gz)
 
 $(GLIBC_SRC)/elf/Versions: $(GLIBC_SRC)/configure
 

+ 1 - 1
LibOS/shim/test/apps

@@ -1 +1 @@
-Subproject commit 52fd0acf430cdfd77279a1233cf1409bf946ad90
+Subproject commit 5f77829328eb03e32b6d3a500ed9c61f0f3b4b66

+ 1 - 4
Pal/lib/Makefile

@@ -46,10 +46,7 @@ MBEDTLS_URI ?= https://github.com/ARMmbed/mbedtls/archive/
 MBEDTLS_CHECKSUM ?= ec72ecf39275327f52b5ee9787271313a0d2960e7342b488d223a118ba164caa
 
 crypto/$(MBEDTLS_SRC):
-	wget --timeout=10 $(MBEDTLS_URI)/$(MBEDTLS_SRC) -O tmp
-	@[ "`sha256sum tmp`" = "$(MBEDTLS_CHECKSUM)  tmp" ] || \
-		(echo "*** $@ has a wrong checksum ***"; rm -f tmp; exit 255)
-	mv -f tmp $@
+	../../Scripts/download --output $@ --url $(MBEDTLS_URI)/$(MBEDTLS_SRC) --sha256 $(MBEDTLS_CHECKSUM)
 
 ifeq ($(DEBUG),1)
 MBED_BUILD_TYPE=Debug

+ 2 - 1
Pal/src/host/Linux-SGX/Makefile

@@ -2,6 +2,7 @@ include ../../../../Makefile.configs
 include Makefile.am
 
 ias_cert_url ?= https://certificates.trustedservices.intel.com/Intel_SGX_Attestation_RootCA.pem
+ias_cert_sha256 ?= e7b9113b647bc6bd421d4f140076a3acc31e410e01bf12883841e824cdaf1564
 ias_cert_file = quote/$(notdir $(ias_cert_url))
 
 CFLAGS	+= -I. -Iinclude -I../.. -I../../../include -I../../../lib -I../../../lib/crypto/mbedtls/include -Isgx-driver
@@ -76,7 +77,7 @@ quote/generated-cacert.h: $(ias_cert_file)
 	@echo "#define IAS_CA_CERT \"$(shell cat $< | tr -d '[\r\n]')\"" > $@
 
 $(ias_cert_file):
-	@wget $(ias_cert_url) -O $@
+	@../../../../Scripts/download --output $@ --url $(ias_cert_url) --sha256 $(ias_cert_sha256)
 
 debugger/sgx_gdb.so: debugger/sgx_gdb.c debugger/sgx_gdb.h sgx_arch.h
 	@echo [ host/Linux-SGX/$@ ]

+ 78 - 0
Scripts/download

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+declare -a urls
+
+usage() {
+    echo "Usage: download --url https://example.org/test --url https://mirror.example/blah --output test.tar --sha256 1234..abc"
+    exit "${1:-0}"
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --url)
+            urls+=("$2")
+            shift
+            ;;
+        --output)
+            output="$2"
+            shift
+            ;;
+        --sha256)
+            sha256="$2"
+            shift
+            ;;
+        --help|-h)
+            usage
+            ;;
+        *)
+            usage 1
+            ;;
+    esac
+    shift
+done
+
+if [ -z "${output:-}" ] || [ -z "${sha256:-}" ] || [ -z "${urls[*]}" ]; then
+    usage 1
+fi
+
+if [ -n "${DL_CACHE:-}" ]; then
+    if [ -f "$DL_CACHE/$sha256" ]; then
+        echo "download: Found '$output' (${sha256:0:8}...) in cache."
+        cp "$DL_CACHE/$sha256" "$output"
+        exit 0
+    fi
+fi
+
+if [ "${DL_OFFLINE:-}" == "true" ]; then
+    echo "download: ERROR: File '$output' (${sha256:0:8}...) not found in cache and offline mode is active!"
+    exit 1
+fi
+
+tmpd="$(mktemp -d -p "${DL_CACHE:-.}")"
+cleanup() {
+    rm -rf "$tmpd"
+}
+trap cleanup EXIT
+
+for url in "${urls[@]}"; do
+    echo "download: Trying to fetch $url"
+    wget --timeout=10 -O "$tmpd/unverified" "$url" || true
+    sha256_received="$(sha256sum "$tmpd/unverified" | cut -d ' ' -f 1)"
+    if [ "$sha256" != "$sha256_received" ]; then
+        echo "download: WARNING: Hash mismatch: Expected $sha256 but received $sha256_received"
+        continue
+    fi
+    echo "download: Fetched '$output' (${sha256:0:8}...) successfully."
+    if [ -n "${DL_CACHE:-}" ]; then
+        mv "$tmpd/unverified" "$DL_CACHE/$sha256"
+        cp "$DL_CACHE/$sha256" "$output"
+        exit 0
+    fi
+    mv "$tmpd/unverified" "$output"
+    exit 0
+done
+
+echo "download: ERROR: Failed to download '$output' (${sha256:0:8}...)! No URLs left to try."
+exit 1