Use cabal-cache for caching

haskell · Nov 25, 2022 · 3847328 · 3847328
1 parent 432e68c
commit 3847328
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 18 deletions.
diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -13,15 +13,18 @@ fi
 
 if [ "${RUNNER_OS}" != "FreeBSD" ] ; then
 	if [ "${DISTRO}" != "Debian" ] ; then # ! armv7 or aarch64 linux
-		ghcup install ghc --set --isolate="$HOME/.local"     --force "$GHC_VER"
-		ghcup install cabal     --isolate="$HOME/.local/bin" --force "$CABAL_VER"
+		ghcup -v install ghc       --isolate="$HOME/.local"     --force 8.10.7
+		ghcup -v install ghc --set --isolate="$HOME/.local"     --force "$GHC_VER"
+		ghcup -v install cabal     --isolate="$HOME/.local/bin" --force "$CABAL_VER"
 		ghc --version
 		cabal --version
 		GHC="ghc-${GHC_VER}"
+		GHC_CABAL_CACHE="ghc-8.10.7"
 	else
-		ghcup install cabal --isolate="$HOME/.local/bin" --force "$CABAL_VER"
+		ghcup -v install cabal --isolate="$HOME/.local/bin" --force "$CABAL_VER"
 		cabal --version
 		GHC="ghc"
+		GHC_CABAL_CACHE="ghc"
 	fi
 else
 	GHC="ghc"
@@ -33,27 +36,64 @@ ecabal() {
 	cabal "$@"
 }
 
+sync_from() {
+	cabal-cache sync-from-archive \
+		--host-name-override=s3.us-west-004.backblazeb2.com \
+		--host-port-override=443 \
+		--host-ssl-override=True \
+		--region us-west-1 \
+		--archive-uri s3://ghcup-hs
+}
+
+sync_to() {
+	cabal-cache sync-to-archive \
+		--host-name-override=s3.us-west-004.backblazeb2.com \
+		--host-port-override=443 \
+		--host-ssl-override=True \
+		--region us-west-1 \
+		--archive-uri s3://ghcup-hs
+}
+
+build_with_cache() {
+	ecabal configure "$@"
+	sync_from
+	ecabal build "$@"
+	sync_to
+}
+
 # build
 ecabal update
 
 
+# make sure cabal-cache is available
+if ! command -v cabal-cache ; then
+	(
+	cd /tmp
+	ecabal install -w "${GHC_CABAL_CACHE}" --overwrite-policy=always --install-method=copy --installdir="$HOME/.local/bin" cabal-cache
+	mkdir -p out
+	cp "$HOME/.local/bin/cabal-cache" "out/cabal-cache-${RUNNER_OS}-${DISTRO}-${ARCH}"
+	)
+fi
+
+
 if [ "${RUNNER_OS}" = "Linux" ] ; then
 	if [ "${ARCH}" = "32" ] ; then
-		ecabal build -w "${GHC}" --ghc-options='-split-sections -optl-static' -ftui
+		build_with_cache -w "${GHC}" --ghc-options='-split-sections -optl-static' -ftui
 	elif [ "${ARCH}" = "64" ] ; then
-		ecabal build -w "${GHC}" --ghc-options='-split-sections -optl-static' -ftui
+		build_with_cache -w "${GHC}" --ghc-options='-split-sections -optl-static' -ftui
 	else
-		ecabal build -w "${GHC}" -ftui
+		build_with_cache -w "${GHC}" -ftui
 	fi
 elif [ "${RUNNER_OS}" = "FreeBSD" ] ; then
-	ecabal build -w "${GHC}" --ghc-options='-split-sections' --constraint="zlib +bundled-c-zlib" --constraint="zip +disable-zstd" -ftui
+	build_with_cache -w "${GHC}" --ghc-options='-split-sections' --constraint="zlib +bundled-c-zlib" --constraint="zip +disable-zstd" -ftui
 elif [ "${RUNNER_OS}" = "Windows" ] ; then
-	ecabal build -w "${GHC}" --constraint="zlib +bundled-c-zlib" --constraint="lzma +static"
+	build_with_cache -w "${GHC}" --constraint="zlib +bundled-c-zlib" --constraint="lzma +static"
 else
-	ecabal build -w "${GHC}" --constraint="zlib +bundled-c-zlib" --constraint="lzma +static" -ftui
+	build_with_cache -w "${GHC}" --constraint="zlib +bundled-c-zlib" --constraint="lzma +static" -ftui
 fi
 
-mkdir out
+
+mkdir -p out
 binary=$(ecabal new-exec -w "${GHC}" --verbose=0 --offline sh -- -c 'command -v ghcup')
 ver=$("${binary}" --numeric-version)
 if [ "${RUNNER_OS}" = "macOS" ] ; then

diff --git a/.github/scripts/prereq.sh b/.github/scripts/prereq.sh
@@ -1,6 +1,16 @@
 #!/bin/sh
 
-mkdir -p "$HOME"/.local/bin
+if [ "${RUNNER_OS}" = "macOS" ] ; then
+	mkdir -p "$HOME"/.local/bin
+	ls -lah "$HOME"/.local/bin
+	find "$HOME"/.local
+	rm -rf "$HOME"/.local/bin
+	rm -rf "$HOME"/.local/share
+	rm -rf "$HOME"/.local/lib
+	mkdir -p "$HOME"/.local/bin
+else
+	mkdir -p "$HOME"/.local/bin
+fi
 
 export OS="$RUNNER_OS"
 export PATH="$HOME/.local/bin:$PATH"
@@ -39,7 +49,8 @@ if [ "${RUNNER_OS}" = "Linux" ] ; then
             perl \
             bash \
             diffutils \
-            git
+            git \
+			gzip
 
           apk add --no-cache \
             zlib \
@@ -57,10 +68,24 @@ if [ "${RUNNER_OS}" = "Linux" ] ; then
             ncurses-static
 	elif [ "${DISTRO}" = "Ubuntu" ] ; then
 		sudo apt-get update -y
-		sudo apt-get install -y libnuma-dev zlib1g-dev libgmp-dev libgmp10 libssl-dev liblzma-dev libbz2-dev git wget lsb-release software-properties-common gnupg2 apt-transport-https gcc autoconf automake build-essential curl
+		sudo apt-get install -y libnuma-dev zlib1g-dev libgmp-dev libgmp10 libssl-dev liblzma-dev libbz2-dev git wget lsb-release software-properties-common gnupg2 apt-transport-https gcc autoconf automake build-essential curl gzip
+		if [ "${ARCH}" = "64" ] ; then
+			curl -O -L https://github.com/haskell-works/cabal-cache/releases/download/v1.0.4.0/cabal-cache-x86_64-linux.gz
+			gunzip cabal-cache-x86_64-linux.gz
+			mv cabal-cache-x86_64-linux "$HOME"/.local/bin/cabal-cache
+			chmod +x "$HOME"/.local/bin/cabal-cache
+			"$HOME"/.local/bin/cabal-cache --help
+		fi
 	elif [ "${DISTRO}" = "Debian" ] ; then
 		apt-get update -y
-		apt-get install -y libnuma-dev zlib1g-dev libgmp-dev libgmp10 libssl-dev liblzma-dev libbz2-dev git wget lsb-release software-properties-common gnupg2 apt-transport-https gcc autoconf automake build-essential curl ghc
+		apt-get install -y libnuma-dev zlib1g-dev libgmp-dev libgmp10 libssl-dev liblzma-dev libbz2-dev git wget lsb-release software-properties-common gnupg2 apt-transport-https gcc autoconf automake build-essential curl ghc gzip
+		if [ "${ARCH}" = "64" ] ; then
+			curl -O -L https://github.com/haskell-works/cabal-cache/releases/download/v1.0.4.0/cabal-cache-x86_64-linux.gz
+			gunzip cabal-cache-x86_64-linux.gz
+			mv cabal-cache-x86_64-linux "$HOME"/.local/bin/cabal-cache
+			chmod +x "$HOME"/.local/bin/cabal-cache
+			"$HOME"/.local/bin/cabal-cache --help
+		fi
 	fi
 elif [ "${RUNNER_OS}" = "macOS" ] ; then
 	if ! command -v brew ; then
@@ -75,5 +100,29 @@ elif [ "${RUNNER_OS}" = "macOS" ] ; then
 	if ! command -v realpath ; then
 		brew install coreutils
 	fi
+
+    if [ "${ARCH}" = "ARM64" ] ; then
+		brew install llvm autoconf automake
+		export PATH="$HOME/.brew/opt/llvm/bin:$PATH"
+		export CC=$HOME/.brew/opt/llvm/bin/clang
+		export CXX=$HOME/.brew/opt/llvm/bin/clang++
+		export LD=ld
+		export AR=$HOME/.brew/opt/llvm/bin/llvm-ar
+		export RANLIB=$HOME/.brew/opt/llvm/bin/llvm-ranlib
+    elif [ "${ARCH}" = "64" ] ; then
+		curl -O -L https://github.com/haskell-works/cabal-cache/releases/download/v1.0.4.0/cabal-cache-x86_64-darwin.gz
+		gunzip cabal-cache-x86_64-darwin.gz
+		mv cabal-cache-x86_64-darwin "$HOME"/.local/bin/cabal-cache
+		chmod +x "$HOME"/.local/bin/cabal-cache
+		"$HOME"/.local/bin/cabal-cache --help
+	fi
+elif [ "${RUNNER_OS}" = "Windows" ] ; then
+    if [ "${ARCH}" = "64" ] ; then
+		curl -O -L https://github.com/haskell-works/cabal-cache/releases/download/v1.0.4.0/cabal-cache-x86_64-windows.gz
+		gunzip cabal-cache-x86_64-windows.gz
+		mv cabal-cache-x86_64-windows "$HOME"/.local/bin/cabal-cache
+		chmod +x "$HOME"/.local/bin/cabal-cache
+		"$HOME"/.local/bin/cabal-cache --help
+	fi
 fi
 
diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
@@ -18,6 +18,7 @@ jobs:
       BOOTSTRAP_HASKELL_CABAL_VERSION: 3.6.2.0
       BOOTSTRAP_HASKELL_GHC_VERSION: 8.10.7
       BOOTSTRAP_HASKELL_NONINTERACTIVE: yes
+      ARCH: 64
       JSON_VERSION: "0.0.7"
     strategy:
       matrix:

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -19,6 +19,8 @@ jobs:
       CACHE_VER: 1
       MACOSX_DEPLOYMENT_TARGET: 10.13
       JSON_VERSION: "0.0.7"
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
     strategy:
       matrix:
         include:
@@ -44,11 +46,11 @@ jobs:
             ARCH: 64
           - os: ubuntu-latest
             ARTIFACT: "armv7-linux-ghcup"
-            GHC_VER: 8.10.7
+            GHC_VER: 8.8.4
             ARCH: ARM
           - os: ubuntu-latest
             ARTIFACT: "aarch64-linux-ghcup"
-            GHC_VER: 8.10.7
+            GHC_VER: 8.8.4
             ARCH: ARM64
     steps:
       - name: Checkout code
@@ -92,6 +94,8 @@ jobs:
             DISTRO: Debian
             CABAL_VER: ${{ env.CABAL_VER }}
             JSON_VERSION: ${{ env.JSON_VERSION }}
+            AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+            AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
 
       - if: matrix.ARCH == 'ARM64' && runner.os == 'Linux'
         uses: uraimo/run-on-arch-action@v2
@@ -107,6 +111,9 @@ jobs:
             DISTRO: Debian
             CABAL_VER: ${{ env.CABAL_VER }}
             JSON_VERSION: ${{ env.JSON_VERSION }}
+            AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+            AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+            HOMEBREW_CHANGE_ARCH_TO_ARM: 1
 
       - if: runner.os != 'Linux'
         name: Run build (windows/mac)
@@ -169,11 +176,11 @@ jobs:
             DISTRO: na
           - os: ubuntu-latest
             ARTIFACT: "armv7-linux-ghcup"
-            GHC_VER: 8.10.7
+            GHC_VER: 8.8.4
             ARCH: ARM
           - os: ubuntu-latest
             ARTIFACT: "aarch64-linux-ghcup"
-            GHC_VER: 8.10.7
+            GHC_VER: 8.8.4
             ARCH: ARM64
     steps:
       - name: Checkout code
@@ -246,6 +253,7 @@ jobs:
             DISTRO: Debian
             CABAL_VER: ${{ env.CABAL_VER }}
             JSON_VERSION: ${{ env.JSON_VERSION }}
+            HOMEBREW_CHANGE_ARCH_TO_ARM: 1
 
       - if: runner.os != 'Linux'
         name: Run test (windows/mac)