Even more tools with download cache

uniget-org · Sep 12, 2024 · 145bc1b · 145bc1b
1 parent 9c986ba
commit 145bc1b
Show file tree

Hide file tree

Showing 35 changed files with 320 additions and 144 deletions.
diff --git a/scripts/rewrite-add-download-cache.sh b/scripts/rewrite-add-download-cache.sh
@@ -20,11 +20,6 @@ find tools -type f -name Dockerfile.template \
         continue
     fi
 
-    if grep -q "cosign verify-blob " "${FILENAME}"; then
-        echo "Missing handler ${FILENAME}"
-        continue
-    fi
-
     if grep -q "^unzip -q " "${FILENAME}"; then
         sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?url="([^"]+)"\nfilename=".+"\ncurl --silent --show-error --location --fail --remote-name "([^"]+)"\nunzip -q -o -d "([^"]+)" "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\nunzip -t "${uniget_cache_download}/${filename}"\nunzip -q -o -d "\6" "${uniget_cache_download}/${filename}"@' "${FILENAME}"
         continue

diff --git a/tools/autocert/Dockerfile.template b/tools/autocert/Dockerfile.template
@@ -11,33 +11,36 @@ COPY --link --from=cosign / /usr/local/
 ARG name
 ARG version
 RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
-check-github-release-asset "smallstep/autocert" "v${version}" "autocert_linux_${version}_amd64.tar.gz"
+url="${url}"
+filename="$(basename "${url}")"
+
+check-github-release-asset "smallstep/autocert" "v${version}" "${filename}"
 echo "### Downloading autocert ${version}"
-curl --silent --show-error --location --fail --remote-name \
-    "https://github.com/smallstep/autocert/releases/download/v${version}/autocert_linux_${version}_amd64.tar.gz"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
 
 echo "### Checking keyless signature for autocert ${version}"
-curl --silent --show-error --location --fail --remote-name \
-    "https://github.com/smallstep/autocert/releases/download/v${version}/autocert_linux_${version}_amd64.tar.gz.pem"
-curl --silent --show-error --location --fail --remote-name \
-    "https://github.com/smallstep/autocert/releases/download/v${version}/autocert_linux_${version}_amd64.tar.gz.sig"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}.pem" \
+    "${url}.pem"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}.sig" \
+    "${url}.sig"
 SHA="$(
     git ls-remote --tags https://github.com/smallstep/autocert "v${version}" \
     | cut -f1
 )"
 COSIGN_EXPERIMENTAL=1 cosign verify-blob \
-    --cert "autocert_linux_${version}_amd64.tar.gz.pem" \
-    --signature "autocert_linux_${version}_amd64.tar.gz.sig" \
+    --cert "${uniget_cache_download}/${filename}.pem" \
+    --signature "${uniget_cache_download}/${filename}.sig" \
     --certificate-oidc-issuer https://token.actions.githubusercontent.com \
     --certificate-identity "https://github.com/smallstep/workflows/.github/workflows/goreleaser.yml@refs/heads/main" \
     --certificate-github-workflow-name "Create Release & Upload Assets" \
     --certificate-github-workflow-ref "refs/tags/v${version}" \
     --certificate-github-workflow-repository "smallstep/autocert" \
     --certificate-github-workflow-sha "${SHA}" \
     --certificate-github-workflow-trigger push \
-    "autocert_linux_${version}_amd64.tar.gz"
+    "${uniget_cache_download}/${filename}"
 
-tar --extract --gzip --directory="${prefix}/bin/" --file="autocert_linux_${version}_amd64.tar.gz" --strip-components=1 --no-same-owner \
+tar --file="autocert_linux_${version}_amd64.tar.gz" --list
+tar --file="autocert_linux_${version}_amd64.tar.gz" --extract --gzip --directory="${prefix}/bin/" --strip-components=1 --no-same-owner \
     "autocert_${version}/autocert"
-#rm "autocert_linux_${version}_amd64.tar.gz"
 EOF
diff --git a/tools/cfssl/Dockerfile.template b/tools/cfssl/Dockerfile.template
@@ -8,11 +8,15 @@ SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
 ARG BINARIES="cfssl-bundle cfssl-certinfo cfssl-newkey cfssl-scan cfssljson cfssl mkbundle multirootca"
-RUN <<EOF # download cache
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 for BINARY in ${BINARIES}; do
-    check-github-release-asset "cloudflare/cfssl" "v${version}" "${BINARY}_${version}_linux_amd64"
-    curl --silent --show-error --location --fail --output "${prefix}/bin/${BINARY}" \
-        "https://github.com/cloudflare/cfssl/releases/download/v${version}/${BINARY}_${version}_linux_amd64"
-    chmod +x "${prefix}/bin/${BINARY}"
+    url="https://github.com/cloudflare/cfssl/releases/download/v${version}/${BINARY}_${version}_linux_amd64"
+    filename="$(basename "${url}")"
+    check-github-release-asset "cloudflare/cfssl" "v${version}" "${filename}"
+    curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+        "${url}"
+    install --mode=0755 \
+        "${uniget_cache_download}/${filename}" \
+        "${prefix}/bin/${BINARY}"
 done
 EOF
diff --git a/tools/dinit/Dockerfile.template b/tools/dinit/Dockerfile.template
@@ -7,8 +7,13 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail --output "${prefix}/bin/dinit" \
-    "https://gitlab.com/tozd/dinit/-/releases/v${version}/downloads/linux-${alt_arch}/dinit"
-chmod +x "${prefix}/bin/dinit"
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://gitlab.com/tozd/dinit/-/releases/v${version}/downloads/linux-${alt_arch}/dinit"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+install --mode=0755 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/bin/dinit"
 EOF
diff --git a/tools/distrobox/Dockerfile.template b/tools/distrobox/Dockerfile.template
@@ -8,9 +8,16 @@ SHELL [ "bash", "-clo", "errexit" ]
 WORKDIR /tmp/distrobox
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail "https://github.com/89luca89/distrobox/archive/refs/tags/${version}.tar.gz" \
-| tar --extract --gzip --strip-components=1 --no-same-owner
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://github.com/89luca89/distrobox/archive/refs/tags/${version}.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --strip-components=1 --no-same-owner
+
 ./install --prefix="${prefix}"
 mv "${prefix}/share/zsh/site-functions/" "${prefix}/share/zsh/vendor-completions/"
 EOF
diff --git a/tools/docker-credential-ecr-login/Dockerfile.template b/tools/docker-credential-ecr-login/Dockerfile.template
@@ -5,12 +5,16 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
-
 ARG name
 ARG version
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${version}/linux-${alt_arch}/docker-credential-ecr-login"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
 
-RUN <<EOF
-curl --silent --show-error --location --fail --output "${prefix}/bin/docker-credential-ecr-login" \
-    "https://amazon-ecr-credential-helper-releases.s3.us-east-2.amazonaws.com/${version}/linux-${alt_arch}/docker-credential-ecr-login"
-chmod +x "${prefix}/bin/docker-credential-ecr-login"
+install --mode=0755 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/bin/docker-credential-ecr-login"
 EOF
diff --git a/tools/falco/Dockerfile.template b/tools/falco/Dockerfile.template
@@ -7,9 +7,15 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail "https://download.falco.org/packages/bin/${arch}/falco-${version}-${arch}.tar.gz" \
-| tar --extract --gzip --directory="${prefix}" --strip-components=1 --no-same-owner
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://download.falco.org/packages/bin/${arch}/falco-${version}-${arch}.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}" --strip-components=1 --no-same-owner
 
 mv "${prefix}/usr/bin/" "${prefix}/"
 mv "${prefix}/usr/src/" "${prefix}/"

diff --git a/tools/forgejo/Dockerfile.template b/tools/forgejo/Dockerfile.template
@@ -7,7 +7,7 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 url="$(
     curl --silent --show-error --location --fail "https://codeberg.org/api/v1/repos/forgejo/forgejo/releases" \
     | jq --raw-output --arg version "${version}" --arg alt_arch "${alt_arch}" '
@@ -19,8 +19,12 @@ url="$(
     '
 )"
 test -n "${url}"
+filename="$(basename "${url}")"
+
 check-download "${url}"
-curl --silent --show-error --location --fail "${url}" \
-| xz --decompress >"${prefix}/bin/forgejo"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+xz --decompress "${uniget_cache_download}/${filename}" >"${prefix}/bin/forgejo"
 chmod +x "${prefix}/bin/forgejo"
 EOF
diff --git a/tools/gitlab-runner/Dockerfile.template b/tools/gitlab-runner/Dockerfile.template
@@ -9,8 +9,14 @@ ARG name
 ARG version
 COPY gitlab-runner.service.go-template /uniget_bootstrap/etc/systemd/system/
 COPY config.toml /uniget_bootstrap/etc/gitlab-runner/
-RUN <<EOF
-curl --silent --show-error --location --fail --output "${prefix}/bin/gitlab-runner" \
-    "https://gitlab-runner-downloads.s3.amazonaws.com/v${version}/binaries/gitlab-runner-linux-${alt_arch}"
-chmod +x "${prefix}/bin/gitlab-runner"
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://gitlab-runner-downloads.s3.amazonaws.com/v${version}/binaries/gitlab-runner-linux-${alt_arch}"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+install --mode=0755 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/bin/gitlab-runner"
 EOF
diff --git a/tools/glab/Dockerfile.template b/tools/glab/Dockerfile.template
@@ -5,14 +5,19 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
-
 ARG name
 ARG version
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://gitlab.com/gitlab-org/cli/-/releases/v${version}/downloads/glab_${version}_Linux_x86_64.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
 
-RUN <<EOF
-curl --silent --show-error --location --fail "https://gitlab.com/gitlab-org/cli/-/releases/v${version}/downloads/glab_${version}_Linux_x86_64.tar.gz" \
-| tar --extract --gzip --directory="${prefix}/bin" --strip-components=1 --no-same-owner \
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --strip-components=1 --no-same-owner \
     bin/glab
+
 "${prefix}/bin/glab" completion -s bash >"${prefix}/share/bash-completion/completions/glab"
 "${prefix}/bin/glab" completion -s fish >"${prefix}/share/fish/vendor_completions.d/glab.fish"
 "${prefix}/bin/glab" completion -s zsh >"${prefix}/share/zsh/vendor-completions/_glab"

diff --git a/tools/helm/Dockerfile.template b/tools/helm/Dockerfile.template
@@ -7,10 +7,17 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail "https://get.helm.sh/helm-v${version}-linux-${alt_arch}.tar.gz" \
-| tar --extract --gzip --directory="${prefix}/bin" --strip-components=1 --no-same-owner \
-    linux-${alt_arch}/helm
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://get.helm.sh/helm-v${version}-linux-${alt_arch}.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --strip-components=1 --no-same-owner \
+    "linux-${alt_arch}/helm"
+
 "${prefix}/bin/helm" completion bash >"${prefix}/share/bash-completion/completions/helm"
 "${prefix}/bin/helm" completion fish >"${prefix}/share/fish/vendor_completions.d/helm.fish"
 "${prefix}/bin/helm" completion zsh >"${prefix}/share/zsh/vendor-completions/_helm"

diff --git a/tools/influx-cli/Dockerfile.template b/tools/influx-cli/Dockerfile.template
@@ -7,10 +7,17 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail "https://dl.influxdata.com/influxdb/releases/influxdb2-client-${version}-linux-${alt_arch}.tar.gz" \
-| tar --extract --gzip --directory="${prefix}/bin/" --strip-component=1 --no-same-owner \
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://dl.influxdata.com/influxdb/releases/influxdb2-client-${version}-linux-${alt_arch}.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin/" --strip-component=1 --no-same-owner \
     "./influx"
+
 "${prefix}/bin/influx" completion bash >"${prefix}/share/bash-completion/completions/influx"
 "${prefix}/bin/influx" completion zsh >"${prefix}/share/zsh/vendor-completions/_influx"
 EOF
diff --git a/tools/jenkins-remoting/Dockerfile.template b/tools/jenkins-remoting/Dockerfile.template
@@ -5,10 +5,17 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
+WORKDIR /uniget_bootstrap/opt/jenkins
 ARG name
 ARG version
-RUN <<EOF
-mkdir -p "${prefix}/opt/jenkins"
-curl --silent --show-error --location --fail --output "${prefix}/opt/jenkins/remoting.jar" \
-    "https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${version}/remoting-${version}.jar"
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${version}/remoting-${version}.jar"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+install --mode=0644 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/opt/jenkins/${filename}"
 EOF
diff --git a/tools/jf/Dockerfile.template b/tools/jf/Dockerfile.template
@@ -5,14 +5,19 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
-
 ARG name
 ARG version
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/${version}/jfrog-cli-linux-${alt_arch}/jf"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+install --mode=0755 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/bin/jf"
 
-RUN <<EOF
-curl --silent --show-error --location --fail --output "${prefix}/bin/jf" \
-    "https://releases.jfrog.io/artifactory/jfrog-cli/v2-jf/${version}/jfrog-cli-linux-${alt_arch}/jf"
-chmod +x "${prefix}/bin/jf"
 "${prefix}/bin/jf" completion bash >"${prefix}/share/bash-completion/completions/jf"
 "${prefix}/bin/jf" completion fish >"${prefix}/share/fish/vendor_completions.d/jf.fish"
 "${prefix}/bin/jf" completion zsh >"${prefix}/share/zsh/vendor-completions/_jf"

diff --git a/tools/kubectl-plugins/Dockerfile.template b/tools/kubectl-plugins/Dockerfile.template
@@ -27,7 +27,7 @@ EOF
 COPY <<EOF /uniget_bootstrap/etc/profile.d/krew.sh
 export PATH="/libexec/krew/bin:${PATH}"
 EOF
-RUN <<EOF
+RUN <<EOF # no download cache
 export KREW_ROOT="${prefix}/libexec/krew"
 mkdir -p "${KREW_ROOT}"
 krew update

diff --git a/tools/lectl/Dockerfile.template b/tools/lectl/Dockerfile.template
@@ -5,12 +5,16 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
-
 ARG name
 ARG version
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://github.com/sahsanu/lectl/raw/v${version}/lectl"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
 
-RUN <<EOF
-curl --silent --show-error --location --fail --output "${prefix}/bin/lectl" \
-    "https://github.com/sahsanu/lectl/raw/v${version}/lectl"
-chmod +x "${prefix}/bin/lectl"
+install --mode=0755 \
+    "${uniget_cache_download}/${filename}" \
+    "${prefix}/bin/lectl"
 EOF
diff --git a/tools/make/Dockerfile.template b/tools/make/Dockerfile.template
@@ -8,9 +8,16 @@ SHELL [ "bash", "-clo", "errexit" ]
 WORKDIR /tmp/make
 ARG name
 ARG version
-RUN <<EOF
-curl --silent --show-error --location --fail "https://ftp.gnu.org/gnu/make/make-${version}.tar.gz" \
-| tar --extract --gzip --strip-components=1
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://ftp.gnu.org/gnu/make/make-${version}.tar.gz"
+filename="$(basename "${url}")"
+
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --strip-components=1
+
 ./configure --prefix="${prefix}"
 make -j$(nproc) LDFLAGS=-static
 make install