From 49e572dd090fbe7642255056e3399ca37e1b649d Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 12:05:59 +0000 Subject: [PATCH 01/57] Add initial buildWolfiDockerImage implementation --- .../dev/ci/internal/ci/wolfi_operations.go | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index b478a12e575ad..63984d72362f4 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -2,9 +2,13 @@ package ci import ( "fmt" + "os" + "path/filepath" + "strings" "github.com/sourcegraph/log" + "github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images" bk "github.com/sourcegraph/sourcegraph/enterprise/dev/ci/internal/buildkite" "github.com/sourcegraph/sourcegraph/enterprise/dev/ci/internal/ci/operations" "github.com/sourcegraph/sourcegraph/internal/lazyregexp" @@ -101,3 +105,85 @@ func buildWolfi(target string, tag string, dependOnPackages bool) func(*bk.Pipel ) } } + +// Build a candidate Wolfi docker image +func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps bool) operations.Operation { + return func(pipeline *bk.Pipeline) { + image := strings.ReplaceAll(app, "/", "-") + localImage := "sourcegraph/wolfi-" + image + ":" + version + + cmds := []bk.StepOpt{ + bk.Key(candidateImageStepKey(app)), + bk.Cmd(fmt.Sprintf(`echo "Building Wolfi %s image..."`, app)), + bk.Env("DOCKER_BUILDKIT", "1"), + bk.Env("IMAGE", localImage), + bk.Env("VERSION", version), + } + + // Add Sentry environment variables if we are building off main branch + // to enable building the webapp with source maps enabled + if uploadSourcemaps { + cmds = append(cmds, + bk.Env("SENTRY_UPLOAD_SOURCE_MAPS", "1"), + bk.Env("SENTRY_ORGANIZATION", "sourcegraph"), + bk.Env("SENTRY_PROJECT", "sourcegraph-dot-com"), + ) + } + + // Allow all build scripts to emit info annotations + buildAnnotationOptions := bk.AnnotatedCmdOpts{ + Annotations: &bk.AnnotationOpts{ + Type: bk.AnnotationTypeInfo, + IncludeNames: true, + }, + } + + if _, err := os.Stat(filepath.Join("docker-images", app)); err == nil { + // Building Docker image located under $REPO_ROOT/docker-images/ + cmds = append(cmds, + bk.Cmd("ls -lah "+filepath.Join("docker-images", app, "build-wolfi.sh")), + bk.Cmd(filepath.Join("docker-images", app, "build-wolfi.sh"))) + } else { + // Building Docker images located under $REPO_ROOT/cmd/ + cmdDir := func() string { + folder := app + if app == "blobstore2" { + // experiment: cmd/blobstore is a Go rewrite of docker-images/blobstore. While + // it is incomplete, we do not want cmd/blobstore/Dockerfile to get publishe + // under the same name. + // https://github.com/sourcegraph/sourcegraph/issues/45594 + // TODO(blobstore): remove this when making Go blobstore the default + folder = "blobstore" + } + // If /enterprise/cmd/... does not exist, build just /cmd/... instead. + if _, err := os.Stat(filepath.Join("enterprise/cmd", folder)); err != nil { + return "cmd/" + folder + } + return "enterprise/cmd/" + folder + }() + preBuildScript := cmdDir + "/pre-build.sh" + if _, err := os.Stat(preBuildScript); err == nil { + // Allow all + cmds = append(cmds, bk.AnnotatedCmd(preBuildScript, buildAnnotationOptions)) + } + cmds = append(cmds, bk.AnnotatedCmd(cmdDir+"/build-wolfi.sh", buildAnnotationOptions)) + } + + // Add "wolfi" to image name so we don't overwrite Alpine dev images + wolfiApp := fmt.Sprintf("wolfi-%s", app) + devImage := images.DevRegistryImage(wolfiApp, tag) + cmds = append(cmds, + // Retag the local image for dev registry + bk.Cmd(fmt.Sprintf("docker tag %s %s", localImage, devImage)), + // Publish tagged image + // TODO: Re-enable when we're happy this is building as expected + // bk.Cmd(fmt.Sprintf("docker push %s || exit 10", devImage)), + // Retry in case of flakes when pushing + bk.AutomaticRetryStatus(3, 10), + // Retry in case of flakes when pushing + bk.AutomaticRetryStatus(3, 222), + ) + + pipeline.AddStep(fmt.Sprintf(":octopus: :docker: :construction: Build %s", app), cmds...) + } +} From 1874e050f6fb2dd3a29e0fd259d507a64c244f05 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 12:25:18 +0000 Subject: [PATCH 02/57] Add ops set for building wolfi images --- .../dev/ci/internal/ci/wolfi_operations.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 63984d72362f4..363572b15f1bd 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -56,6 +56,24 @@ func WolfiPackagesOperations(changedFiles []string) *operations.Set { return ops } +// BuildWolfiOperations builds the specified docker images, or all images if none are provided +func BuildWolfiOperations(buildImages []string, version string, tag string) *operations.Set { + // If buildImages is not specified, rebuild all images + // TODO: Maintain a list of Wolfi-based images? + if len(buildImages) == 0 { + buildImages = images.SourcegraphDockerImages + } + + wolfiImageBuildOps := operations.NewNamedSet("Wolfi image builds") + + for _, dockerImage := range buildImages { + // Don't upload sourcemaps + wolfiImageBuildOps.Append(buildCandidateDockerImage(dockerImage, version, tag, false)) + } + + return wolfiImageBuildOps +} + // Dependency tree between steps: // (buildPackage[1], buildPackage[2], ...) <-- buildRepoIndex <-- (buildWolfi[1], buildWolfi[2], ...) From 77a72755614004b8dd0818c6046193dc8cc02317 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 12:25:38 +0000 Subject: [PATCH 03/57] Build wolfi images on branches and tweak package/base logic --- enterprise/dev/ci/internal/ci/pipeline.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index b632809ecb139..acc69f23608f1 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -101,18 +101,28 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // false means not optional, so this build will fail if Bazel build doesn't pass. ops.Merge(BazelOperations(false)) case runtype.WolfiExpBranch: - if c.Diff.Has(changed.WolfiPackages) { + // Rebuild packages if package configs have changed + updatePackages := c.Diff.Has(changed.WolfiPackages) + // Rebuild base images if base image OR package configs have changed + updateBaseImages := c.Diff.Has(changed.WolfiBaseImages) || updatePackages + + if updatePackages { ops.Merge(WolfiPackagesOperations(c.ChangedFiles[changed.WolfiPackages])) } - if c.Diff.Has(changed.WolfiBaseImages) { + if updateBaseImages { ops.Merge( WolfiBaseImagesOperations( - c.ChangedFiles[changed.WolfiBaseImages], + c.ChangedFiles[changed.WolfiBaseImages], // TODO: If packages have changed need to update all base images. Requires a list of all base images c.Version, - c.Diff.Has(changed.WolfiPackages), + updatePackages, ), ) } + // Always rebuild Wolfi images + ops.Merge( + // TODO: Just hardcode a single image initially + BuildWolfiOperations([]string{"gitserver"}, c.Version, c.candidateImageTag()), + ) case runtype.PullRequest: // First, we set up core test operations that apply both to PRs and to other run From 7c088d9083d024ae28de4e1f642395ecfccbc1c3 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 12:41:28 +0000 Subject: [PATCH 04/57] Build candidate wolfi image --- enterprise/dev/ci/internal/ci/wolfi_operations.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 363572b15f1bd..6f88680dce14f 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -68,7 +68,8 @@ func BuildWolfiOperations(buildImages []string, version string, tag string) *ope for _, dockerImage := range buildImages { // Don't upload sourcemaps - wolfiImageBuildOps.Append(buildCandidateDockerImage(dockerImage, version, tag, false)) + // wolfiImageBuildOps.Append(buildCandidateDockerImage(dockerImage, version, tag, false)) + wolfiImageBuildOps.Append(buildCandidateWolfiDockerImage(dockerImage, version, tag, false)) } return wolfiImageBuildOps From d22c18a25fc881aeff99d4ad47fa372b6b348dcf Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:16:07 +0000 Subject: [PATCH 05/57] Add wolfi-specific build scripts for gitserver --- cmd/gitserver/Dockerfile.wolfi | 22 ++++++++++++++++++++++ cmd/gitserver/build-wolfi.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 cmd/gitserver/Dockerfile.wolfi create mode 100755 cmd/gitserver/build-wolfi.sh diff --git a/cmd/gitserver/Dockerfile.wolfi b/cmd/gitserver/Dockerfile.wolfi new file mode 100644 index 0000000000000..d6333c8c8f8b2 --- /dev/null +++ b/cmd/gitserver/Dockerfile.wolfi @@ -0,0 +1,22 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-gitserver-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +RUN mkdir -p /data/repos && chown -R sourcegraph:sourcegraph /data/repos +USER sourcegraph + +WORKDIR / + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/gitserver"] +COPY gitserver /usr/local/bin/ diff --git a/cmd/gitserver/build-wolfi.sh b/cmd/gitserver/build-wolfi.sh new file mode 100755 index 0000000000000..8b97141467989 --- /dev/null +++ b/cmd/gitserver/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) + +cleanup() { + rm -rf "$OUTPUT" +} + +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From 214675734448ea650ddb24dbd95af5cb41025ad2 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:20:23 +0000 Subject: [PATCH 06/57] Move Wolfi build files from cmd/ to enterprise/cmd --- {cmd => enterprise/cmd}/gitserver/Dockerfile.wolfi | 0 {cmd => enterprise/cmd}/gitserver/build-wolfi.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {cmd => enterprise/cmd}/gitserver/Dockerfile.wolfi (100%) rename {cmd => enterprise/cmd}/gitserver/build-wolfi.sh (100%) diff --git a/cmd/gitserver/Dockerfile.wolfi b/enterprise/cmd/gitserver/Dockerfile.wolfi similarity index 100% rename from cmd/gitserver/Dockerfile.wolfi rename to enterprise/cmd/gitserver/Dockerfile.wolfi diff --git a/cmd/gitserver/build-wolfi.sh b/enterprise/cmd/gitserver/build-wolfi.sh similarity index 100% rename from cmd/gitserver/build-wolfi.sh rename to enterprise/cmd/gitserver/build-wolfi.sh From b0012fe31d4219dfe7be8f8b4260ac9b837b888b Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:30:38 +0000 Subject: [PATCH 07/57] Re-enable wolfi docker push --- enterprise/dev/ci/internal/ci/wolfi_operations.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 6f88680dce14f..2581652e097d2 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -195,14 +195,13 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b // Retag the local image for dev registry bk.Cmd(fmt.Sprintf("docker tag %s %s", localImage, devImage)), // Publish tagged image - // TODO: Re-enable when we're happy this is building as expected - // bk.Cmd(fmt.Sprintf("docker push %s || exit 10", devImage)), + bk.Cmd(fmt.Sprintf("docker push %s || exit 10", devImage)), // Retry in case of flakes when pushing bk.AutomaticRetryStatus(3, 10), // Retry in case of flakes when pushing bk.AutomaticRetryStatus(3, 222), ) - pipeline.AddStep(fmt.Sprintf(":octopus: :docker: :construction: Build %s", app), cmds...) + pipeline.AddStep(fmt.Sprintf(":octopus: :docker: :construction: Build Wolfi-based %s", app), cmds...) } } From 9c75a068112026a0d240eacca90503ebb41ed3dd Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:45:44 +0000 Subject: [PATCH 08/57] Add wolfi builds for frontend --- enterprise/cmd/frontend/Dockerfile.wolfi | 22 +++++++++++++++++ enterprise/cmd/frontend/build-wolfi.sh | 29 +++++++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 4 ++-- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 enterprise/cmd/frontend/Dockerfile.wolfi create mode 100755 enterprise/cmd/frontend/build-wolfi.sh diff --git a/enterprise/cmd/frontend/Dockerfile.wolfi b/enterprise/cmd/frontend/Dockerfile.wolfi new file mode 100644 index 0000000000000..633eb6e5b761b --- /dev/null +++ b/enterprise/cmd/frontend/Dockerfile.wolfi @@ -0,0 +1,22 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +ENV CONFIGURATION_MODE=server PGDATABASE=sg PGHOST=pgsql PGPORT=5432 PGSSLMODE=disable PGUSER=sg CODEINTEL_PGDATABASE=sg CODEINTEL_PGHOST=codeintel-db CODEINTEL_PGPORT=5432 CODEINTEL_PGSSLMODE=disable CODEINTEL_PGUSER=sg PUBLIC_REPO_REDIRECTS=true +RUN mkdir -p /mnt/cache/frontend && chown -R sourcegraph:sourcegraph /mnt/cache/frontend +USER sourcegraph + +CMD ["serve"] +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/frontend"] +COPY frontend /usr/local/bin/ diff --git a/enterprise/cmd/frontend/build-wolfi.sh b/enterprise/cmd/frontend/build-wolfi.sh new file mode 100755 index 0000000000000..a8357f7efc87f --- /dev/null +++ b/enterprise/cmd/frontend/build-wolfi.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script builds the frontend docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -eu + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +echo "--- docker build" +docker build -f enterprise/cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index acc69f23608f1..aba00e618e973 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -120,8 +120,8 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { } // Always rebuild Wolfi images ops.Merge( - // TODO: Just hardcode a single image initially - BuildWolfiOperations([]string{"gitserver"}, c.Version, c.candidateImageTag()), + // TODO: Just hardcode specific images initially + BuildWolfiOperations([]string{"gitserver", "frontend"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From 818f95eb0797db0c5d6ed51edab747983fa03af3 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:56:29 +0000 Subject: [PATCH 09/57] Add github-proxy wolfi build --- cmd/github-proxy/Dockerfile.wolfi | 20 +++++++++++++++++ cmd/github-proxy/build-wolfi.sh | 26 +++++++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 cmd/github-proxy/Dockerfile.wolfi create mode 100755 cmd/github-proxy/build-wolfi.sh diff --git a/cmd/github-proxy/Dockerfile.wolfi b/cmd/github-proxy/Dockerfile.wolfi new file mode 100644 index 0000000000000..614d82bbe603e --- /dev/null +++ b/cmd/github-proxy/Dockerfile.wolfi @@ -0,0 +1,20 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM sourcegraph-wolfi/sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +ENV LOG_REQUEST=true +USER sourcegraph + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/github-proxy"] +COPY github-proxy /usr/local/bin/ diff --git a/cmd/github-proxy/build-wolfi.sh b/cmd/github-proxy/build-wolfi.sh new file mode 100755 index 0000000000000..b9a54576c3c21 --- /dev/null +++ b/cmd/github-proxy/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/cmd/github-proxy" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f cmd/github-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index aba00e618e973..da2f844beda56 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From 16b0d48a465891b86a7b9b28700eb766cca6d95a Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 14:58:33 +0000 Subject: [PATCH 10/57] Add loadtest wolfi build --- cmd/loadtest/Dockerfile.wolfi | 17 +++++++++++++++ cmd/loadtest/build-wolfi.sh | 26 +++++++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 cmd/loadtest/Dockerfile.wolfi create mode 100755 cmd/loadtest/build-wolfi.sh diff --git a/cmd/loadtest/Dockerfile.wolfi b/cmd/loadtest/Dockerfile.wolfi new file mode 100644 index 0000000000000..cec84b21713f9 --- /dev/null +++ b/cmd/loadtest/Dockerfile.wolfi @@ -0,0 +1,17 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM sourcegraph-wolfi/sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/loadtest"] +COPY loadtest /usr/local/bin/ diff --git a/cmd/loadtest/build-wolfi.sh b/cmd/loadtest/build-wolfi.sh new file mode 100755 index 0000000000000..c960697f5414e --- /dev/null +++ b/cmd/loadtest/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/cmd/loadtest" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f cmd/loadtest/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index da2f844beda56..e7c087746e52a 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From 44e11498f1b4cbd268439d6e1f01bb67cf7ce668 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Tue, 31 Jan 2023 15:02:42 +0000 Subject: [PATCH 11/57] Add migrator build --- cmd/migrator/Dockerfile.wolfi | 20 +++++ cmd/migrator/build-wolfi.sh | 92 +++++++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 cmd/migrator/Dockerfile.wolfi create mode 100755 cmd/migrator/build-wolfi.sh diff --git a/cmd/migrator/Dockerfile.wolfi b/cmd/migrator/Dockerfile.wolfi new file mode 100644 index 0000000000000..e1c8ef5b4e52b --- /dev/null +++ b/cmd/migrator/Dockerfile.wolfi @@ -0,0 +1,20 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM sourcegraph-wolfi/sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/migrator"] +COPY migrator /usr/local/bin/ +COPY schema-descriptions /schema-descriptions diff --git a/cmd/migrator/build-wolfi.sh b/cmd/migrator/build-wolfi.sh new file mode 100755 index 0000000000000..7b09373862d95 --- /dev/null +++ b/cmd/migrator/build-wolfi.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# This script builds the migrator docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -eu + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg=${1:-"github.com/sourcegraph/sourcegraph/cmd/migrator"} +output="$OUTPUT/$(basename "$pkg")" +# shellcheck disable=SC2153 +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$output" "$pkg" + +echo "--- compile schema descriptions" +mkdir -p "${OUTPUT}/schema-descriptions" + +# See internal/database/migration/cliutil/drift-schemas/generate-all.sh +gcs_versions=( + v3.20.0 v3.20.1 + v3.21.0 v3.21.1 v3.21.2 + v3.22.0 v3.22.1 + v3.23.0 + v3.24.0 v3.24.1 + v3.25.0 v3.25.1 v3.25.2 + v3.26.0 v3.26.1 v3.26.2 v3.26.3 + v3.27.0 v3.27.1 v3.27.2 v3.27.3 v3.27.4 v3.27.5 + v3.28.0 + v3.29.0 v3.29.1 + v3.30.0 v3.30.1 v3.30.2 v3.30.3 v3.30.4 + v3.31.0 v3.31.1 v3.31.2 + v3.32.0 v3.32.1 + v3.33.0 v3.33.1 v3.33.2 + v3.34.0 v3.34.1 v3.34.2 + v3.35.0 v3.35.1 v3.35.2 + v3.36.0 v3.36.1 v3.36.2 v3.36.3 + v3.37.0 + v3.38.0 v3.38.1 + v3.39.0 v3.39.1 + v3.40.0 v3.40.1 v3.40.2 + v3.41.0 v3.41.1 +) +gcs_filenames=( + internal_database_schema.json + internal_database_schema.codeintel.json + internal_database_schema.codeinsights.json +) + +function download_gcs() { + outfile="${OUTPUT}/schema-descriptions/${1}-${2}" + if ! curl -fsSL "https://storage.googleapis.com/sourcegraph-assets/migrations/drift/${1}-${2}" 2>/dev/null >"${outfile}"; then + rm "${outfile}" + fi +} + +for version in "${gcs_versions[@]}"; do + echo "Persisting schemas for ${version} from GCS..." + for filename in "${gcs_filenames[@]}"; do + download_gcs "${version}" "${filename}" + done +done + +git_versions=( + v3.42.0 v3.42.1 v3.42.2 + v3.43.0 v3.43.1 v3.43.2 + v4.0.0 v4.0.1 + v4.1.0 v4.1.1 v4.1.2 +) +for version in "${git_versions[@]}"; do + echo "Persisting schemas for ${version} from Git..." + git show "${version}:internal/database/schema.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.json" + git show "${version}:internal/database/schema.codeintel.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.codeintel.json" + git show "${version}:internal/database/schema.codeinsights.json" >"${OUTPUT}/schema-descriptions/${version}-internal_database_schema.codeinsights.json" +done + +echo "--- docker build" +docker build -f cmd/migrator/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index e7c087746e52a..ad28401d4350e 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From ccf1cb0fa5b023e07d190e942aa3eeb7ed8db65e Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 10:42:48 +0000 Subject: [PATCH 12/57] Update wolfi base image location --- cmd/github-proxy/Dockerfile.wolfi | 2 +- cmd/loadtest/Dockerfile.wolfi | 2 +- cmd/migrator/Dockerfile.wolfi | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/github-proxy/Dockerfile.wolfi b/cmd/github-proxy/Dockerfile.wolfi index 614d82bbe603e..0630fe30b9ea0 100644 --- a/cmd/github-proxy/Dockerfile.wolfi +++ b/cmd/github-proxy/Dockerfile.wolfi @@ -2,7 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM sourcegraph-wolfi/sourcegraph-base:latest +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" ARG DATE="unknown" diff --git a/cmd/loadtest/Dockerfile.wolfi b/cmd/loadtest/Dockerfile.wolfi index cec84b21713f9..f5c90ceb1d4f2 100644 --- a/cmd/loadtest/Dockerfile.wolfi +++ b/cmd/loadtest/Dockerfile.wolfi @@ -2,7 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM sourcegraph-wolfi/sourcegraph-base:latest +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" ARG DATE="unknown" diff --git a/cmd/migrator/Dockerfile.wolfi b/cmd/migrator/Dockerfile.wolfi index e1c8ef5b4e52b..33bdfe3ad61d0 100644 --- a/cmd/migrator/Dockerfile.wolfi +++ b/cmd/migrator/Dockerfile.wolfi @@ -2,7 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM sourcegraph-wolfi/sourcegraph-base:latest +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" ARG DATE="unknown" From ede1c6608d413657ef9a40e80111dd81b0ad022f Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 10:52:00 +0000 Subject: [PATCH 13/57] Add wolfi builds for OSS frontend and gitserver --- cmd/frontend/Dockerfile.wolfi | 21 +++++++++++++++++++++ cmd/frontend/build-wolfi.sh | 28 ++++++++++++++++++++++++++++ cmd/gitserver/Dockerfile.wolfi | 22 ++++++++++++++++++++++ cmd/gitserver/build-wolfi.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 cmd/frontend/Dockerfile.wolfi create mode 100755 cmd/frontend/build-wolfi.sh create mode 100644 cmd/gitserver/Dockerfile.wolfi create mode 100755 cmd/gitserver/build-wolfi.sh diff --git a/cmd/frontend/Dockerfile.wolfi b/cmd/frontend/Dockerfile.wolfi new file mode 100644 index 0000000000000..cb99cb40bfa97 --- /dev/null +++ b/cmd/frontend/Dockerfile.wolfi @@ -0,0 +1,21 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +ENV CONFIGURATION_MODE=server PGDATABASE=sg PGHOST=pgsql PGPORT=5432 PGSSLMODE=disable PGUSER=sg CODEINTEL_PGDATABASE=sg CODEINTEL_PGHOST=codeintel-db CODEINTEL_PGPORT=5432 CODEINTEL_PGSSLMODE=disable CODEINTEL_PGUSER=sg PUBLIC_REPO_REDIRECTS=true +USER sourcegraph + +CMD ["serve"] +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/frontend"] +COPY frontend /usr/local/bin/ diff --git a/cmd/frontend/build-wolfi.sh b/cmd/frontend/build-wolfi.sh new file mode 100755 index 0000000000000..d62847079a017 --- /dev/null +++ b/cmd/frontend/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/cmd/frontend" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +echo "--- docker build $IMAGE" +docker build -f cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/cmd/gitserver/Dockerfile.wolfi b/cmd/gitserver/Dockerfile.wolfi new file mode 100644 index 0000000000000..d6333c8c8f8b2 --- /dev/null +++ b/cmd/gitserver/Dockerfile.wolfi @@ -0,0 +1,22 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-gitserver-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +RUN mkdir -p /data/repos && chown -R sourcegraph:sourcegraph /data/repos +USER sourcegraph + +WORKDIR / + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/gitserver"] +COPY gitserver /usr/local/bin/ diff --git a/cmd/gitserver/build-wolfi.sh b/cmd/gitserver/build-wolfi.sh new file mode 100755 index 0000000000000..8b97141467989 --- /dev/null +++ b/cmd/gitserver/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) + +cleanup() { + rm -rf "$OUTPUT" +} + +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From bf8afa8ec7af579b24042728e175d7618f1c4ecb Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 10:54:39 +0000 Subject: [PATCH 14/57] Add enterprise/migrator build script --- enterprise/cmd/migrator/build-wolfi.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 enterprise/cmd/migrator/build-wolfi.sh diff --git a/enterprise/cmd/migrator/build-wolfi.sh b/enterprise/cmd/migrator/build-wolfi.sh new file mode 100755 index 0000000000000..b17ca5f0216d2 --- /dev/null +++ b/enterprise/cmd/migrator/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. + +./cmd/migrator/build-wolfi.sh github.com/sourcegraph/sourcegraph/enterprise/cmd/migrator From 217df1a28f1af64df2379ca32c69b50885bb5914 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:07:10 +0000 Subject: [PATCH 15/57] Add wolfi builds for repo-updater --- cmd/repo-updater/Dockerfile.wolfi | 18 ++++++++++++++ cmd/repo-updater/build-wolfi.sh | 28 ++++++++++++++++++++++ enterprise/cmd/repo-updater/build-wolfi.sh | 6 +++++ 3 files changed, 52 insertions(+) create mode 100644 cmd/repo-updater/Dockerfile.wolfi create mode 100755 cmd/repo-updater/build-wolfi.sh create mode 100755 enterprise/cmd/repo-updater/build-wolfi.sh diff --git a/cmd/repo-updater/Dockerfile.wolfi b/cmd/repo-updater/Dockerfile.wolfi new file mode 100644 index 0000000000000..1a1d168db9ea7 --- /dev/null +++ b/cmd/repo-updater/Dockerfile.wolfi @@ -0,0 +1,18 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-repo-updater-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/repo-updater"] +COPY repo-updater /usr/local/bin/ diff --git a/cmd/repo-updater/build-wolfi.sh b/cmd/repo-updater/build-wolfi.sh new file mode 100755 index 0000000000000..22a6cfa474445 --- /dev/null +++ b/cmd/repo-updater/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +path_to_package=${1:-github.com/sourcegraph/sourcegraph/cmd/repo-updater} +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +for pkg in $path_to_package; do + go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename "$pkg")" "$pkg" +done + +docker build -f cmd/repo-updater/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/cmd/repo-updater/build-wolfi.sh b/enterprise/cmd/repo-updater/build-wolfi.sh new file mode 100755 index 0000000000000..dd01ecab21dc1 --- /dev/null +++ b/enterprise/cmd/repo-updater/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. + +./cmd/repo-updater/build-wolfi.sh github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater From 1d51b946ab0b961df84c8bc23767a3c180453ab0 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:11:39 +0000 Subject: [PATCH 16/57] Add wolfi builds for searcher --- cmd/searcher/Dockerfile.wolfi | 21 +++++++++++++++++++++ cmd/searcher/build-wolfi.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 cmd/searcher/Dockerfile.wolfi create mode 100755 cmd/searcher/build-wolfi.sh diff --git a/cmd/searcher/Dockerfile.wolfi b/cmd/searcher/Dockerfile.wolfi new file mode 100644 index 0000000000000..771d0c8c54543 --- /dev/null +++ b/cmd/searcher/Dockerfile.wolfi @@ -0,0 +1,21 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-searcher-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +ENV CACHE_DIR=/mnt/cache/searcher +RUN mkdir -p ${CACHE_DIR} && chown -R sourcegraph:sourcegraph ${CACHE_DIR} +USER sourcegraph + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/searcher"] +COPY searcher /usr/local/bin/ diff --git a/cmd/searcher/build-wolfi.sh b/cmd/searcher/build-wolfi.sh new file mode 100755 index 0000000000000..54244d9d1703e --- /dev/null +++ b/cmd/searcher/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/cmd/searcher" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f cmd/searcher/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From 1862fa6fb1bf4c162b3a52c9b22a6077c8a9196c Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:16:20 +0000 Subject: [PATCH 17/57] Add wolfi builds for symbols --- cmd/symbols/Dockerfile.wolfi | 60 +++++++++++++++++++++++++++ cmd/symbols/build-wolfi.sh | 14 +++++++ enterprise/cmd/symbols/build-wolfi.sh | 10 +++++ 3 files changed, 84 insertions(+) create mode 100644 cmd/symbols/Dockerfile.wolfi create mode 100755 cmd/symbols/build-wolfi.sh create mode 100755 enterprise/cmd/symbols/build-wolfi.sh diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi new file mode 100644 index 0000000000000..a637263281647 --- /dev/null +++ b/cmd/symbols/Dockerfile.wolfi @@ -0,0 +1,60 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM cgr.dev/chainguard/go:latest AS symbols-build +# hadolint ignore=DL3002 +USER root + +ENV GO111MODULE on +ENV GOARCH amd64 +ENV GOOS linux +ENV CGO_ENABLED 1 + +COPY . /repo + +WORKDIR /repo + +ARG VERSION="unknown" +ENV VERSION $VERSION + +ARG PKG +ENV PKG=$PKG + +RUN \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/root/go/pkg/mod \ + go build \ + -trimpath \ + -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" \ + -buildmode exe \ + -tags dist \ + -o /symbols \ + $PKG + +FROM us.gcr.io/sourcegraph-dev/wolfi-symbols-base:latest AS symbols + +# TODO(security): This container should not run as root! +# +# See https://github.com/sourcegraph/sourcegraph/issues/13237 +# hadolint ignore=DL3002 +USER root + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +COPY --from=symbols-build /symbols /usr/local/bin/symbols + +# symbols is cgo, ensure we have the requisite dynamic libraries +RUN env SANITY_CHECK=true /usr/local/bin/symbols + +ENV CACHE_DIR=/mnt/cache/symbols +RUN mkdir -p ${CACHE_DIR} +EXPOSE 3184 +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/symbols"] diff --git a/cmd/symbols/build-wolfi.sh b/cmd/symbols/build-wolfi.sh new file mode 100755 index 0000000000000..e5cb3d3e8dee3 --- /dev/null +++ b/cmd/symbols/build-wolfi.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script builds the symbols docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -eu + +echo "--- docker build symbols" +docker build -f cmd/symbols/Dockerfile.wolfi -t "$IMAGE" "$(pwd)" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION \ + --build-arg PKG="${PKG:-github.com/sourcegraph/sourcegraph/cmd/symbols}" diff --git a/enterprise/cmd/symbols/build-wolfi.sh b/enterprise/cmd/symbols/build-wolfi.sh new file mode 100755 index 0000000000000..7703024fd3f0a --- /dev/null +++ b/enterprise/cmd/symbols/build-wolfi.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# This script builds the symbols docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -eu + +env \ + PKG=github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \ + cmd/symbols/build-wolfi.sh From 880f6b98d3c065427d0234b76a8be7ac13fa7e69 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:34:10 +0000 Subject: [PATCH 18/57] Build additional full images --- enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index ad28401d4350e..cbe7945e4e14e 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From fd4fafb4006ef8edb8461197504eab36a5c6bb60 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:37:33 +0000 Subject: [PATCH 19/57] Add wolfi builds for batcheshelper --- enterprise/cmd/batcheshelper/Dockerfile.wolfi | 16 ++++++++++++ enterprise/cmd/batcheshelper/build-wolfi.sh | 26 +++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 enterprise/cmd/batcheshelper/Dockerfile.wolfi create mode 100755 enterprise/cmd/batcheshelper/build-wolfi.sh diff --git a/enterprise/cmd/batcheshelper/Dockerfile.wolfi b/enterprise/cmd/batcheshelper/Dockerfile.wolfi new file mode 100644 index 0000000000000..06f5530851c69 --- /dev/null +++ b/enterprise/cmd/batcheshelper/Dockerfile.wolfi @@ -0,0 +1,16 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-batcheshelper-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +COPY batcheshelper /usr/local/bin/ diff --git a/enterprise/cmd/batcheshelper/build-wolfi.sh b/enterprise/cmd/batcheshelper/build-wolfi.sh new file mode 100755 index 0000000000000..cd7c935c68029 --- /dev/null +++ b/enterprise/cmd/batcheshelper/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/batcheshelper" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f enterprise/cmd/batcheshelper/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index cbe7945e4e14e..c5cf84c59375e 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols", "batcheshelper"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From a1028a16191ec2a6e60c1e374e2c27c7f045344d Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 11:44:50 +0000 Subject: [PATCH 20/57] Add wolfi build for precise code intel --- .../Dockerfile.wolfi | 20 +++++++++++++ .../precise-code-intel-worker/build-wolfi.sh | 29 +++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi create mode 100755 enterprise/cmd/precise-code-intel-worker/build-wolfi.sh diff --git a/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi new file mode 100644 index 0000000000000..c48f727b86ccc --- /dev/null +++ b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi @@ -0,0 +1,20 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM sourcegraph-wolfi/sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +EXPOSE 3188 + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/precise-code-intel-worker"] +COPY precise-code-intel-worker /usr/local/bin/ diff --git a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh new file mode 100755 index 0000000000000..45a3647621293 --- /dev/null +++ b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script builds the precise-code-intel-worker docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -eu + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +echo "--- docker build" +docker build -f enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index c5cf84c59375e..e72c6dca2acd3 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols", "batcheshelper"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols", "batcheshelper", "precise-code-intel-worker"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From d17877cc782039bca137bac7767c17aad615586f Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 12:02:24 +0000 Subject: [PATCH 21/57] Add wolfi build for enterprise worker --- enterprise/cmd/worker/Dockerfile.wolfi | 20 ++++++++++++++++ enterprise/cmd/worker/build-wolfi.sh | 29 +++++++++++++++++++++++ enterprise/dev/ci/internal/ci/pipeline.go | 3 ++- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 enterprise/cmd/worker/Dockerfile.wolfi create mode 100755 enterprise/cmd/worker/build-wolfi.sh diff --git a/enterprise/cmd/worker/Dockerfile.wolfi b/enterprise/cmd/worker/Dockerfile.wolfi new file mode 100644 index 0000000000000..4dac697458531 --- /dev/null +++ b/enterprise/cmd/worker/Dockerfile.wolfi @@ -0,0 +1,20 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +EXPOSE 3189 + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/worker"] +COPY worker /usr/local/bin/ diff --git a/enterprise/cmd/worker/build-wolfi.sh b/enterprise/cmd/worker/build-wolfi.sh new file mode 100755 index 0000000000000..e77aca857fdfe --- /dev/null +++ b/enterprise/cmd/worker/build-wolfi.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script builds the enterprise worker docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -eu + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/worker" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +echo "--- docker build" +docker build -f enterprise/cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index e72c6dca2acd3..e67e16c9f0f71 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,8 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "symbols", "batcheshelper", "precise-code-intel-worker"}, c.Version, c.candidateImageTag()), + // TODO: Debug "symbols", + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From 26af851e09c065a4283ec48be62fe20e5fe83d1e Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 12:03:36 +0000 Subject: [PATCH 22/57] Fix wolfi base image for precise code intel --- enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi index c48f727b86ccc..791cb6cd26042 100644 --- a/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi +++ b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi @@ -2,7 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM sourcegraph-wolfi/sourcegraph-base:latest +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" ARG DATE="unknown" From c81407cab3b62e3c1ed02ecd94f1b71827270282 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 13:03:12 +0000 Subject: [PATCH 23/57] Always build x86_64 Wolfi images This means that even when building on an ARM mac, you'll build an x86_64 image --- cmd/frontend/build-wolfi.sh | 1 + cmd/github-proxy/build-wolfi.sh | 1 + cmd/gitserver/build-wolfi.sh | 1 + cmd/loadtest/build-wolfi.sh | 1 + cmd/migrator/build-wolfi.sh | 1 + cmd/repo-updater/build-wolfi.sh | 1 + cmd/searcher/build-wolfi.sh | 1 + cmd/symbols/build-wolfi.sh | 1 + enterprise/cmd/frontend/build-wolfi.sh | 1 + enterprise/cmd/gitserver/build-wolfi.sh | 1 + enterprise/cmd/precise-code-intel-worker/build-wolfi.sh | 1 + enterprise/cmd/worker/build-wolfi.sh | 1 + 12 files changed, 12 insertions(+) diff --git a/cmd/frontend/build-wolfi.sh b/cmd/frontend/build-wolfi.sh index d62847079a017..2661d60edf8c2 100755 --- a/cmd/frontend/build-wolfi.sh +++ b/cmd/frontend/build-wolfi.sh @@ -22,6 +22,7 @@ go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/vers echo "--- docker build $IMAGE" docker build -f cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/github-proxy/build-wolfi.sh b/cmd/github-proxy/build-wolfi.sh index b9a54576c3c21..3b46012a9dbfb 100755 --- a/cmd/github-proxy/build-wolfi.sh +++ b/cmd/github-proxy/build-wolfi.sh @@ -20,6 +20,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/github-proxy" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/github-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/gitserver/build-wolfi.sh b/cmd/gitserver/build-wolfi.sh index 8b97141467989..3eee79c318fdb 100755 --- a/cmd/gitserver/build-wolfi.sh +++ b/cmd/gitserver/build-wolfi.sh @@ -22,6 +22,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/loadtest/build-wolfi.sh b/cmd/loadtest/build-wolfi.sh index c960697f5414e..dd1932eb88789 100755 --- a/cmd/loadtest/build-wolfi.sh +++ b/cmd/loadtest/build-wolfi.sh @@ -20,6 +20,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/loadtest" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/loadtest/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/migrator/build-wolfi.sh b/cmd/migrator/build-wolfi.sh index 7b09373862d95..012660cb9787d 100755 --- a/cmd/migrator/build-wolfi.sh +++ b/cmd/migrator/build-wolfi.sh @@ -86,6 +86,7 @@ done echo "--- docker build" docker build -f cmd/migrator/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/repo-updater/build-wolfi.sh b/cmd/repo-updater/build-wolfi.sh index 22a6cfa474445..38024b183b9ec 100755 --- a/cmd/repo-updater/build-wolfi.sh +++ b/cmd/repo-updater/build-wolfi.sh @@ -22,6 +22,7 @@ for pkg in $path_to_package; do done docker build -f cmd/repo-updater/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/searcher/build-wolfi.sh b/cmd/searcher/build-wolfi.sh index 54244d9d1703e..75b6d37f8dd11 100755 --- a/cmd/searcher/build-wolfi.sh +++ b/cmd/searcher/build-wolfi.sh @@ -20,6 +20,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/searcher" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/searcher/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/symbols/build-wolfi.sh b/cmd/symbols/build-wolfi.sh index e5cb3d3e8dee3..fd49d082d07ec 100755 --- a/cmd/symbols/build-wolfi.sh +++ b/cmd/symbols/build-wolfi.sh @@ -7,6 +7,7 @@ set -eu echo "--- docker build symbols" docker build -f cmd/symbols/Dockerfile.wolfi -t "$IMAGE" "$(pwd)" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/frontend/build-wolfi.sh b/enterprise/cmd/frontend/build-wolfi.sh index a8357f7efc87f..dba5b931dbc51 100755 --- a/enterprise/cmd/frontend/build-wolfi.sh +++ b/enterprise/cmd/frontend/build-wolfi.sh @@ -23,6 +23,7 @@ go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/vers echo "--- docker build" docker build -f enterprise/cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/gitserver/build-wolfi.sh b/enterprise/cmd/gitserver/build-wolfi.sh index 8b97141467989..3eee79c318fdb 100755 --- a/enterprise/cmd/gitserver/build-wolfi.sh +++ b/enterprise/cmd/gitserver/build-wolfi.sh @@ -22,6 +22,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh index 45a3647621293..ec9f3563135a0 100755 --- a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh +++ b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh @@ -23,6 +23,7 @@ go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/vers echo "--- docker build" docker build -f enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/worker/build-wolfi.sh b/enterprise/cmd/worker/build-wolfi.sh index e77aca857fdfe..47cf7da0358a3 100755 --- a/enterprise/cmd/worker/build-wolfi.sh +++ b/enterprise/cmd/worker/build-wolfi.sh @@ -23,6 +23,7 @@ go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/vers echo "--- docker build" docker build -f enterprise/cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ From 0a2cc44d51d94b7f1aeb88b5fe99a52b61a1b2b0 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 13:03:51 +0000 Subject: [PATCH 24/57] Build symbols again --- enterprise/dev/ci/internal/ci/pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index e67e16c9f0f71..393f1b377a154 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -122,7 +122,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { ops.Merge( // TODO: Just hardcode specific images initially // TODO: Debug "symbols", - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker", "symbols"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: From 285fae49d04f8826b6d63443457aeaf5826bd8db Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 13:36:50 +0000 Subject: [PATCH 25/57] Try using non-Alpine golang container for symbols build --- cmd/symbols/Dockerfile.wolfi | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi index a637263281647..088169c7ff082 100644 --- a/cmd/symbols/Dockerfile.wolfi +++ b/cmd/symbols/Dockerfile.wolfi @@ -2,7 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM cgr.dev/chainguard/go:latest AS symbols-build +FROM golang:1.19.3 AS symbols-build # hadolint ignore=DL3002 USER root @@ -11,6 +11,11 @@ ENV GOARCH amd64 ENV GOOS linux ENV CGO_ENABLED 1 +RUN apt-get update && \ + apt-get install -y \ + gcc \ + g++ + COPY . /repo WORKDIR /repo From 2c9f4a3e6ce733680b76ffc3071bb546ce9fad74 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 2 Feb 2023 14:13:38 +0000 Subject: [PATCH 26/57] Revert to alpine golang build env --- cmd/symbols/Dockerfile.wolfi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi index 088169c7ff082..7e8b1b29dd015 100644 --- a/cmd/symbols/Dockerfile.wolfi +++ b/cmd/symbols/Dockerfile.wolfi @@ -2,7 +2,8 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -FROM golang:1.19.3 AS symbols-build +# TODO: See if we can switch back to cgr.dev/chainguard/go:latest +FROM golang:1.19.3-alpine AS symbols-build # hadolint ignore=DL3002 USER root From fd8ee3d4d1e121520b3cd8b92cb6af81208abaae Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 09:52:52 +0000 Subject: [PATCH 27/57] Build full wolfi images on the bazel queue --- enterprise/dev/ci/internal/ci/wolfi_operations.go | 1 + 1 file changed, 1 insertion(+) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 2581652e097d2..2c39d07130d69 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -137,6 +137,7 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b bk.Env("DOCKER_BUILDKIT", "1"), bk.Env("IMAGE", localImage), bk.Env("VERSION", version), + bk.Agent("queue", "bazel"), } // Add Sentry environment variables if we are building off main branch From 3be6b3e94e78833b9481a59594b82ae57bfdb5a6 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 10:07:46 +0000 Subject: [PATCH 28/57] Disable building symbols and bazel runner for now --- enterprise/dev/ci/internal/ci/pipeline.go | 2 +- enterprise/dev/ci/internal/ci/wolfi_operations.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index 393f1b377a154..e67e16c9f0f71 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -122,7 +122,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { ops.Merge( // TODO: Just hardcode specific images initially // TODO: Debug "symbols", - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker", "symbols"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 2c39d07130d69..f03b14079ad25 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -137,7 +137,7 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b bk.Env("DOCKER_BUILDKIT", "1"), bk.Env("IMAGE", localImage), bk.Env("VERSION", version), - bk.Agent("queue", "bazel"), + // bk.Agent("queue", "bazel"), // TODO: Re-enable } // Add Sentry environment variables if we are building off main branch From f687b2072a4bcee0eda80ed16c86c2a087e6b785 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 10:36:35 +0000 Subject: [PATCH 29/57] Tweak local-build scripts --- wolfi-images/local-build.sh | 20 ++++++++++---------- wolfi-packages/local-build.sh | 11 ++++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/wolfi-images/local-build.sh b/wolfi-images/local-build.sh index 20ab433d230c7..4f561f3eb0027 100755 --- a/wolfi-images/local-build.sh +++ b/wolfi-images/local-build.sh @@ -15,21 +15,22 @@ cd "wolfi-images" # Normalise name by adding .yaml if necessary name=${1%/} -name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1.yaml/') +file_name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1.yaml/') +image_name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1/') -if [ ! -f "$name" ]; then - echo "File '$name' does not exist" +if [ ! -f "$file_name" ]; then + echo "File '$file_name' does not exist" exit 1 fi ## Build base image using apko build container -echo " * Building base image '$name' using apko" +echo " * Building base image '$image_name' using apko" docker run \ -v "$PWD":/work \ cgr.dev/chainguard/apko \ - build --debug "${name}" \ - "sourcegraph-wolfi/$name-base:latest" \ - "sourcegraph-wolfi-$name-base.tar" || + build --debug "${file_name}" \ + "sourcegraph-wolfi/$image_name-base:latest" \ + "sourcegraph-wolfi-$image_name-base.tar" || (echo "*** Build failed ***" && exit 1) # To build images against a local repo with a custom signing key: @@ -41,10 +42,9 @@ docker run \ ## Import into Docker echo " * Loading tarball into Docker" -docker load <"sourcegraph-wolfi-$name-base.tar" +docker load <"sourcegraph-wolfi-$image_name-base.tar" ## Cleanup echo " * Cleaning up tarball and SBOM" -rm "sourcegraph-wolfi-$name-base.tar" +rm "sourcegraph-wolfi-$image_name-base.tar" rm sbom* -rmdir keys/ packages/ diff --git a/wolfi-packages/local-build.sh b/wolfi-packages/local-build.sh index 38f8a80c798a1..c657b17be876b 100755 --- a/wolfi-packages/local-build.sh +++ b/wolfi-packages/local-build.sh @@ -15,16 +15,17 @@ fi # Normalise name by adding .yaml if necessary name=${1%/} -name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1.yaml/') +file_name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1.yaml/') +image_name=$(echo "$name" | sed -r 's/^([a-zA-Z0-9_-]+)$/\1.yaml/') -if [ ! -f "$name" ]; then - echo "File '$name' does not exist" +if [ ! -f "$file_name" ]; then + echo "File '$file_name' does not exist" exit 1 fi -echo "Building package '$name'" +echo "Building package '$image_name'" # Mounting /tmp can be useful for debugging: -v "$HOME/tmp":/tmp \ docker run --privileged \ -v "$PWD":/work \ - cgr.dev/chainguard/melange build "$name" --arch x86_64 + cgr.dev/chainguard/melange build "$file_name" --arch x86_64 From ce0fc9e8a80ecf6f1a3179044a95ee13b22ad63c Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 10:40:07 +0000 Subject: [PATCH 30/57] Add rough initial versions of server wolfi builds --- cmd/server/Dockerfile.wolfi | 72 +++++++++++++++++++++++ cmd/server/build-wolfi.sh | 112 ++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 cmd/server/Dockerfile.wolfi create mode 100755 cmd/server/build-wolfi.sh diff --git a/cmd/server/Dockerfile.wolfi b/cmd/server/Dockerfile.wolfi new file mode 100644 index 0000000000000..9a101a3ad57dc --- /dev/null +++ b/cmd/server/Dockerfile.wolfi @@ -0,0 +1,72 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +# TODO: This is untested and WIP + +FROM sourcegraph-wolfi/server-base:latest + +# TODO(security): This container should not be running as root! +# +# The default user in sourcegraph/alpine is a non-root `sourcegraph` user but because old deployments +# cannot be easily migrated we have not changed this from root -> sourcegraph. See: +# https://github.com/sourcegraph/sourcegraph/issues/13238 +# hadolint ignore=DL3002 +USER root + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +# IMPORTANT: If you update the syntax-highlighter version below, you MUST confirm +# the ENV variables from its Dockerfile (https://github.com/sourcegraph/sourcegraph/blob/main/docker-images/syntax-highlighter/Dockerfile) +# have been appropriately set in cmd/server/shared/shared.go. +# hadolint ignore=DL3022 +COPY --from=docker.io/sourcegraph/syntax-highlighter:186324_2022-12-01_02d3b4384446 /syntax_highlighter /usr/local/bin/ + + +# install blobstore (keep this up to date with the upstream Docker image +# referenced in docker-images/) +RUN apk add --no-cache --verbose openjdk11 +# hadolint ignore=DL3022 +COPY --from=sourcegraph/blobstore:server /opt/s3proxy /opt/s3proxy + + +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /bin/prom-wrapper /bin +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /bin/alertmanager /bin +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /alertmanager.sh /alertmanager.sh +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /bin/prometheus /bin +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /prometheus.sh /prometheus.sh +# hadolint ignore=DL3022 +COPY --from=sourcegraph/prometheus:server /usr/share/prometheus /usr/share/prometheus + +# hadolint ignore=DL3022 +COPY --from=sourcegraph/grafana:server /usr/share/grafana /usr/share/grafana + +COPY . / + +# hadolint ignore=DL3022 +COPY --from=sourcegraph/grafana:server /sg_config_grafana/provisioning/dashboards /sg_config_grafana/provisioning/dashboards + +# hadolint ignore=DL3022 +COPY --from=sourcegraph/postgres_exporter:server /usr/local/bin/postgres_exporter /usr/local/bin/postgres_exporter + +RUN echo "hosts: files dns" > /etc/nsswitch.conf + +# symbols is cgo, ensure we have the requisite dynamic libraries +RUN env SANITY_CHECK=true /usr/local/bin/symbols + +WORKDIR / + +ENV GO111MODULES=on LANG=en_US.utf8 +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/server"] diff --git a/cmd/server/build-wolfi.sh b/cmd/server/build-wolfi.sh new file mode 100755 index 0000000000000..d63c8629f5f53 --- /dev/null +++ b/cmd/server/build-wolfi.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# TODO: This is untested and WIP + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -eux + +# Fail early if env vars are not set +[ -n "$VERSION" ] +[ -n "$IMAGE" ] + +OUTPUT=$(mktemp -d -t sgserver_XXXXXXX) +export OUTPUT +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +parallel_run() { + ./dev/ci/parallel_run.sh "$@" +} +export -f parallel_run + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +# Additional images passed in here when this script is called externally by our +# enterprise build scripts. +additional_images=() +if [ $# -eq 0 ]; then + additional_images+=("github.com/sourcegraph/sourcegraph/cmd/frontend" "github.com/sourcegraph/sourcegraph/cmd/worker" "github.com/sourcegraph/sourcegraph/cmd/migrator" "github.com/sourcegraph/sourcegraph/cmd/repo-updater" "github.com/sourcegraph/sourcegraph/cmd/symbols") +else + additional_images+=("$@") +fi +export additional_images + +# Overridable server package path for when this script is called externally by +# our enterprise build scripts. +export server_pkg=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/cmd/server} + +cp -a ./cmd/server/rootfs/. "$OUTPUT" +export BINDIR="$OUTPUT/usr/local/bin" +mkdir -p "$BINDIR" + +go_build() { + local package="$1" + + if [[ "${CI_DEBUG_PROFILE:-"false"}" == "true" ]]; then + env time -v ./cmd/server/go-build.sh "$package" + else + ./cmd/server/go-build.sh "$package" + fi +} +export -f go_build + +echo "--- go build" + +PACKAGES=( + github.com/sourcegraph/sourcegraph/cmd/github-proxy + github.com/sourcegraph/sourcegraph/cmd/gitserver + github.com/sourcegraph/sourcegraph/cmd/searcher + github.com/sourcegraph/zoekt/cmd/zoekt-archive-index + github.com/sourcegraph/zoekt/cmd/zoekt-git-index + github.com/sourcegraph/zoekt/cmd/zoekt-sourcegraph-indexserver + github.com/sourcegraph/zoekt/cmd/zoekt-webserver +) + +PACKAGES+=("${additional_images[@]}") +PACKAGES+=("$server_pkg") + +parallel_run go_build {} ::: "${PACKAGES[@]}" + +echo "--- build scripts" +cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT" +cp -a ./cmd/gitserver/p4-fusion-install-alpine.sh "$OUTPUT" + +echo "--- monitoring generation" +# For code generation we need to match the local machine so we can run the generator +if [[ "$OSTYPE" == "darwin"* ]]; then + pushd monitoring && GOOS=darwin go generate && popd +else + pushd monitoring && go generate && popd +fi + +echo "--- prometheus" +cp -r docker-images/prometheus/config "$OUTPUT/sg_config_prometheus" +mkdir "$OUTPUT/sg_prometheus_add_ons" +cp dev/prometheus/linux/prometheus_targets.yml "$OUTPUT/sg_prometheus_add_ons" +IMAGE=sourcegraph/prometheus:server CACHE=true docker-images/prometheus/build.sh + +echo "--- grafana" +cp -r docker-images/grafana/config "$OUTPUT/sg_config_grafana" +cp -r dev/grafana/linux "$OUTPUT/sg_config_grafana/provisioning/datasources" +IMAGE=sourcegraph/grafana:server CACHE=true docker-images/grafana/build.sh + +echo "--- postgres exporter" +IMAGE=sourcegraph/postgres_exporter:server CACHE=true docker-images/postgres_exporter/build.sh + +echo "--- blobstore" +IMAGE=sourcegraph/blobstore:server docker-images/blobstore/build.sh + +echo "--- docker build" +docker build -f cmd/server/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --platform="${PLATFORM:-linux/amd64}" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From 8db3b5651425b0bb14a4cb4ecb2f01b1f7bd95af Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 15:07:34 +0000 Subject: [PATCH 31/57] Try building images with bazel agent again --- enterprise/dev/ci/internal/ci/pipeline.go | 3 +-- enterprise/dev/ci/internal/ci/wolfi_operations.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index e67e16c9f0f71..09fcdc294524d 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,8 +121,7 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - // TODO: Debug "symbols", - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker", "symbols"}, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index f03b14079ad25..2c39d07130d69 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -137,7 +137,7 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b bk.Env("DOCKER_BUILDKIT", "1"), bk.Env("IMAGE", localImage), bk.Env("VERSION", version), - // bk.Agent("queue", "bazel"), // TODO: Re-enable + bk.Agent("queue", "bazel"), } // Add Sentry environment variables if we are building off main branch From f099623e076522879307f2d027d6886139a41cd0 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 15:12:33 +0000 Subject: [PATCH 32/57] Actually use bazel for gitserver builds --- cmd/gitserver/build-wolfi.sh | 2 +- enterprise/cmd/gitserver/build-wolfi.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gitserver/build-wolfi.sh b/cmd/gitserver/build-wolfi.sh index 3eee79c318fdb..420cf1c6b60e2 100755 --- a/cmd/gitserver/build-wolfi.sh +++ b/cmd/gitserver/build-wolfi.sh @@ -19,7 +19,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/enterprise/cmd/gitserver/build-wolfi.sh b/enterprise/cmd/gitserver/build-wolfi.sh index 3eee79c318fdb..420cf1c6b60e2 100755 --- a/enterprise/cmd/gitserver/build-wolfi.sh +++ b/enterprise/cmd/gitserver/build-wolfi.sh @@ -19,7 +19,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ From fdcc1a55cc47cfbec1da6436c06ecbc314af1625 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 15:18:48 +0000 Subject: [PATCH 33/57] Build symbols in chainguard:go --- cmd/symbols/Dockerfile.wolfi | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi index 7e8b1b29dd015..851130ecd703a 100644 --- a/cmd/symbols/Dockerfile.wolfi +++ b/cmd/symbols/Dockerfile.wolfi @@ -3,7 +3,7 @@ # file if you change the regular Dockerfile. # TODO: See if we can switch back to cgr.dev/chainguard/go:latest -FROM golang:1.19.3-alpine AS symbols-build +FROM cgr.dev/chainguard/go:latest AS symbols-build # hadolint ignore=DL3002 USER root @@ -12,10 +12,10 @@ ENV GOARCH amd64 ENV GOOS linux ENV CGO_ENABLED 1 -RUN apt-get update && \ - apt-get install -y \ - gcc \ - g++ +# RUN apt-get update && \ +# apt-get install -y \ +# gcc \ +# g++ COPY . /repo From 33e275ba57c0edfcc5f508765865ec3f97efed09 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 3 Feb 2023 15:50:17 +0000 Subject: [PATCH 34/57] Build all Go binaries for Wolfi with Bazel --- cmd/frontend/build-wolfi.sh | 2 +- cmd/github-proxy/build-wolfi.sh | 2 +- cmd/loadtest/build-wolfi.sh | 2 +- cmd/migrator/build-wolfi.sh | 2 +- cmd/repo-updater/build-wolfi.sh | 2 +- cmd/searcher/build-wolfi.sh | 2 +- enterprise/cmd/batcheshelper/build-wolfi.sh | 2 +- enterprise/cmd/frontend/build-wolfi.sh | 2 +- enterprise/cmd/precise-code-intel-worker/build-wolfi.sh | 2 +- enterprise/cmd/worker/build-wolfi.sh | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/frontend/build-wolfi.sh b/cmd/frontend/build-wolfi.sh index 2661d60edf8c2..7cd5e537976ce 100755 --- a/cmd/frontend/build-wolfi.sh +++ b/cmd/frontend/build-wolfi.sh @@ -18,7 +18,7 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/cmd/frontend" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build $IMAGE" docker build -f cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ diff --git a/cmd/github-proxy/build-wolfi.sh b/cmd/github-proxy/build-wolfi.sh index 3b46012a9dbfb..5eabc7220b50a 100755 --- a/cmd/github-proxy/build-wolfi.sh +++ b/cmd/github-proxy/build-wolfi.sh @@ -17,7 +17,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/github-proxy" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/github-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/cmd/loadtest/build-wolfi.sh b/cmd/loadtest/build-wolfi.sh index dd1932eb88789..f0d26c1442ba0 100755 --- a/cmd/loadtest/build-wolfi.sh +++ b/cmd/loadtest/build-wolfi.sh @@ -17,7 +17,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/loadtest" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/loadtest/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/cmd/migrator/build-wolfi.sh b/cmd/migrator/build-wolfi.sh index 012660cb9787d..f9e8acb73deee 100755 --- a/cmd/migrator/build-wolfi.sh +++ b/cmd/migrator/build-wolfi.sh @@ -21,7 +21,7 @@ echo "--- go build" pkg=${1:-"github.com/sourcegraph/sourcegraph/cmd/migrator"} output="$OUTPUT/$(basename "$pkg")" # shellcheck disable=SC2153 -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$output" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$output" "$pkg" echo "--- compile schema descriptions" mkdir -p "${OUTPUT}/schema-descriptions" diff --git a/cmd/repo-updater/build-wolfi.sh b/cmd/repo-updater/build-wolfi.sh index 38024b183b9ec..f6a88c4948ae1 100755 --- a/cmd/repo-updater/build-wolfi.sh +++ b/cmd/repo-updater/build-wolfi.sh @@ -18,7 +18,7 @@ export GOOS=linux export CGO_ENABLED=0 for pkg in $path_to_package; do - go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename "$pkg")" "$pkg" + bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename "$pkg")" "$pkg" done docker build -f cmd/repo-updater/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ diff --git a/cmd/searcher/build-wolfi.sh b/cmd/searcher/build-wolfi.sh index 75b6d37f8dd11..b927f4431fc03 100755 --- a/cmd/searcher/build-wolfi.sh +++ b/cmd/searcher/build-wolfi.sh @@ -17,7 +17,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/searcher" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/searcher/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/enterprise/cmd/batcheshelper/build-wolfi.sh b/enterprise/cmd/batcheshelper/build-wolfi.sh index cd7c935c68029..d12265fec6575 100755 --- a/enterprise/cmd/batcheshelper/build-wolfi.sh +++ b/enterprise/cmd/batcheshelper/build-wolfi.sh @@ -16,7 +16,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/batcheshelper" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f enterprise/cmd/batcheshelper/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/enterprise/cmd/frontend/build-wolfi.sh b/enterprise/cmd/frontend/build-wolfi.sh index dba5b931dbc51..0a01402834b68 100755 --- a/enterprise/cmd/frontend/build-wolfi.sh +++ b/enterprise/cmd/frontend/build-wolfi.sh @@ -19,7 +19,7 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ diff --git a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh index ec9f3563135a0..3adfabdf9c76e 100755 --- a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh +++ b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh @@ -19,7 +19,7 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ diff --git a/enterprise/cmd/worker/build-wolfi.sh b/enterprise/cmd/worker/build-wolfi.sh index 47cf7da0358a3..e9f5dadbfcc9e 100755 --- a/enterprise/cmd/worker/build-wolfi.sh +++ b/enterprise/cmd/worker/build-wolfi.sh @@ -19,7 +19,7 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/worker" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ From 1e403d91930620a9c02ba0f43a1afb2ae8ff9041 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Fri, 7 Apr 2023 14:25:14 +0100 Subject: [PATCH 35/57] Add wolfi build scripts for embeddings image --- enterprise/cmd/embeddings/Dockerfile.wolfi | 17 ++++++++++++++ enterprise/cmd/embeddings/build-wolfi.sh | 26 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 enterprise/cmd/embeddings/Dockerfile.wolfi create mode 100755 enterprise/cmd/embeddings/build-wolfi.sh diff --git a/enterprise/cmd/embeddings/Dockerfile.wolfi b/enterprise/cmd/embeddings/Dockerfile.wolfi new file mode 100644 index 0000000000000..4bd3877bd6325 --- /dev/null +++ b/enterprise/cmd/embeddings/Dockerfile.wolfi @@ -0,0 +1,17 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +EXPOSE 9991 +WORKDIR / + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/embeddings"] +COPY embeddings /usr/local/bin/ diff --git a/enterprise/cmd/embeddings/build-wolfi.sh b/enterprise/cmd/embeddings/build-wolfi.sh new file mode 100755 index 0000000000000..f2394c1da2ee6 --- /dev/null +++ b/enterprise/cmd/embeddings/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/embeddings" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f enterprise/cmd/embeddings/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From 157a7bd1b0acf292b2a94b0839d60584c0d208e2 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Wed, 12 Apr 2023 17:37:01 +0100 Subject: [PATCH 36/57] Rough initial version of the server image --- cmd/server/Dockerfile.wolfi | 23 +++++++++-------- cmd/server/build-wolfi.sh | 10 +++----- enterprise/cmd/server/build-wolfi.sh | 17 +++++++++++++ wolfi-images/server.yaml | 38 +++++++++++++++++++++++----- 4 files changed, 63 insertions(+), 25 deletions(-) create mode 100755 enterprise/cmd/server/build-wolfi.sh diff --git a/cmd/server/Dockerfile.wolfi b/cmd/server/Dockerfile.wolfi index 9a101a3ad57dc..eede4feb20966 100644 --- a/cmd/server/Dockerfile.wolfi +++ b/cmd/server/Dockerfile.wolfi @@ -4,7 +4,8 @@ # TODO: This is untested and WIP -FROM sourcegraph-wolfi/server-base:latest +FROM us.gcr.io/sourcegraph-dev/wolfi-server-base:latest +# FROM --platform=x86_64 sourcegraph-wolfi/server-base:latest-amd64 # TODO(security): This container should not be running as root! # @@ -26,13 +27,13 @@ LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/comm # IMPORTANT: If you update the syntax-highlighter version below, you MUST confirm # the ENV variables from its Dockerfile (https://github.com/sourcegraph/sourcegraph/blob/main/docker-images/syntax-highlighter/Dockerfile) # have been appropriately set in cmd/server/shared/shared.go. +# TODO: Update this to use the correct syntax-highlighter build # hadolint ignore=DL3022 -COPY --from=docker.io/sourcegraph/syntax-highlighter:186324_2022-12-01_02d3b4384446 /syntax_highlighter /usr/local/bin/ +COPY --from=us.gcr.io/sourcegraph-dev/wolfi-syntax-highlighter:latest /usr/local/bin/syntax_highlighter /usr/local/bin/ # install blobstore (keep this up to date with the upstream Docker image # referenced in docker-images/) -RUN apk add --no-cache --verbose openjdk11 # hadolint ignore=DL3022 COPY --from=sourcegraph/blobstore:server /opt/s3proxy /opt/s3proxy @@ -52,21 +53,21 @@ COPY --from=sourcegraph/prometheus:server /usr/share/prometheus /usr/share/prome # hadolint ignore=DL3022 COPY --from=sourcegraph/grafana:server /usr/share/grafana /usr/share/grafana - -COPY . / - # hadolint ignore=DL3022 COPY --from=sourcegraph/grafana:server /sg_config_grafana/provisioning/dashboards /sg_config_grafana/provisioning/dashboards -# hadolint ignore=DL3022 -COPY --from=sourcegraph/postgres_exporter:server /usr/local/bin/postgres_exporter /usr/local/bin/postgres_exporter - -RUN echo "hosts: files dns" > /etc/nsswitch.conf +COPY . / +# TODO: Check # symbols is cgo, ensure we have the requisite dynamic libraries RUN env SANITY_CHECK=true /usr/local/bin/symbols WORKDIR / -ENV GO111MODULES=on LANG=en_US.utf8 +# TODO: Nginx expects these directories but doesn't create them by default, figure out why +RUN mkdir /var/lib/nginx/tmp /var/run + +# TODO: Check all paths in script still line up +ENV GO111MODULES=on +# ENV LANG=en_US.utf8 # TODO: Not setting this seems to fix a postgres startup issue ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/server"] diff --git a/cmd/server/build-wolfi.sh b/cmd/server/build-wolfi.sh index d63c8629f5f53..daba08ba23bd9 100755 --- a/cmd/server/build-wolfi.sh +++ b/cmd/server/build-wolfi.sh @@ -74,10 +74,6 @@ PACKAGES+=("$server_pkg") parallel_run go_build {} ::: "${PACKAGES[@]}" -echo "--- build scripts" -cp -a ./cmd/symbols/ctags-install-alpine.sh "$OUTPUT" -cp -a ./cmd/gitserver/p4-fusion-install-alpine.sh "$OUTPUT" - echo "--- monitoring generation" # For code generation we need to match the local machine so we can run the generator if [[ "$OSTYPE" == "darwin"* ]]; then @@ -90,7 +86,7 @@ echo "--- prometheus" cp -r docker-images/prometheus/config "$OUTPUT/sg_config_prometheus" mkdir "$OUTPUT/sg_prometheus_add_ons" cp dev/prometheus/linux/prometheus_targets.yml "$OUTPUT/sg_prometheus_add_ons" -IMAGE=sourcegraph/prometheus:server CACHE=true docker-images/prometheus/build.sh +IMAGE=sourcegraph/prometheus:server CACHE=true docker-images/prometheus/build-wolfi.sh echo "--- grafana" cp -r docker-images/grafana/config "$OUTPUT/sg_config_grafana" @@ -98,10 +94,10 @@ cp -r dev/grafana/linux "$OUTPUT/sg_config_grafana/provisioning/datasources" IMAGE=sourcegraph/grafana:server CACHE=true docker-images/grafana/build.sh echo "--- postgres exporter" -IMAGE=sourcegraph/postgres_exporter:server CACHE=true docker-images/postgres_exporter/build.sh +IMAGE=sourcegraph/postgres_exporter:server CACHE=true docker-images/postgres_exporter/build-wolfi.sh echo "--- blobstore" -IMAGE=sourcegraph/blobstore:server docker-images/blobstore/build.sh +IMAGE=sourcegraph/blobstore:server docker-images/blobstore/build-wolfi.sh echo "--- docker build" docker build -f cmd/server/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ diff --git a/enterprise/cmd/server/build-wolfi.sh b/enterprise/cmd/server/build-wolfi.sh new file mode 100755 index 0000000000000..8acff2dde1275 --- /dev/null +++ b/enterprise/cmd/server/build-wolfi.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. + +export SERVER_PKG=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/enterprise/cmd/server} + +./cmd/server/build-wolfi.sh \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/worker \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/migrator \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/embeddings diff --git a/wolfi-images/server.yaml b/wolfi-images/server.yaml index 136318c8a23dc..a1c0676463111 100644 --- a/wolfi-images/server.yaml +++ b/wolfi-images/server.yaml @@ -11,13 +11,17 @@ contents: - ca-certificates-bundle - tzdata - tini - - mailcap + # - mailcap # Conflicts with nginx # Dev tools - may not be required in production - busybox - curl - wget # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + ## Dev testing tools + - apk-tools + - vim + ## server packages - bash - ca-certificates # TODO: Required? @@ -25,19 +29,23 @@ contents: - git-lfs - git-p4 - libev - - nodejs # TODO: Is this the same as nodejs-current? Test upgrade from 14.5.0 to 18.12 + - libstdc++ # TODO: Is this still required? + - nginx + - nodejs-16 # TODO: Earliest version from Wolfi; test upgrade from 14.5.0 to 16.19.1 + - openjdk-11 - openssh-client - pcre - postgresql-12 - postgresql-12-contrib + - prometheus-postgres-exporter=0.12.0-r1 # IMPORTANT: Pinned version for managed updates - python3 # TODO: Missing python2; required? - - redis # TODO: 7.0.7; test upgrade from 5.0 + - redis # TODO: 7.0.10; test upgrade from 5.0 - sqlite-libs - su-exec ## Missing packages - #- nginx #- python2 + #- libc6-compat - musl-glibc compat library, I think not needed - comby@sourcegraph - ctags@sourcegraph @@ -49,15 +57,31 @@ accounts: groups: - groupname: sourcegraph gid: 101 + - groupname: postgres + gid: 70 + - groupname: nginx + gid: 102 + - groupname: redis + gid: 103 - groupname: grafana - gid: 201 + gid: 104 users: - username: sourcegraph uid: 100 gid: 101 + # These users should all be /bin/nologin + - username: postgres + uid: 70 + gid: 70 + - username: nginx + uid: 101 + gid: 102 + - username: redis + uid: 102 + gid: 103 - username: grafana - uid: 200 - gid: 201 + uid: 103 + gid: 104 # NOTE: This is ignored (see build output) # To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` From c37af38413154b1fa07384510cd06c46db6d062d Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Wed, 12 Apr 2023 17:41:02 +0100 Subject: [PATCH 37/57] Specify platform for symbols go-build This is required to build the binary for server --- cmd/symbols/go-build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/symbols/go-build.sh b/cmd/symbols/go-build.sh index 0d0fb2aa60722..c209a4b282995 100755 --- a/cmd/symbols/go-build.sh +++ b/cmd/symbols/go-build.sh @@ -17,8 +17,9 @@ echo "--- docker symbols build" # Required due to use of RUN --mount=type=cache in Dockerfile. export DOCKER_BUILDKIT=1 -docker build -f cmd/symbols/Dockerfile -t symbols-build "$(pwd)" \ +docker build -f cmd/symbols/Dockerfile.wolfi -t symbols-build "$(pwd)" \ --target=symbols-build \ + --platform="${PLATFORM:-linux/amd64}" \ # TODO(will): This is required for server image to build, but will break local builds --progress=plain \ --build-arg VERSION \ --build-arg PKG="${PKG:-github.com/sourcegraph/sourcegraph/cmd/symbols}" From a31e23a4ef8a9fafc153187bb5a28bc01978967d Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 11:08:05 +0100 Subject: [PATCH 38/57] Build 3rd party Wolfi images (#47368) This PR migrates our 3rd party docker images from docker-images/ to Wolfi. https://github.com/sourcegraph/sourcegraph/pull/47182 tracks our 1st party images in `cmd/`. ## Test plan - Manual testing - These images will not be live once merged, and will undergo full validation before this happens. --- cmd/worker/Dockerfile.wolfi | 20 +++++ cmd/worker/build-wolfi.sh | 29 +++++++ docker-images/blobstore/Dockerfile.wolfi | 61 +++++++++++++++ docker-images/blobstore/build-wolfi.sh | 16 ++++ docker-images/cadvisor/Dockerfile.wolfi | 30 ++++++++ docker-images/cadvisor/build-wolfi.sh | 9 +++ docker-images/codeinsights-db/build-wolfi.sh | 8 ++ docker-images/codeintel-db/build-wolfi.sh | 7 ++ .../indexed-searcher/Dockerfile.wolfi | 32 ++++++++ docker-images/indexed-searcher/build-wolfi.sh | 13 ++++ docker-images/jaeger-agent/Dockerfile.wolfi | 13 ++++ docker-images/jaeger-agent/build-wolfi.sh | 13 ++++ .../jaeger-all-in-one/Dockerfile.wolfi | 47 ++++++++++++ .../jaeger-all-in-one/build-wolfi.sh | 13 ++++ .../config/sampling_strategies.json | 6 ++ docker-images/node-exporter/Dockerfile.wolfi | 15 ++++ docker-images/node-exporter/build-wolfi.sh | 9 +++ .../opentelemetry-collector/Dockerfile.wolfi | 20 +++++ .../opentelemetry-collector/build-wolfi.sh | 10 +++ .../postgres-12-alpine/Dockerfile.wolfi | 27 +++++++ .../postgres-12-alpine/build-wolfi.sh | 9 +++ .../rootfs/postgres-wolfi.sh | 30 ++++++++ .../postgres_exporter/Dockerfile.wolfi | 20 +++++ .../postgres_exporter/build-wolfi.sh | 61 +++++++++++++++ docker-images/prometheus-gcp/build-wolfi.sh | 9 +++ docker-images/prometheus/Dockerfile.wolfi | 75 +++++++++++++++++++ docker-images/prometheus/build-wolfi.sh | 62 +++++++++++++++ docker-images/redis-cache/Dockerfile.wolfi | 5 ++ docker-images/redis-cache/build-wolfi.sh | 6 ++ docker-images/redis-store/Dockerfile.wolfi | 5 ++ docker-images/redis-store/build-wolfi.sh | 6 ++ docker-images/redis_exporter/Dockerfile.wolfi | 6 ++ docker-images/redis_exporter/build-wolfi.sh | 6 ++ docker-images/search-indexer/Dockerfile.wolfi | 31 ++++++++ docker-images/search-indexer/build-wolfi.sh | 13 ++++ docker-images/sg/Dockerfile.wolfi | 15 ++++ docker-images/sg/build-wolfi.sh | 28 +++++++ .../syntax-highlighter/Dockerfile.wolfi | 35 +++++++++ .../syntax-highlighter/build-wolfi.sh | 6 ++ enterprise/dev/ci/internal/ci/pipeline.go | 19 ++++- .../dev/ci/internal/ci/wolfi_operations.go | 3 - .../dev/ci/scripts/wolfi/build-base-image.sh | 3 + wolfi-images/blobstore.yaml | 39 ++++++++++ wolfi-images/cadvisor.yaml | 41 ++++++++++ wolfi-images/gitserver.yaml | 2 +- wolfi-images/jaeger-agent.yaml | 43 +++++++++++ wolfi-images/jaeger-all-in-one.yaml | 42 +++++++++++ wolfi-images/node-exporter.yaml | 32 ++++++++ wolfi-images/opentelemetry-collector.yaml | 43 +++++++++++ wolfi-images/postgres-exporter.yaml | 39 ++++++++++ wolfi-images/postgresql-12.yaml | 33 ++++++++ wolfi-images/redis-exporter.yaml | 41 ++++++++++ wolfi-images/redis.yaml | 34 +++++++++ wolfi-images/repo-updater.yaml | 2 +- wolfi-images/search-indexer.yaml | 44 +++++++++++ wolfi-images/searcher.yaml | 2 +- wolfi-images/server.yaml | 2 +- wolfi-images/sourcegraph-dev.yaml | 10 ++- wolfi-images/symbols.yaml | 2 +- wolfi-images/syntax-highlighter.yaml | 43 +++++++++++ wolfi-packages/.gitignore | 1 + wolfi-packages/cadvisor.yaml | 36 +++++++++ wolfi-packages/http-server-stabilizer.yaml | 1 + wolfi-packages/jaeger.yaml | 56 ++++++++++++++ wolfi-packages/opentelemetry-collector.yaml | 46 ++++++++++++ .../builder.template.yaml | 36 +++++++++ wolfi-packages/redis_exporter.yaml | 37 +++++++++ 67 files changed, 1548 insertions(+), 10 deletions(-) create mode 100644 cmd/worker/Dockerfile.wolfi create mode 100755 cmd/worker/build-wolfi.sh create mode 100644 docker-images/blobstore/Dockerfile.wolfi create mode 100755 docker-images/blobstore/build-wolfi.sh create mode 100644 docker-images/cadvisor/Dockerfile.wolfi create mode 100755 docker-images/cadvisor/build-wolfi.sh create mode 100755 docker-images/codeinsights-db/build-wolfi.sh create mode 100755 docker-images/codeintel-db/build-wolfi.sh create mode 100644 docker-images/indexed-searcher/Dockerfile.wolfi create mode 100755 docker-images/indexed-searcher/build-wolfi.sh create mode 100644 docker-images/jaeger-agent/Dockerfile.wolfi create mode 100755 docker-images/jaeger-agent/build-wolfi.sh create mode 100644 docker-images/jaeger-all-in-one/Dockerfile.wolfi create mode 100755 docker-images/jaeger-all-in-one/build-wolfi.sh create mode 100644 docker-images/jaeger-all-in-one/config/sampling_strategies.json create mode 100644 docker-images/node-exporter/Dockerfile.wolfi create mode 100755 docker-images/node-exporter/build-wolfi.sh create mode 100644 docker-images/opentelemetry-collector/Dockerfile.wolfi create mode 100755 docker-images/opentelemetry-collector/build-wolfi.sh create mode 100644 docker-images/postgres-12-alpine/Dockerfile.wolfi create mode 100755 docker-images/postgres-12-alpine/build-wolfi.sh create mode 100755 docker-images/postgres-12-alpine/rootfs/postgres-wolfi.sh create mode 100644 docker-images/postgres_exporter/Dockerfile.wolfi create mode 100755 docker-images/postgres_exporter/build-wolfi.sh create mode 100755 docker-images/prometheus-gcp/build-wolfi.sh create mode 100644 docker-images/prometheus/Dockerfile.wolfi create mode 100755 docker-images/prometheus/build-wolfi.sh create mode 100644 docker-images/redis-cache/Dockerfile.wolfi create mode 100755 docker-images/redis-cache/build-wolfi.sh create mode 100644 docker-images/redis-store/Dockerfile.wolfi create mode 100755 docker-images/redis-store/build-wolfi.sh create mode 100644 docker-images/redis_exporter/Dockerfile.wolfi create mode 100755 docker-images/redis_exporter/build-wolfi.sh create mode 100644 docker-images/search-indexer/Dockerfile.wolfi create mode 100755 docker-images/search-indexer/build-wolfi.sh create mode 100644 docker-images/sg/Dockerfile.wolfi create mode 100755 docker-images/sg/build-wolfi.sh create mode 100644 docker-images/syntax-highlighter/Dockerfile.wolfi create mode 100755 docker-images/syntax-highlighter/build-wolfi.sh create mode 100644 wolfi-images/blobstore.yaml create mode 100644 wolfi-images/cadvisor.yaml create mode 100644 wolfi-images/jaeger-agent.yaml create mode 100644 wolfi-images/jaeger-all-in-one.yaml create mode 100644 wolfi-images/node-exporter.yaml create mode 100644 wolfi-images/opentelemetry-collector.yaml create mode 100644 wolfi-images/postgres-exporter.yaml create mode 100644 wolfi-images/postgresql-12.yaml create mode 100644 wolfi-images/redis-exporter.yaml create mode 100644 wolfi-images/redis.yaml create mode 100644 wolfi-images/search-indexer.yaml create mode 100644 wolfi-images/syntax-highlighter.yaml create mode 100644 wolfi-packages/.gitignore create mode 100644 wolfi-packages/cadvisor.yaml create mode 100644 wolfi-packages/jaeger.yaml create mode 100644 wolfi-packages/opentelemetry-collector.yaml create mode 100644 wolfi-packages/opentelemetry-collector/builder.template.yaml create mode 100644 wolfi-packages/redis_exporter.yaml diff --git a/cmd/worker/Dockerfile.wolfi b/cmd/worker/Dockerfile.wolfi new file mode 100644 index 0000000000000..4dac697458531 --- /dev/null +++ b/cmd/worker/Dockerfile.wolfi @@ -0,0 +1,20 @@ +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +EXPOSE 3189 + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/worker"] +COPY worker /usr/local/bin/ diff --git a/cmd/worker/build-wolfi.sh b/cmd/worker/build-wolfi.sh new file mode 100755 index 0000000000000..f339340cf3094 --- /dev/null +++ b/cmd/worker/build-wolfi.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script builds the worker docker image. + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/cmd/worker" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +echo "--- docker build" +docker build -f cmd/worker/Dockerfile -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/blobstore/Dockerfile.wolfi b/docker-images/blobstore/Dockerfile.wolfi new file mode 100644 index 0000000000000..c9e702f95ef12 --- /dev/null +++ b/docker-images/blobstore/Dockerfile.wolfi @@ -0,0 +1,61 @@ +# Build s3proxy from source +# hadolint ignore=DL3022 +FROM maven:3.8.6-openjdk-11-slim AS builder + +# hadolint ignore=DL3008,DL3009 +RUN apt-get update && \ + apt-get install -y --no-install-recommends git + +RUN git clone https://github.com/sourcegraph/s3proxy /build +WORKDIR /build +RUN mvn package -DskipTests && \ + mv target/ /opt/s3proxy && \ + cp src/main/resources/run-docker-container.sh /opt/s3proxy + +# Build our final Wolfi-based image +FROM us.gcr.io/sourcegraph-dev/wolfi-blobstore-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +COPY --from=builder /opt/s3proxy /opt/s3proxy + +ENV \ + LOG_LEVEL="info" \ + S3PROXY_AUTHORIZATION="none" \ + S3PROXY_ENDPOINT="http://0.0.0.0:9000" \ + S3PROXY_IDENTITY="local-identity" \ + S3PROXY_CREDENTIAL="local-credential" \ + S3PROXY_VIRTUALHOST="" \ + S3PROXY_CORS_ALLOW_ALL="false" \ + S3PROXY_CORS_ALLOW_ORIGINS="" \ + S3PROXY_CORS_ALLOW_METHODS="" \ + S3PROXY_CORS_ALLOW_HEADERS="" \ + S3PROXY_IGNORE_UNKNOWN_HEADERS="false" \ + S3PROXY_ENCRYPTED_BLOBSTORE="" \ + S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" \ + S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" \ + S3PROXY_V4_MAX_NON_CHUNKED_REQ_SIZE=33554432 \ + JCLOUDS_PROVIDER="filesystem" \ + JCLOUDS_ENDPOINT="" \ + JCLOUDS_REGION="" \ + JCLOUDS_REGIONS="us-east-1" \ + JCLOUDS_IDENTITY="remote-identity" \ + JCLOUDS_CREDENTIAL="remote-credential" \ + JCLOUDS_KEYSTONE_VERSION="" \ + JCLOUDS_KEYSTONE_SCOPE="" \ + JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME="" \ + JCLOUDS_FILESYSTEM_BASEDIR="/data" + +RUN mkdir -p /data && chown -R sourcegraph:sourcegraph /data +USER sourcegraph + +EXPOSE 9000 +WORKDIR /opt/s3proxy +ENTRYPOINT ["/sbin/tini", "--", "/opt/s3proxy/run-docker-container.sh"] diff --git a/docker-images/blobstore/build-wolfi.sh b/docker-images/blobstore/build-wolfi.sh new file mode 100755 index 0000000000000..821a62ca8f48e --- /dev/null +++ b/docker-images/blobstore/build-wolfi.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +cd "$(dirname "${BASH_SOURCE[0]}")" +set -ex + +# Enable image build caching via CACHE=true +BUILD_CACHE="--no-cache" +if [[ "$CACHE" == "true" ]]; then + BUILD_CACHE="" +fi + +# shellcheck disable=SC2086 +docker build ${BUILD_CACHE} -f Dockerfile.wolfi -t "${IMAGE:-"sourcegraph/blobstore"}" . \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/cadvisor/Dockerfile.wolfi b/docker-images/cadvisor/Dockerfile.wolfi new file mode 100644 index 0000000000000..d5a12137ac314 --- /dev/null +++ b/docker-images/cadvisor/Dockerfile.wolfi @@ -0,0 +1,30 @@ +# TODO: Experimental cAdvisor Dockerfile. Entirely untested, and may require additional libraries + +# NOTE: Check the README before updating +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest +LABEL com.sourcegraph.cadvisor.version=v0.47.0 + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +# Reflects cAdvisor Dockerfile at https://github.com/google/cadvisor/blob/v0.45.0/deploy/Dockerfile +# alongside additional Sourcegraph defaults. +ENTRYPOINT ["/usr/bin/cadvisor", "-logtostderr", \ + # sourcegraph cAdvisor custom port + "-port=48080", \ + # For metrics categories, see: + # https://github.com/google/cadvisor/blob/master/docs/storage/prometheus.md#prometheus-container-metrics + # and cross-reference with search in the codebase to decide which categories we need + "-enable_metrics=cpu,diskIO,memory,network", \ + # Aligned 1:1 with Kubelet defaults: + # https://sourcegraph.com/github.com/google/cadvisor@v0.45.0/-/blob/deploy/kubernetes/overlays/examples/cadvisor-args.yaml + "-docker_only", \ + "-housekeeping_interval=10s", \ + "-max_housekeeping_interval=15s", \ + "-event_storage_event_limit=default=0", \ + "-event_storage_age_limit=default=0"] diff --git a/docker-images/cadvisor/build-wolfi.sh b/docker-images/cadvisor/build-wolfi.sh new file mode 100755 index 0000000000000..ab75ab0fc0b9f --- /dev/null +++ b/docker-images/cadvisor/build-wolfi.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +cd "$(dirname "${BASH_SOURCE[0]}")" +set -ex + +docker build --no-cache -f Dockerfile.wolfi -t "${IMAGE:-"sourcegraph/cadvisor"}" . \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/codeinsights-db/build-wolfi.sh b/docker-images/codeinsights-db/build-wolfi.sh new file mode 100755 index 0000000000000..73949926e050d --- /dev/null +++ b/docker-images/codeinsights-db/build-wolfi.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +# This image is identical to our "sourcegraph/postgres-12-alpine" image, +# but runs with a different uid to avoid migration issues +IMAGE="${IMAGE:-sourcegraph/codeinsights-db}" POSTGRES_UID=70 PING_UID=700 ../postgres-12-alpine/build-wolfi.sh diff --git a/docker-images/codeintel-db/build-wolfi.sh b/docker-images/codeintel-db/build-wolfi.sh new file mode 100755 index 0000000000000..edfef833c67e1 --- /dev/null +++ b/docker-images/codeintel-db/build-wolfi.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +# This image is identical to our "sourcegraph/postgres-12-alpine" image. +IMAGE="${IMAGE:-sourcegraph/codeintel-db}" ../postgres-12-alpine/build-wolfi.sh diff --git a/docker-images/indexed-searcher/Dockerfile.wolfi b/docker-images/indexed-searcher/Dockerfile.wolfi new file mode 100644 index 0000000000000..7ffa8d90038f1 --- /dev/null +++ b/docker-images/indexed-searcher/Dockerfile.wolfi @@ -0,0 +1,32 @@ +# Note: to be able to use an ARG with a FROM it has to be at the TOP of the Dockerfile +ARG ZOEKT_IMAGE="index.docker.io/sourcegraph/zoekt-webserver" +ARG ZOEKT_VERSION +FROM $ZOEKT_IMAGE:$ZOEKT_VERSION AS zoekt_upstream + +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +ENV DATA_DIR /data/index +RUN mkdir -p ${DATA_DIR} +RUN chown -R sourcegraph:sourcegraph /data + +USER sourcegraph +WORKDIR /home/sourcegraph + +COPY --from=zoekt_upstream /usr/local/bin/zoekt-webserver /usr/local/bin/ + +# zoekt-webserver has a large stable heap size (10s of gigs), and as such the +# default GOGC=100 could be better tuned. https://dave.cheney.net/tag/gogc +# In go1.18 the GC changed significantly and from experimentation we tuned it +# down from 50 to 25. +ENV GOGC=25 + +ENTRYPOINT ["/sbin/tini", "--"] +CMD zoekt-webserver -index $DATA_DIR -pprof -rpc -indexserver_proxy diff --git a/docker-images/indexed-searcher/build-wolfi.sh b/docker-images/indexed-searcher/build-wolfi.sh new file mode 100755 index 0000000000000..0300af790ac52 --- /dev/null +++ b/docker-images/indexed-searcher/build-wolfi.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +ZOEKT_VERSION=$(go mod edit -print | awk '/sourcegraph\/zoekt/ {print substr($2, 2)}') + +docker build --no-cache -f Dockerfile.wolfi -t "${IMAGE:-"sourcegraph/indexed-searcher"}" . \ + --progress=plain \ + --build-arg ZOEKT_VERSION="$ZOEKT_VERSION" \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/jaeger-agent/Dockerfile.wolfi b/docker-images/jaeger-agent/Dockerfile.wolfi new file mode 100644 index 0000000000000..730a6e3c48cf0 --- /dev/null +++ b/docker-images/jaeger-agent/Dockerfile.wolfi @@ -0,0 +1,13 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-jaeger-agent-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.jaeger.version=${JAEGER_VERSION} + +EXPOSE 5775/udp 6831/udp 6832/udp 5778 +ENTRYPOINT ["/usr/local/bin/jaeger-agent"] diff --git a/docker-images/jaeger-agent/build-wolfi.sh b/docker-images/jaeger-agent/build-wolfi.sh new file mode 100755 index 0000000000000..66eccb287b5f2 --- /dev/null +++ b/docker-images/jaeger-agent/build-wolfi.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +IMAGE=${IMAGE:-sourcegraph/jaeger-agent} + +docker build --no-cache -f Dockerfile.wolfi -t "${IMAGE}" . \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/jaeger-all-in-one/Dockerfile.wolfi b/docker-images/jaeger-all-in-one/Dockerfile.wolfi new file mode 100644 index 0000000000000..1bf1c4e0d5122 --- /dev/null +++ b/docker-images/jaeger-all-in-one/Dockerfile.wolfi @@ -0,0 +1,47 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-jaeger-all-in-one-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.jaeger.version=${JAEGER_VERSION} + +COPY ./config/sampling_strategies.json /etc/jaeger/sampling_strategies.json + +# # Ensure the /tmp directory is chown'd to user jaeger +RUN chown -R jaeger /tmp +USER jaeger +VOLUME ["/tmp"] + +# Agent zipkin.thrift compact +EXPOSE 5775/udp + +# Agent jaeger.thrift compact +EXPOSE 6831/udp + +# Agent jaeger.thrift binary +EXPOSE 6832/udp + +# Agent config HTTP +EXPOSE 5778 + +# Collector HTTP +EXPOSE 14268 + +# Collector gRPC +EXPOSE 14250 + +# Web HTTP +EXPOSE 16686 + +# Used in order to reverse proxy the Jaeger UI +ENV QUERY_BASE_PATH="/-/debug/jaeger" + +# Default configuration file for setting sampling strategies, we override the command in docker-compose +ENV SAMPLING_STRATEGIES_FILE=/etc/jaeger/sampling_strategies.json + +ENTRYPOINT ["/usr/local/bin/jaeger-all-in-one"] +CMD ["--sampling.strategies-file=/etc/jaeger/sampling_strategies.json"] diff --git a/docker-images/jaeger-all-in-one/build-wolfi.sh b/docker-images/jaeger-all-in-one/build-wolfi.sh new file mode 100755 index 0000000000000..b53f565047821 --- /dev/null +++ b/docker-images/jaeger-all-in-one/build-wolfi.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +IMAGE=${IMAGE:-sourcegraph/jaeger-all-in-one} + +docker build --no-cache -f Dockerfile.wolfi -t "${IMAGE}" . \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/jaeger-all-in-one/config/sampling_strategies.json b/docker-images/jaeger-all-in-one/config/sampling_strategies.json new file mode 100644 index 0000000000000..cbad08349a8a1 --- /dev/null +++ b/docker-images/jaeger-all-in-one/config/sampling_strategies.json @@ -0,0 +1,6 @@ +{ + "default_strategy": { + "type": "probabilistic", + "param": 1 + } +} diff --git a/docker-images/node-exporter/Dockerfile.wolfi b/docker-images/node-exporter/Dockerfile.wolfi new file mode 100644 index 0000000000000..a484ddc3d4844 --- /dev/null +++ b/docker-images/node-exporter/Dockerfile.wolfi @@ -0,0 +1,15 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-node-exporter-base:latest +# hadolint ignore=DL3048 +LABEL com.sourcegraph.node_exporter.version=v1.5.0 + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +EXPOSE 9100 +USER nobody +ENTRYPOINT [ "/usr/bin/node_exporter" ] diff --git a/docker-images/node-exporter/build-wolfi.sh b/docker-images/node-exporter/build-wolfi.sh new file mode 100755 index 0000000000000..89631182d2cd7 --- /dev/null +++ b/docker-images/node-exporter/build-wolfi.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +cd "$(dirname "${BASH_SOURCE[0]}")" +set -ex + +docker build -f ./Dockerfile.wolfi -t "${IMAGE:-sourcegraph/node-exporter}" . \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/opentelemetry-collector/Dockerfile.wolfi b/docker-images/opentelemetry-collector/Dockerfile.wolfi new file mode 100644 index 0000000000000..13f7da3d44ecb --- /dev/null +++ b/docker-images/opentelemetry-collector/Dockerfile.wolfi @@ -0,0 +1,20 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-opentelemetry-collector-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +# Set up a home directory +RUN mkdir -p /otel-collector +WORKDIR /otel-collector + +# TODO: Alpine image runs as root - required? + +# Set up bundled configuration - see README +COPY ./configs /etc/otel-collector/configs + +ENTRYPOINT [ "/usr/bin/otelcol-sourcegraph" ] diff --git a/docker-images/opentelemetry-collector/build-wolfi.sh b/docker-images/opentelemetry-collector/build-wolfi.sh new file mode 100755 index 0000000000000..560126e998c85 --- /dev/null +++ b/docker-images/opentelemetry-collector/build-wolfi.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/opentelemetry-collector}" . \ + --platform linux/amd64 \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/postgres-12-alpine/Dockerfile.wolfi b/docker-images/postgres-12-alpine/Dockerfile.wolfi new file mode 100644 index 0000000000000..ac6f2708aada3 --- /dev/null +++ b/docker-images/postgres-12-alpine/Dockerfile.wolfi @@ -0,0 +1,27 @@ +# Are you bumping postgres minor or major version? +# Please review the changes in /usr/local/share/postgresql/postgresql.conf.sample +# If there is any change, you should ping @team/delivery +# And Delivery will make sure changes are reflected in our deploy repository + +# Dockerfile for Wolfi-based images +# This is currently being tested in parallel to Alpine - you don't need to update this +# file if you change the regular Dockerfile. + +FROM us.gcr.io/sourcegraph-dev/wolfi-postgresql-12-base:latest + +# To remain compatibility with codeinsights-db and codeintel-db, user and group +# IDs are set here, rather than in the base image + +ARG PING_UID=99 +ARG POSTGRES_UID=999 + +# We modify the postgres user/group to reconcile with our previous debian based images +# and avoid issues with customers migrating. +RUN addgroup -g $PING_UID ping &&\ + adduser -D -u $POSTGRES_UID postgres postgres &&\ + mkdir -p /data/pgdata-12 && chown -R postgres:postgres /data + +COPY rootfs / + +USER postgres +ENTRYPOINT ["/postgres-wolfi.sh"] diff --git a/docker-images/postgres-12-alpine/build-wolfi.sh b/docker-images/postgres-12-alpine/build-wolfi.sh new file mode 100755 index 0000000000000..e01f277198c0c --- /dev/null +++ b/docker-images/postgres-12-alpine/build-wolfi.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +POSTGRES_UID=${POSTGRES_UID:-999} +PING_UID=${PING_UID:-99} + +docker build -t "${IMAGE:-index.docker.io/sourcegraph/wolfi-postgres-12}" --build-arg POSTGRES_UID="$POSTGRES_UID" --build-arg PING_UID="$PING_UID" -f ./Dockerfile.wolfi . diff --git a/docker-images/postgres-12-alpine/rootfs/postgres-wolfi.sh b/docker-images/postgres-12-alpine/rootfs/postgres-wolfi.sh new file mode 100755 index 0000000000000..ad1ee3ea287ba --- /dev/null +++ b/docker-images/postgres-12-alpine/rootfs/postgres-wolfi.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euxo pipefail +cd / + +# shellcheck source=./env.sh +source /env.sh + +# Allow the container to be started with root in Kubernetes and change permissions +# of the parent volume directory to be owned entirely by the postgres user. +if [ "$(id -u)" = '0' ]; then + mkdir -p "$PGDATA" + chown -R postgres:postgres "$(dirname "$PGDATA")" + chmod 750 "$(dirname "$PGDATA")" "$PGDATA" + su-exec postgres "${BASH_SOURCE[0]}" "$@" +fi + +if [ ! -s "$PGDATA/PG_VERSION" ]; then + echo "[INFO] Initializing Postgres database '$POSTGRES_DB' from scratch in $PGDATA" + /initdb.sh +fi + +/conf.sh + +if [ ! -s "${REINDEX_COMPLETED_FILE}" ]; then + echo "[INFO] Re-creating all indexes for database '$POSTGRES_DB'" + /reindex.sh +fi + +exec postgres diff --git a/docker-images/postgres_exporter/Dockerfile.wolfi b/docker-images/postgres_exporter/Dockerfile.wolfi new file mode 100644 index 0000000000000..289225ffabfcf --- /dev/null +++ b/docker-images/postgres_exporter/Dockerfile.wolfi @@ -0,0 +1,20 @@ + +FROM us.gcr.io/sourcegraph-dev/wolfi-postgres-exporter-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +USER postgres_exporter + +COPY ./*.yaml /config/ +ENV PG_EXPORTER_EXTEND_QUERY_PATH=/config/queries.yaml + +EXPOSE 9187 + +# TODO: Error 'Error opening config file "postgres_exporter.yml"' occurs at startup in v0.12.0 - confirm this is working as expected +ENTRYPOINT [ "/usr/bin/postgres_exporter"] diff --git a/docker-images/postgres_exporter/build-wolfi.sh b/docker-images/postgres_exporter/build-wolfi.sh new file mode 100755 index 0000000000000..fb01de6a8c60c --- /dev/null +++ b/docker-images/postgres_exporter/build-wolfi.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +cd "$(dirname "${BASH_SOURCE[0]}")" +set -ex + +OUTPUT=$(mktemp -d -t sgpostgres_exporter_XXXXXXX) +export OUTPUT +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +mkdir -p "${OUTPUT}" +OUTPUT_FILE="${OUTPUT}/queries.yaml" +CODEINTEL_OUTPUT_FILE="${OUTPUT}/code_intel_queries.yaml" +CODEINSIGHTS_OUTPUT_FILE="${OUTPUT}/code_insights_queries.yaml" + +for source in ./config/*.yaml; do + { + if [[ "$source" == *"codeintel"* || "$source" == *"codeinsights"* ]]; then + echo "# skipping $source" + continue + fi + echo "# source: ${source}" + cat "$source" + echo "" + } >>"${OUTPUT_FILE}" +done + +for source in ./config/*.yaml; do + { + if [[ "$source" == *"frontend"* || "$source" == *"codeinsights"* ]]; then + echo "# skipping $source" + continue + fi + echo "# source: ${source}" + cat "$source" + echo "" + } >>"${CODEINTEL_OUTPUT_FILE}" +done + +for source in ./config/*.yaml; do + { + if [[ "$source" == *"frontend"* || "$source" == *"codeintel"* ]]; then + echo "# skipping $source" + continue + fi + echo "# source: ${source}" + cat "$source" + echo "" + } >>"${CODEINSIGHTS_OUTPUT_FILE}" +done + +echo "${OUTPUT_FILE}" +echo "${CODEINTEL_OUTPUT_FILE}" +echo "${CODEINSIGHTS_OUTPUT_FILE}" + +docker build -f ./Dockerfile.wolfi -t "${IMAGE:-sourcegraph/postgres_exporter}" "${OUTPUT}" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/prometheus-gcp/build-wolfi.sh b/docker-images/prometheus-gcp/build-wolfi.sh new file mode 100755 index 0000000000000..6c35e3b4aefa4 --- /dev/null +++ b/docker-images/prometheus-gcp/build-wolfi.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +export BASE_IMAGE="gke.gcr.io/prometheus-engine/prometheus:v2.35.0-gmp.2-gke.0" +export IMAGE="${IMAGE:-sourcegraph/prometheus-gcp}" + +../prometheus/build-wolfi.sh diff --git a/docker-images/prometheus/Dockerfile.wolfi b/docker-images/prometheus/Dockerfile.wolfi new file mode 100644 index 0000000000000..e5fced716ab14 --- /dev/null +++ b/docker-images/prometheus/Dockerfile.wolfi @@ -0,0 +1,75 @@ +# sourcegraph/prometheus - learn more about this image in https://docs.sourcegraph.com/dev/background-information/observability/prometheus + +# Note: to be able to use an ARG with a FROM it has to be at the TOP of the Dockerfile +# This allows the base image to be substituted for a GCP image that ships metrics to managed Prometheus. Default base image is regular upstream Prometheus +# https://cloud.google.com/stackdriver/docs/managed-prometheus/setup-unmanaged#run-gmp +# To upgrade Prometheus or Alertmanager, see https://docs.sourcegraph.com/dev/background-information/observability/prometheus#upgrading-prometheus-or-alertmanager +ARG BASE_IMAGE="prom/prometheus:v2.42.0@sha256:5689a4360cf1479bbad944b1311a90b495546c498c46f48e44fa7e5ac6412191" +# https://github.com/hadolint/hadolint/issues/339 +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE} AS prom_upstream +FROM prom/alertmanager:v0.25.0@sha256:db8303fa05341f5dc6b19b36a97325cd1b8307254ed9042a2c554af71f3c0284 AS am_upstream + +# Build monitoring definitions +FROM cgr.dev/chainguard/bash:latest AS monitoring_builder +RUN mkdir -p '/generated/prometheus' +COPY ./.bin/monitoring-generator /bin/monitoring-generator +RUN PROMETHEUS_DIR='/generated/prometheus' GRAFANA_DIR='' DOCS_DIR='' NO_PRUNE=true /bin/monitoring-generator +RUN ls '/generated/prometheus' + +# Prepare final image +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +# Should reflect versions above +LABEL com.sourcegraph.prometheus.version=v2.42.0 +LABEL com.sourcegraph.alertmanager.version=v0.25.0 + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +# Prometheus - extended from https://github.com/prometheus/prometheus/blob/VERSION/Dockerfile +# Check the upstream image (replacing VERSION with the appropriate Prometheus version) when upgrading +# TODO: Check directories in sourcegraph-base +COPY --from=prom_upstream /bin/prometheus /bin/prometheus +COPY --from=prom_upstream /bin/promtool /bin/promtool +COPY --from=prom_upstream /etc/prometheus/prometheus.yml /etc/prometheus/prometheus.yml +COPY --from=prom_upstream /usr/share/prometheus/console_libraries/ /usr/share/prometheus/console_libraries/ +COPY --from=prom_upstream /usr/share/prometheus/consoles/ /usr/share/prometheus/consoles/ +COPY --from=prom_upstream /LICENSE /LICENSE +COPY --from=prom_upstream /NOTICE /NOTICE +# hadolint ignore=DL3010 +COPY --from=prom_upstream /npm_licenses.tar.bz2 /npm_licenses.tar.bz2 + +# Alertmanager - extended from https://github.com/prometheus/alertmanager/blob/VERSION/Dockerfile +# Check the upstream image (replacing VERSION with the appropriate Prometheus version) when upgrading +COPY --from=am_upstream /bin/alertmanager /bin/alertmanager + +RUN ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/ + + +# Add required directories and switch to sourcegraph user +RUN mkdir -p /prometheus && chown -R sourcegraph:sourcegraph /prometheus +RUN mkdir -p /alertmanager && chown -R sourcegraph:sourcegraph /alertmanager +USER sourcegraph + +COPY ./.bin/prom-wrapper /bin/prom-wrapper +COPY ./prometheus.sh /prometheus.sh +COPY ./alertmanager.sh /alertmanager.sh + +# Copy config +COPY --from=monitoring_builder /generated/prometheus/* /sg_config_prometheus/ +COPY config/*_rules.yml /sg_config_prometheus/ +COPY config/prometheus.yml /sg_config_prometheus/ +COPY config/alertmanager.yml /sg_config_prometheus/ + +ENTRYPOINT ["/bin/prom-wrapper"] +# Note that upstream's 'VOLUME' directive was deliberately removed. Including it makes it impossible +# to chmod the directory to our 'sourcegraph' user. +WORKDIR /prometheus +# Prometheus is reverse-proxied from 9092 to 9090 +EXPOSE 9090 diff --git a/docker-images/prometheus/build-wolfi.sh b/docker-images/prometheus/build-wolfi.sh new file mode 100755 index 0000000000000..dedc029352669 --- /dev/null +++ b/docker-images/prometheus/build-wolfi.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -ex + +cd "$(dirname "${BASH_SOURCE[0]}")" + +# We build out of tree to prevent triggering dev watch scripts when we copy go +# files. +BUILDDIR=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$BUILDDIR" +} +trap cleanup EXIT + +# Copy assets +cp -R . "$BUILDDIR" + +# Build args for Go cross-compilation. +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +# Cross-compile prom-wrapper before building the image. +go build \ + -trimpath \ + -installsuffix netgo \ + -tags "dist netgo" \ + -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" \ + -o "$BUILDDIR"/.bin/prom-wrapper ./cmd/prom-wrapper + +# Cross-compile monitoring generator before building the image. +pushd "../../monitoring" +go build \ + -trimpath \ + -o "$BUILDDIR"/.bin/monitoring-generator . + +# Final pre-build stage. +pushd "$BUILDDIR" + +# Note: This chmod is so that both the `sourcegraph` user and host system user (what `whoami` reports on +# Linux) both have access to the files in the container AND files mounted by `-v` into the container without it +# running as root. For more details, see: +# https://github.com/sourcegraph/sourcegraph/pull/11832#discussion_r451109637 +chmod -R 777 config + +# Enable image build caching via CACHE=true +BUILD_CACHE="--no-cache" +if [[ "$CACHE" == "true" ]]; then + BUILD_CACHE="" +fi + +# shellcheck disable=SC2086 +docker build ${BUILD_CACHE} -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/prometheus}" . \ + --progress=plain \ + --build-arg BASE_IMAGE \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + +# cd out of $BUILDDIR for cleanup +popd diff --git a/docker-images/redis-cache/Dockerfile.wolfi b/docker-images/redis-cache/Dockerfile.wolfi new file mode 100644 index 0000000000000..b10df6143716d --- /dev/null +++ b/docker-images/redis-cache/Dockerfile.wolfi @@ -0,0 +1,5 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-redis-base:latest + +COPY redis.conf /etc/redis/redis.conf + +ENTRYPOINT ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker-images/redis-cache/build-wolfi.sh b/docker-images/redis-cache/build-wolfi.sh new file mode 100755 index 0000000000000..dbbf9584aee96 --- /dev/null +++ b/docker-images/redis-cache/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +docker build -t "${IMAGE:-sourcegraph/redis-cache}" -f Dockerfile.wolfi . diff --git a/docker-images/redis-store/Dockerfile.wolfi b/docker-images/redis-store/Dockerfile.wolfi new file mode 100644 index 0000000000000..b10df6143716d --- /dev/null +++ b/docker-images/redis-store/Dockerfile.wolfi @@ -0,0 +1,5 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-redis-base:latest + +COPY redis.conf /etc/redis/redis.conf + +ENTRYPOINT ["redis-server", "/etc/redis/redis.conf"] diff --git a/docker-images/redis-store/build-wolfi.sh b/docker-images/redis-store/build-wolfi.sh new file mode 100755 index 0000000000000..988dfc8cb7eda --- /dev/null +++ b/docker-images/redis-store/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +docker build -t "${IMAGE:-sourcegraph/redis-store}" -f Dockerfile.wolfi . diff --git a/docker-images/redis_exporter/Dockerfile.wolfi b/docker-images/redis_exporter/Dockerfile.wolfi new file mode 100644 index 0000000000000..08e5d9fff079b --- /dev/null +++ b/docker-images/redis_exporter/Dockerfile.wolfi @@ -0,0 +1,6 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-redis-exporter-base:latest + +USER sourcegraph +EXPOSE 9121 + +ENTRYPOINT [ "/usr/local/bin/redis_exporter" ] diff --git a/docker-images/redis_exporter/build-wolfi.sh b/docker-images/redis_exporter/build-wolfi.sh new file mode 100755 index 0000000000000..af749ff543c11 --- /dev/null +++ b/docker-images/redis_exporter/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/redis_exporter}" . diff --git a/docker-images/search-indexer/Dockerfile.wolfi b/docker-images/search-indexer/Dockerfile.wolfi new file mode 100644 index 0000000000000..87853a2c6d925 --- /dev/null +++ b/docker-images/search-indexer/Dockerfile.wolfi @@ -0,0 +1,31 @@ +# Note: to be able to use an ARG with a FROM it has to be at the TOP of the Dockerfile +ARG ZOEKT_IMAGE="index.docker.io/sourcegraph/zoekt-indexserver" +ARG ZOEKT_VERSION +FROM $ZOEKT_IMAGE:$ZOEKT_VERSION AS zoekt_upstream + +FROM us.gcr.io/sourcegraph-dev/wolfi-search-indexer-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} + +ENV SRC_FRONTEND_INTERNAL http://sourcegraph-frontend-internal +ENV DATA_DIR /data/index +RUN mkdir -p ${DATA_DIR} +RUN chown -R sourcegraph:sourcegraph /data + +USER sourcegraph +WORKDIR /home/sourcegraph + +COPY --from=zoekt_upstream \ + /usr/local/bin/zoekt-sourcegraph-indexserver \ + /usr/local/bin/zoekt-archive-index \ + /usr/local/bin/zoekt-git-index \ + /usr/local/bin/zoekt-merge-index \ + /usr/local/bin/ + +ENTRYPOINT ["/sbin/tini", "--", "zoekt-sourcegraph-indexserver"] diff --git a/docker-images/search-indexer/build-wolfi.sh b/docker-images/search-indexer/build-wolfi.sh new file mode 100755 index 0000000000000..d7dcb4dfc41ab --- /dev/null +++ b/docker-images/search-indexer/build-wolfi.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +ZOEKT_VERSION=$(go mod edit -print | awk '/sourcegraph\/zoekt/ {print substr($2, 2)}') + +docker build --no-cache -f Dockerfile.wolfi -t "${IMAGE:-"sourcegraph/search-indexer"}" . \ + --progress=plain \ + --build-arg ZOEKT_VERSION="$ZOEKT_VERSION" \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/sg/Dockerfile.wolfi b/docker-images/sg/Dockerfile.wolfi new file mode 100644 index 0000000000000..1e6706df7614d --- /dev/null +++ b/docker-images/sg/Dockerfile.wolfi @@ -0,0 +1,15 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +COPY sg /usr/local/bin/ + +USER sourcegraph +ENTRYPOINT ["/usr/local/bin/sg"] diff --git a/docker-images/sg/build-wolfi.sh b/docker-images/sg/build-wolfi.sh new file mode 100755 index 0000000000000..7a38d06e13546 --- /dev/null +++ b/docker-images/sg/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +echo "--- go build" +pkg="github.com/sourcegraph/sourcegraph/dev/sg" +go build -trimpath -ldflags "-X main.BuildCommit=$BUILD_COMMIT" -o "$OUTPUT/sg" -buildmode exe "$pkg" + +echo "--- docker build $IMAGE" +docker build -f docker-images/sg/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/docker-images/syntax-highlighter/Dockerfile.wolfi b/docker-images/syntax-highlighter/Dockerfile.wolfi new file mode 100644 index 0000000000000..1f593228613d0 --- /dev/null +++ b/docker-images/syntax-highlighter/Dockerfile.wolfi @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1.3-labs + +# IMPORTANT: The Dockerfile for building syntect_server includes a `-mno-outline-atomics` +# flag for arm64 builds - this is not done for Wolfi package builds + +####################### +# Compile final image # +####################### +FROM us.gcr.io/sourcegraph-dev/wolfi-syntax-highlighter-base:latest + +EXPOSE 9238 +ENV ROCKET_ENV "production" +ENV ROCKET_LIMITS "{json=10485760}" + +# syntect_server does not need a secret key since it uses no cookies, but +# without one set Rocket emits a warning. +ENV ROCKET_SECRET_KEY "SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA=" + +# When keep-alive is on, we observe connection resets in our Go clients of +# syntect_server. It is unclear why this is, especially because our Go clients do +# not reuse the connection (i.e. we make a fresh connection every time). +# Disabling keep-alive does resolve the issue though, our best guess is that +# this is a bug in Hyper 0.10 (see https://github.com/SergioBenitez/Rocket/issues/928#issuecomment-464632953). +# See https://github.com/sourcegraph/sourcegraph/issues/2615 for details on +# what we observed when this was enabled with the default 5s. +ENV ROCKET_KEEP_ALIVE=0 + +# The more workers, the more resilient syntect_server is to getting stuck on +# bad grammar/file combinations. If it happens with four workers, only 1/4th of +# requests will be affected for a short period of time. Each worker can require +# at peak around 1.1 GiB of memory. +ENV WORKERS=4 + +ENV QUIET=true +CMD ["sh", "-c", "/usr/local/bin/http-server-stabilizer -listen=:9238 -prometheus-app-name=syntax_highlighter -workers=$WORKERS -- env ROCKET_PORT={{.Port}} /usr/local/bin/syntax_highlighter"] diff --git a/docker-images/syntax-highlighter/build-wolfi.sh b/docker-images/syntax-highlighter/build-wolfi.sh new file mode 100755 index 0000000000000..8c245555ca8ef --- /dev/null +++ b/docker-images/syntax-highlighter/build-wolfi.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -ex +cd "$(dirname "${BASH_SOURCE[0]}")" + +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/syntax-highlighter}" . diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index 09fcdc294524d..70b3965ebab2b 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -121,7 +121,24 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{"gitserver", "frontend", "github-proxy", "loadtest", "migrator", "repo-updater", "searcher", "batcheshelper", "precise-code-intel-worker", "worker", "symbols"}, c.Version, c.candidateImageTag()), + BuildWolfiOperations([]string{ + "blobstore", + "cadvisor", + "codeinsights-db", + "codeintel-db", + "indexed-searcher", + "node-exporter", + "opentelemetry-collector", + "postgres-12-alpine", + "prometheus", + "prometheus-gcp", + "redis-cache", + "redis-store", + "redis_exporter", + "search-indexer", + "sg", + "syntax-highlighter", + }, c.Version, c.candidateImageTag()), ) case runtype.PullRequest: diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 2c39d07130d69..826f3aa9477a3 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -37,7 +37,6 @@ func WolfiBaseImagesOperations(changedFiles []string, tag string, packagesChange func WolfiPackagesOperations(changedFiles []string) *operations.Set { // TODO: Should we require the image name, or the full path to the yaml file? ops := operations.NewNamedSet("Dependency packages") - logger := log.Scoped("gen-pipeline", "generates the pipeline for ci") var stepKeys []string for _, c := range changedFiles { @@ -46,8 +45,6 @@ func WolfiPackagesOperations(changedFiles []string) *operations.Set { buildFunc, key := buildPackage(match[1]) stepKeys = append(stepKeys, key) ops.Append(buildFunc) - } else { - logger.Fatal(fmt.Sprintf("Unable to extract package name from '%s', matches were %+v\n", c, match)) } } diff --git a/enterprise/dev/ci/scripts/wolfi/build-base-image.sh b/enterprise/dev/ci/scripts/wolfi/build-base-image.sh index a28d7c662102f..1198d05d2bc65 100755 --- a/enterprise/dev/ci/scripts/wolfi/build-base-image.sh +++ b/enterprise/dev/ci/scripts/wolfi/build-base-image.sh @@ -61,3 +61,6 @@ docker load <"$tarball" docker tag "$image_name" "us.gcr.io/sourcegraph-dev/wolfi-${name}-base:$tag" docker push "us.gcr.io/sourcegraph-dev/wolfi-${name}-base:$tag" +# Temporary convenience during initial development, as this doesn't scale to multiple branches! +docker tag "$image_name" "us.gcr.io/sourcegraph-dev/wolfi-${name}-base:latest" +docker push "us.gcr.io/sourcegraph-dev/wolfi-${name}-base:latest" diff --git a/wolfi-images/blobstore.yaml b/wolfi-images/blobstore.yaml new file mode 100644 index 0000000000000..7b6adf19d54b4 --- /dev/null +++ b/wolfi-images/blobstore.yaml @@ -0,0 +1,39 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + repositories: + - https://packages.wolfi.dev/os + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## blobstore packages + - openjdk-11 + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/cadvisor.yaml b/wolfi-images/cadvisor.yaml new file mode 100644 index 0000000000000..8beb49704b5cf --- /dev/null +++ b/wolfi-images/cadvisor.yaml @@ -0,0 +1,41 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## cadvisor dependencies + - cadvisor@sourcegraph + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/gitserver.yaml b/wolfi-images/gitserver.yaml index 294308030e376..17005a058ed06 100644 --- a/wolfi-images/gitserver.yaml +++ b/wolfi-images/gitserver.yaml @@ -4,7 +4,7 @@ contents: - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: ## Base set of packages included in sourcegraph/alpine base image - wolfi-baselayout diff --git a/wolfi-images/jaeger-agent.yaml b/wolfi-images/jaeger-agent.yaml new file mode 100644 index 0000000000000..0e49c07fc14ed --- /dev/null +++ b/wolfi-images/jaeger-agent.yaml @@ -0,0 +1,43 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + - wolfi-baselayout # replaces alpine-baselayout-data + - ca-certificates-bundle + - tzdata + # Included by existing SG base image + - tini + - mailcap + # These are dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## jaeger-agent-specific tools + - jaeger-agent@sourcegraph + +# Run as jaeger +accounts: + run-as: 10001 + groups: + - groupname: jaeger + gid: 10002 + users: + - username: jaeger + uid: 10001 + gid: 10002 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/jaeger-all-in-one.yaml b/wolfi-images/jaeger-all-in-one.yaml new file mode 100644 index 0000000000000..15e2ca3564f73 --- /dev/null +++ b/wolfi-images/jaeger-all-in-one.yaml @@ -0,0 +1,42 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + - wolfi-baselayout # replaces alpine-baselayout-data + - ca-certificates-bundle + - tzdata + # Included by existing SG base image + - tini + - mailcap + # These are dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## jaeger-all-in-one-specific tools + - jaeger-all-in-one@sourcegraph + +# Add jaeger user +accounts: + groups: + - groupname: jaeger + gid: 10002 + users: + - username: jaeger + uid: 10001 + gid: 10002 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/node-exporter.yaml b/wolfi-images/node-exporter.yaml new file mode 100644 index 0000000000000..3c024a72a2ea2 --- /dev/null +++ b/wolfi-images/node-exporter.yaml @@ -0,0 +1,32 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + repositories: + - https://packages.wolfi.dev/os + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + + ## node-exporter-specific packages + - 'prometheus-node-exporter=1.5.0-r3' # IMPORTANT: Pinned version for managed updates + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/opentelemetry-collector.yaml b/wolfi-images/opentelemetry-collector.yaml new file mode 100644 index 0000000000000..2e546194cfd7e --- /dev/null +++ b/wolfi-images/opentelemetry-collector.yaml @@ -0,0 +1,43 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + - wolfi-baselayout # replaces alpine-baselayout-data + - ca-certificates-bundle + - tzdata + # Included by existing SG base image + - tini + - mailcap + # These are dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## opentelemetry-collector-specific packages + - opentelemetry-collector@sourcegraph + +# Run as root +accounts: + run-as: 0 + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/postgres-exporter.yaml b/wolfi-images/postgres-exporter.yaml new file mode 100644 index 0000000000000..d6867b7095acc --- /dev/null +++ b/wolfi-images/postgres-exporter.yaml @@ -0,0 +1,39 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + repositories: + - https://packages.wolfi.dev/os + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + # Debugging tools, can be removed + - busybox + + ## postgres-exporter-specific packages + - 'prometheus-postgres-exporter=0.12.0-r1' # IMPORTANT: Pinned version for managed updates + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + - groupname: postgres_exporter + gid: 102 + users: + - username: sourcegraph + uid: 100 + gid: 101 + - username: postgres_exporter + uid: 20001 + gid: 102 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/postgresql-12.yaml b/wolfi-images/postgresql-12.yaml new file mode 100644 index 0000000000000..627f8aac173ef --- /dev/null +++ b/wolfi-images/postgresql-12.yaml @@ -0,0 +1,33 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - ca-certificates-bundle + - wolfi-baselayout + - busybox + - su-exec + - postgresql-12 + - postgresql-12-client + - postgresql-12-oci-entrypoint + - postgresql-12-contrib + - libpq-12 + +# TODO: Currently missing shadow package which would let us modify users and groups in the Dockerfile +# In the future, we can use accounts: and paths: directives to do that in this image + +accounts: + run-as: root + +entrypoint: + command: /var/lib/postgres/initdb/postgresql-entrypoint.sh postgres + +environment: + PGDATA: /data/pgdata-12 + POSTGRES_USER: sg + POSTGRES_PASSWORD: '' + POSTGRES_DB: sg + +archs: + - x86_64 diff --git a/wolfi-images/redis-exporter.yaml b/wolfi-images/redis-exporter.yaml new file mode 100644 index 0000000000000..104e0c9ffa69c --- /dev/null +++ b/wolfi-images/redis-exporter.yaml @@ -0,0 +1,41 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## redis_exporter packages + - redis_exporter@sourcegraph + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/redis.yaml b/wolfi-images/redis.yaml new file mode 100644 index 0000000000000..90115e828647d --- /dev/null +++ b/wolfi-images/redis.yaml @@ -0,0 +1,34 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + repositories: + - https://packages.wolfi.dev/os + packages: + - redis + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: redis + gid: 65532 + users: + - username: redis + uid: 65532 + run-as: redis + +paths: + - path: /redis-data + type: directory + uid: 65532 + gid: 65532 + permissions: 0o755 + +work-dir: + /redis-data + +entrypoint: + command: redis-server + +archs: +- x86_64 diff --git a/wolfi-images/repo-updater.yaml b/wolfi-images/repo-updater.yaml index 34b5ee2b95463..8730025295369 100644 --- a/wolfi-images/repo-updater.yaml +++ b/wolfi-images/repo-updater.yaml @@ -4,7 +4,7 @@ contents: - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: ## Base set of packages included in sourcegraph/alpine base image - wolfi-baselayout diff --git a/wolfi-images/search-indexer.yaml b/wolfi-images/search-indexer.yaml new file mode 100644 index 0000000000000..15b0e311787a1 --- /dev/null +++ b/wolfi-images/search-indexer.yaml @@ -0,0 +1,44 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## search-indexer-specific packages + - git + - jansson + - ctags@sourcegraph + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-images/searcher.yaml b/wolfi-images/searcher.yaml index 5cc2e8bc0d450..d36f9bc6e5067 100644 --- a/wolfi-images/searcher.yaml +++ b/wolfi-images/searcher.yaml @@ -4,7 +4,7 @@ contents: - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: ## Base set of packages included in sourcegraph/alpine base image - wolfi-baselayout diff --git a/wolfi-images/server.yaml b/wolfi-images/server.yaml index a1c0676463111..675309974f2e8 100644 --- a/wolfi-images/server.yaml +++ b/wolfi-images/server.yaml @@ -4,7 +4,7 @@ contents: - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: ## Base set of packages included in sourcegraph/alpine base image - wolfi-baselayout diff --git a/wolfi-images/sourcegraph-dev.yaml b/wolfi-images/sourcegraph-dev.yaml index 5915279f31b43..a471825dd400d 100644 --- a/wolfi-images/sourcegraph-dev.yaml +++ b/wolfi-images/sourcegraph-dev.yaml @@ -1,9 +1,10 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: - wolfi-baselayout # replaces alpine-baselayout-data - ca-certificates-bundle @@ -23,6 +24,13 @@ contents: # Run as root accounts: run-as: 0 + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 # NOTE: This is ignored (see build output) # To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` diff --git a/wolfi-images/symbols.yaml b/wolfi-images/symbols.yaml index deb3656592ef6..50b2d08ac6237 100644 --- a/wolfi-images/symbols.yaml +++ b/wolfi-images/symbols.yaml @@ -4,7 +4,7 @@ contents: - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub repositories: - https://packages.wolfi.dev/os - - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main/' + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' packages: ## Base set of packages - wolfi-baselayout # replaces alpine-baselayout-data diff --git a/wolfi-images/syntax-highlighter.yaml b/wolfi-images/syntax-highlighter.yaml new file mode 100644 index 0000000000000..8dcefdf61bd16 --- /dev/null +++ b/wolfi-images/syntax-highlighter.yaml @@ -0,0 +1,43 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - https://storage.googleapis.com/package-repository/packages/melange.rsa.pub + repositories: + - https://packages.wolfi.dev/os + - '@sourcegraph https://storage.googleapis.com/package-repository/packages/main' + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + + ## syntax-highlighter packages + - libstdc++ + - http-server-stabilizer@sourcegraph + - syntect-server@sourcegraph + +accounts: + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ diff --git a/wolfi-packages/.gitignore b/wolfi-packages/.gitignore new file mode 100644 index 0000000000000..a049b6f89c7a2 --- /dev/null +++ b/wolfi-packages/.gitignore @@ -0,0 +1 @@ +packages/* diff --git a/wolfi-packages/cadvisor.yaml b/wolfi-packages/cadvisor.yaml new file mode 100644 index 0000000000000..a2987cf44f4c5 --- /dev/null +++ b/wolfi-packages/cadvisor.yaml @@ -0,0 +1,36 @@ +package: + name: cadvisor + version: 0.47.0 + epoch: 0 + description: "Analyzes resource usage and performance characteristics of running containers" + target-architecture: + - x86_64 + copyright: + - paths: + - "*" + attestation: 'Copyright 2014 The cAdvisor Authors' + license: 'Apache License 2.0' + dependencies: + runtime: + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - wolfi-base + - busybox + - ca-certificates-bundle + +pipeline: + - uses: fetch + with: + uri: https://github.com/google/cadvisor/releases/download/v${{package.version}}/cadvisor-v${{package.version}}-linux-amd64 + expected-sha256: caf4491298e0702f9d0c6a1d1949767f5c6400f77e12cd3524d6d3fcc66abc2a + extract: false + - runs: | + mkdir -p ${{targets.destdir}}/usr/bin/ + chmod +x cadvisor-v${{package.version}}-linux-amd64 + cp cadvisor-v${{package.version}}-linux-amd64 ${{targets.destdir}}/usr/bin/cadvisor diff --git a/wolfi-packages/http-server-stabilizer.yaml b/wolfi-packages/http-server-stabilizer.yaml index df024412187e1..de54a8897498e 100644 --- a/wolfi-packages/http-server-stabilizer.yaml +++ b/wolfi-packages/http-server-stabilizer.yaml @@ -32,4 +32,5 @@ pipeline: - uses: go/build with: packages: main.go + prefix: /usr/local output: http-server-stabilizer diff --git a/wolfi-packages/jaeger.yaml b/wolfi-packages/jaeger.yaml new file mode 100644 index 0000000000000..5a9a42f68912f --- /dev/null +++ b/wolfi-packages/jaeger.yaml @@ -0,0 +1,56 @@ +# Melange-based replacement for Coursier +# Previously packaged in the scip-java repo + +package: + name: jaeger + version: 1.42.0 + epoch: 0 + description: "Distributed Tracing Platform" + target-architecture: + - x86_64 + copyright: + - paths: + - "*" + attestation: 'Copyright The Jaeger Authors' + license: 'Apache License 2.0' + dependencies: + runtime: + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - wolfi-base + +pipeline: + - uses: fetch + with: + uri: https://github.com/jaegertracing/jaeger/releases/download/v${{package.version}}/jaeger-${{package.version}}-linux-amd64.tar.gz + expected-sha256: fa34c3065a9a244aca60c4c52cc34e327882d149e90a4327fe92074b4bfc6584 + extract: true + strip-components: 0 + + - runs: | + set -x + mkdir -p ${{targets.destdir}}/usr/local/bin/ + mv jaeger-${{package.version}}-linux-amd64/* ${{targets.destdir}}/usr/local/bin/ + +subpackages: + - name: jaeger-agent + description: Jaeger Agent + pipeline: + - runs: | + set -x + mkdir -p "${{targets.subpkgdir}}/usr/local/bin/" + cp "${{targets.destdir}}/usr/local/bin/jaeger-agent" "${{targets.subpkgdir}}/usr/local/bin/" + - name: jaeger-all-in-one + description: Jaeger All-In-One + pipeline: + - runs: | + set -x + mkdir -p "${{targets.subpkgdir}}/usr/local/bin/" + cp "${{targets.destdir}}/usr/local/bin/jaeger-all-in-one" "${{targets.subpkgdir}}/usr/local/bin/" + diff --git a/wolfi-packages/opentelemetry-collector.yaml b/wolfi-packages/opentelemetry-collector.yaml new file mode 100644 index 0000000000000..fb356cbcbc848 --- /dev/null +++ b/wolfi-packages/opentelemetry-collector.yaml @@ -0,0 +1,46 @@ +package: + name: opentelemetry-collector + version: 0.71.0 # Keep in sync with version in go.mod + epoch: 0 + description: "Vendor-agnostic implementation on how to receive, process and export telemetry data" + target-architecture: + - x86_64 + copyright: + - paths: + - "*" + attestation: 'Copyright 2014 The open-telemetry Authors' + license: 'Apache License 2.0' + dependencies: + runtime: + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - wolfi-base + - busybox + - ca-certificates-bundle + - gettext + - git + - gcc + - go + # - g++ # Not available on Wolfi - required? + +pipeline: + - name: Populate builder template with version variables + runs: | + export OTEL_COLLECTOR_VERSION=${{package.version}} + envsubst opentelemetry-collector/builder.yaml + cat opentelemetry-collector/builder.yaml + - name: Run collector builder + runs: | + go run go.opentelemetry.io/collector/cmd/builder@v${{package.version}} \ + --config opentelemetry-collector/builder.yaml \ + --output-path=/tmp/otelcol-sourcegraph + - name: Package collector + runs: | + mkdir -p ${{targets.destdir}}/usr/bin/ + cp /tmp/otelcol-sourcegraph/otelcol-sourcegraph ${{targets.destdir}}/usr/bin/otelcol-sourcegraph diff --git a/wolfi-packages/opentelemetry-collector/builder.template.yaml b/wolfi-packages/opentelemetry-collector/builder.template.yaml new file mode 100644 index 0000000000000..727ef94db2587 --- /dev/null +++ b/wolfi-packages/opentelemetry-collector/builder.template.yaml @@ -0,0 +1,36 @@ +dist: + module: github.com/sourcegraph/sourcegraph/docker-images/opentelemetry-collector + name: otelcol-sourcegraph + description: "Sourcegraph OpenTelemetry Collector distribution" + otelcol_version: "$OTEL_COLLECTOR_VERSION" + +exporters: + # OpenTelemetry exporters - https://go.opentelemetry.io/collector/exporter + - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v$OTEL_COLLECTOR_VERSION + - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v$OTEL_COLLECTOR_VERSION + - gomod: go.opentelemetry.io/collector/exporter/loggingexporter v$OTEL_COLLECTOR_VERSION + + # Contrib exporters - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter + - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/jaegerexporter v$OTEL_COLLECTOR_VERSION" + - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/googlecloudexporter v$OTEL_COLLECTOR_VERSION" + +receivers: + # OpenTelemetry receivers - https://go.opentelemetry.io/collector/receiver + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v$OTEL_COLLECTOR_VERSION + + # Contrib receivers - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver + - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver v$OTEL_COLLECTOR_VERSION" + - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v$OTEL_COLLECTOR_VERSION" + +extensions: + # OpenTelemetry extensions - https://go.opentelemetry.io/collector/extension + - gomod: go.opentelemetry.io/collector/extension/zpagesextension v$OTEL_COLLECTOR_VERSION + + # Contrib extensions - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v$OTEL_COLLECTOR_VERSION + +processors: + # Contrib extensions - https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v$OTEL_COLLECTOR_VERSION + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v$OTEL_COLLECTOR_VERSION + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v$OTEL_COLLECTOR_VERSION diff --git a/wolfi-packages/redis_exporter.yaml b/wolfi-packages/redis_exporter.yaml new file mode 100644 index 0000000000000..bef6733f3d06c --- /dev/null +++ b/wolfi-packages/redis_exporter.yaml @@ -0,0 +1,37 @@ +package: + name: redis_exporter + version: 1.35.0 + epoch: 0 + description: "Prometheus Exporter for Redis Metrics" + target-architecture: + - x86_64 + copyright: + - paths: + - "*" + attestation: 'Copyright (c) 2016 Oliver' + license: 'MIT License' + dependencies: + runtime: + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - wolfi-base + - busybox + - ca-certificates-bundle + +pipeline: + - uses: fetch + with: + uri: https://github.com/oliver006/redis_exporter/archive/refs/tags/v${{package.version}}.tar.gz + expected-sha256: e634bbeaafec4505e23d14b43d85b588839abc7eccde53e3c3f833d2a98c1fb6 + - uses: go/build + with: + packages: main.go + ldflags: "-s -w -extldflags \"-static\"" + prefix: /usr/local + output: redis_exporter From 4fedb1d3d36db07f7d167dd734d6940f37958512 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 11:27:59 +0100 Subject: [PATCH 39/57] Add bind-tools to all base images --- wolfi-images/batcheshelper.yaml | 2 +- wolfi-images/blobstore.yaml | 2 +- wolfi-images/cadvisor.yaml | 2 +- wolfi-images/gitserver.yaml | 2 +- wolfi-images/jaeger-agent.yaml | 2 +- wolfi-images/jaeger-all-in-one.yaml | 2 +- wolfi-images/opentelemetry-collector.yaml | 2 +- wolfi-images/redis-exporter.yaml | 2 +- wolfi-images/repo-updater.yaml | 2 +- wolfi-images/search-indexer.yaml | 2 +- wolfi-images/searcher.yaml | 2 +- wolfi-images/server.yaml | 2 +- wolfi-images/sourcegraph-dev.yaml | 2 +- wolfi-images/sourcegraph.yaml | 2 +- wolfi-images/symbols.yaml | 2 +- wolfi-images/syntax-highlighter.yaml | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/wolfi-images/batcheshelper.yaml b/wolfi-images/batcheshelper.yaml index 6b9eaff12d2b2..26fe7de779a4b 100644 --- a/wolfi-images/batcheshelper.yaml +++ b/wolfi-images/batcheshelper.yaml @@ -14,7 +14,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## batcheshelper packages - 'git>=2.38.1' diff --git a/wolfi-images/blobstore.yaml b/wolfi-images/blobstore.yaml index 7b6adf19d54b4..63034dd96daed 100644 --- a/wolfi-images/blobstore.yaml +++ b/wolfi-images/blobstore.yaml @@ -14,7 +14,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## blobstore packages - openjdk-11 diff --git a/wolfi-images/cadvisor.yaml b/wolfi-images/cadvisor.yaml index 8beb49704b5cf..51e4f0ed19a84 100644 --- a/wolfi-images/cadvisor.yaml +++ b/wolfi-images/cadvisor.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## cadvisor dependencies - cadvisor@sourcegraph diff --git a/wolfi-images/gitserver.yaml b/wolfi-images/gitserver.yaml index 17005a058ed06..e4d0cc3a79e98 100644 --- a/wolfi-images/gitserver.yaml +++ b/wolfi-images/gitserver.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## gitserver packages - 'git>=2.38.1' diff --git a/wolfi-images/jaeger-agent.yaml b/wolfi-images/jaeger-agent.yaml index 0e49c07fc14ed..8bc48de802d59 100644 --- a/wolfi-images/jaeger-agent.yaml +++ b/wolfi-images/jaeger-agent.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## jaeger-agent-specific tools - jaeger-agent@sourcegraph diff --git a/wolfi-images/jaeger-all-in-one.yaml b/wolfi-images/jaeger-all-in-one.yaml index 15e2ca3564f73..fc6c21c366329 100644 --- a/wolfi-images/jaeger-all-in-one.yaml +++ b/wolfi-images/jaeger-all-in-one.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## jaeger-all-in-one-specific tools - jaeger-all-in-one@sourcegraph diff --git a/wolfi-images/opentelemetry-collector.yaml b/wolfi-images/opentelemetry-collector.yaml index 2e546194cfd7e..9bef4d28f1159 100644 --- a/wolfi-images/opentelemetry-collector.yaml +++ b/wolfi-images/opentelemetry-collector.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## opentelemetry-collector-specific packages - opentelemetry-collector@sourcegraph diff --git a/wolfi-images/redis-exporter.yaml b/wolfi-images/redis-exporter.yaml index 104e0c9ffa69c..c83d0626e1fc6 100644 --- a/wolfi-images/redis-exporter.yaml +++ b/wolfi-images/redis-exporter.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## redis_exporter packages - redis_exporter@sourcegraph diff --git a/wolfi-images/repo-updater.yaml b/wolfi-images/repo-updater.yaml index 8730025295369..2cdaf60f53d8e 100644 --- a/wolfi-images/repo-updater.yaml +++ b/wolfi-images/repo-updater.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## repo-updater packages - coursier@sourcegraph diff --git a/wolfi-images/search-indexer.yaml b/wolfi-images/search-indexer.yaml index 15b0e311787a1..e565ead1978fb 100644 --- a/wolfi-images/search-indexer.yaml +++ b/wolfi-images/search-indexer.yaml @@ -17,7 +17,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## search-indexer-specific packages - git diff --git a/wolfi-images/searcher.yaml b/wolfi-images/searcher.yaml index d36f9bc6e5067..c2d3233dc0580 100644 --- a/wolfi-images/searcher.yaml +++ b/wolfi-images/searcher.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## Searcher packages - pcre # TODO: Can we upgrade to pcre2, which is maintained? diff --git a/wolfi-images/server.yaml b/wolfi-images/server.yaml index 675309974f2e8..5fccd34356c21 100644 --- a/wolfi-images/server.yaml +++ b/wolfi-images/server.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## Dev testing tools - apk-tools diff --git a/wolfi-images/sourcegraph-dev.yaml b/wolfi-images/sourcegraph-dev.yaml index a471825dd400d..090f8513f8485 100644 --- a/wolfi-images/sourcegraph-dev.yaml +++ b/wolfi-images/sourcegraph-dev.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## Dev-specific tools - apk-tools diff --git a/wolfi-images/sourcegraph.yaml b/wolfi-images/sourcegraph.yaml index 5fdc73f20cdb3..4986225d0d3b0 100644 --- a/wolfi-images/sourcegraph.yaml +++ b/wolfi-images/sourcegraph.yaml @@ -14,7 +14,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools accounts: groups: diff --git a/wolfi-images/symbols.yaml b/wolfi-images/symbols.yaml index 50b2d08ac6237..249ec90ca0bfd 100644 --- a/wolfi-images/symbols.yaml +++ b/wolfi-images/symbols.yaml @@ -17,7 +17,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## symbols packages - ca-certificates diff --git a/wolfi-images/syntax-highlighter.yaml b/wolfi-images/syntax-highlighter.yaml index 8dcefdf61bd16..f5d4a74117e8f 100644 --- a/wolfi-images/syntax-highlighter.yaml +++ b/wolfi-images/syntax-highlighter.yaml @@ -16,7 +16,7 @@ contents: - busybox - curl - wget - # - 'bind-tools>=9.16.33-r0' # TODO: Not available in Wolfi repo + - bind-tools ## syntax-highlighter packages - libstdc++ From 575445983e8939bf76c38954d77d042502e78aea Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 11:38:38 +0100 Subject: [PATCH 40/57] Make docker build commands more consistent --- docker-images/node-exporter/build-wolfi.sh | 2 +- docker-images/postgres-12-alpine/build-wolfi.sh | 2 +- docker-images/postgres_exporter/build-wolfi.sh | 2 +- docker-images/redis-cache/build-wolfi.sh | 2 +- docker-images/redis-store/build-wolfi.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-images/node-exporter/build-wolfi.sh b/docker-images/node-exporter/build-wolfi.sh index 89631182d2cd7..76ca60b1afd04 100755 --- a/docker-images/node-exporter/build-wolfi.sh +++ b/docker-images/node-exporter/build-wolfi.sh @@ -2,7 +2,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" set -ex -docker build -f ./Dockerfile.wolfi -t "${IMAGE:-sourcegraph/node-exporter}" . \ +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/node-exporter}" . \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/docker-images/postgres-12-alpine/build-wolfi.sh b/docker-images/postgres-12-alpine/build-wolfi.sh index e01f277198c0c..76c748220704c 100755 --- a/docker-images/postgres-12-alpine/build-wolfi.sh +++ b/docker-images/postgres-12-alpine/build-wolfi.sh @@ -6,4 +6,4 @@ cd "$(dirname "${BASH_SOURCE[0]}")" POSTGRES_UID=${POSTGRES_UID:-999} PING_UID=${PING_UID:-99} -docker build -t "${IMAGE:-index.docker.io/sourcegraph/wolfi-postgres-12}" --build-arg POSTGRES_UID="$POSTGRES_UID" --build-arg PING_UID="$PING_UID" -f ./Dockerfile.wolfi . +docker build -f Dockerfile.wolfi -t "${IMAGE:-index.docker.io/sourcegraph/wolfi-postgres-12}" --build-arg POSTGRES_UID="$POSTGRES_UID" --build-arg PING_UID="$PING_UID" . diff --git a/docker-images/postgres_exporter/build-wolfi.sh b/docker-images/postgres_exporter/build-wolfi.sh index fb01de6a8c60c..61e7bf65ce856 100755 --- a/docker-images/postgres_exporter/build-wolfi.sh +++ b/docker-images/postgres_exporter/build-wolfi.sh @@ -54,7 +54,7 @@ echo "${OUTPUT_FILE}" echo "${CODEINTEL_OUTPUT_FILE}" echo "${CODEINSIGHTS_OUTPUT_FILE}" -docker build -f ./Dockerfile.wolfi -t "${IMAGE:-sourcegraph/postgres_exporter}" "${OUTPUT}" \ +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/postgres_exporter}" "${OUTPUT}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/docker-images/redis-cache/build-wolfi.sh b/docker-images/redis-cache/build-wolfi.sh index dbbf9584aee96..b3cafc63b35ff 100755 --- a/docker-images/redis-cache/build-wolfi.sh +++ b/docker-images/redis-cache/build-wolfi.sh @@ -3,4 +3,4 @@ set -ex cd "$(dirname "${BASH_SOURCE[0]}")" -docker build -t "${IMAGE:-sourcegraph/redis-cache}" -f Dockerfile.wolfi . +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/redis-cache}" . diff --git a/docker-images/redis-store/build-wolfi.sh b/docker-images/redis-store/build-wolfi.sh index 988dfc8cb7eda..feecc6a8bcfce 100755 --- a/docker-images/redis-store/build-wolfi.sh +++ b/docker-images/redis-store/build-wolfi.sh @@ -3,4 +3,4 @@ set -ex cd "$(dirname "${BASH_SOURCE[0]}")" -docker build -t "${IMAGE:-sourcegraph/redis-store}" -f Dockerfile.wolfi . +docker build -f Dockerfile.wolfi -t "${IMAGE:-sourcegraph/redis-store}" . From 53b494a19838c99fb76dcab0a5885d56dd780a63 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 12:42:22 +0100 Subject: [PATCH 41/57] Add wolfi build scripts for llm-proxy --- enterprise/cmd/llm-proxy/Dockerfile.wolfi | 17 +++++++++++++++ enterprise/cmd/llm-proxy/build-wolfi.sh | 26 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 enterprise/cmd/llm-proxy/Dockerfile.wolfi create mode 100755 enterprise/cmd/llm-proxy/build-wolfi.sh diff --git a/enterprise/cmd/llm-proxy/Dockerfile.wolfi b/enterprise/cmd/llm-proxy/Dockerfile.wolfi new file mode 100644 index 0000000000000..a42668bbc60c7 --- /dev/null +++ b/enterprise/cmd/llm-proxy/Dockerfile.wolfi @@ -0,0 +1,17 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/llm-proxy"] +COPY llm-proxy /usr/local/bin/ + +# Embeddings uses 9991, let's take the next one +EXPOSE 9992 diff --git a/enterprise/cmd/llm-proxy/build-wolfi.sh b/enterprise/cmd/llm-proxy/build-wolfi.sh new file mode 100755 index 0000000000000..04ac5708da631 --- /dev/null +++ b/enterprise/cmd/llm-proxy/build-wolfi.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/llm-proxy" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" + +docker build -f enterprise/cmd/llm-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION From 4425f4d53dca832c95763d1bfd14e428844a1aef Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 12:42:56 +0100 Subject: [PATCH 42/57] Add wolfi build scripts for executor-kubernetes --- .../cmd/executor-kubernetes/Dockerfile.wolfi | 14 +++++++ .../cmd/executor-kubernetes/build-wolfi.sh | 28 +++++++++++++ wolfi-images/executor-kubernetes.yaml | 40 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 enterprise/cmd/executor-kubernetes/Dockerfile.wolfi create mode 100755 enterprise/cmd/executor-kubernetes/build-wolfi.sh create mode 100644 wolfi-images/executor-kubernetes.yaml diff --git a/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi b/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi new file mode 100644 index 0000000000000..94dc932b8429b --- /dev/null +++ b/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi @@ -0,0 +1,14 @@ +FROM us.gcr.io/sourcegraph-dev/wolfi-executor-kubernetes-base:latest + +ARG COMMIT_SHA="unknown" +ARG DATE="unknown" +ARG VERSION="unknown" + +LABEL org.opencontainers.image.revision=${COMMIT_SHA} +LABEL org.opencontainers.image.created=${DATE} +LABEL org.opencontainers.image.version=${VERSION} +LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} + +USER sourcegraph +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/executor"] +COPY executor /usr/local/bin/ diff --git a/enterprise/cmd/executor-kubernetes/build-wolfi.sh b/enterprise/cmd/executor-kubernetes/build-wolfi.sh new file mode 100755 index 0000000000000..6cf77d63e3bb2 --- /dev/null +++ b/enterprise/cmd/executor-kubernetes/build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# We want to build multiple go binaries, so we use a custom build step on CI. +cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. +set -ex + +OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "$OUTPUT" +} +trap cleanup EXIT + +# Environment for building linux binaries +export GO111MODULE=on +export GOARCH=amd64 +export GOOS=linux +export CGO_ENABLED=0 + +pushd ./enterprise/cmd/executor 1>/dev/null +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/executor" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +popd 1>/dev/null + +docker build -f enterprise/cmd/executor-kubernetes/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION diff --git a/wolfi-images/executor-kubernetes.yaml b/wolfi-images/executor-kubernetes.yaml new file mode 100644 index 0000000000000..2d71619977c75 --- /dev/null +++ b/wolfi-images/executor-kubernetes.yaml @@ -0,0 +1,40 @@ +contents: + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + repositories: + - https://packages.wolfi.dev/os + packages: + ## Base set of packages included in sourcegraph/alpine base image + - wolfi-baselayout + - ca-certificates-bundle + - tzdata + - tini + - mailcap + # Dev tools - may not be required in production + - busybox + - curl + - wget + - bind-tools + + ## executor-kubernetes-specific packages + - git + +accounts: + run-as: sourcegraph + groups: + - groupname: sourcegraph + gid: 101 + users: + - username: sourcegraph + uid: 100 + gid: 101 + +# NOTE: This is ignored (see build output) +# To force amd64, first run `docker pull --platform linux/arm64 cgr.dev/chainguard/apko` +archs: + - amd64 + +annotations: + org.opencontainers.image.url: https://sourcegraph.com/ + org.opencontainers.image.source: https://github.com/sourcegraph/sourcegraph/ + org.opencontainers.image.documentation: https://docs.sourcegraph.com/ From 5662a681ea3105922ac0209aa3c0c8e1dd90961e Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 12:45:50 +0100 Subject: [PATCH 43/57] Update build-wolfi.sh scripts to match latest build.sh scripts --- cmd/frontend/build-wolfi.sh | 3 +- cmd/github-proxy/build-wolfi.sh | 21 +++++++++++-- cmd/gitserver/build-wolfi.sh | 20 +++++++++++-- cmd/loadtest/build-wolfi.sh | 21 +++++++++++-- cmd/migrator/build-wolfi.sh | 30 ++++++++++++++++--- cmd/repo-updater/build-wolfi.sh | 21 +++++++++++-- cmd/searcher/build-wolfi.sh | 20 +++++++++++-- cmd/server/build-wolfi.sh | 3 -- cmd/symbols/build-wolfi.sh | 1 - cmd/worker/build-wolfi.sh | 20 ++++++++++++- enterprise/cmd/batcheshelper/build-wolfi.sh | 2 +- enterprise/cmd/frontend/build-wolfi.sh | 21 +++++++++++-- enterprise/cmd/gitserver/build-wolfi.sh | 9 +++--- .../precise-code-intel-worker/build-wolfi.sh | 3 +- enterprise/cmd/worker/build-wolfi.sh | 3 +- 15 files changed, 165 insertions(+), 33 deletions(-) diff --git a/cmd/frontend/build-wolfi.sh b/cmd/frontend/build-wolfi.sh index 7cd5e537976ce..d62847079a017 100755 --- a/cmd/frontend/build-wolfi.sh +++ b/cmd/frontend/build-wolfi.sh @@ -18,11 +18,10 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/cmd/frontend" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build $IMAGE" docker build -f cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/github-proxy/build-wolfi.sh b/cmd/github-proxy/build-wolfi.sh index 5eabc7220b50a..9d4eb8011f17f 100755 --- a/cmd/github-proxy/build-wolfi.sh +++ b/cmd/github-proxy/build-wolfi.sh @@ -10,6 +10,24 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + + bazel build //cmd/github-proxy \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/github-proxy --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/github-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -17,10 +35,9 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/github-proxy" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/github-proxy/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/gitserver/build-wolfi.sh b/cmd/gitserver/build-wolfi.sh index 420cf1c6b60e2..1848a5904cf4f 100755 --- a/cmd/gitserver/build-wolfi.sh +++ b/cmd/gitserver/build-wolfi.sh @@ -12,6 +12,23 @@ cleanup() { trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + bazel build //cmd/gitserver \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/gitserver --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -19,10 +36,9 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/loadtest/build-wolfi.sh b/cmd/loadtest/build-wolfi.sh index f0d26c1442ba0..a6201fee245dd 100755 --- a/cmd/loadtest/build-wolfi.sh +++ b/cmd/loadtest/build-wolfi.sh @@ -10,6 +10,24 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + + bazel build //cmd/loadtest \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/loadtest --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/loadtest/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -17,10 +35,9 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/loadtest" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/loadtest/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/migrator/build-wolfi.sh b/cmd/migrator/build-wolfi.sh index f9e8acb73deee..1fe756433dac7 100755 --- a/cmd/migrator/build-wolfi.sh +++ b/cmd/migrator/build-wolfi.sh @@ -3,7 +3,7 @@ # This script builds the migrator docker image. cd "$(dirname "${BASH_SOURCE[0]}")/../.." -set -eu +set -ex OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) cleanup() { @@ -11,6 +11,24 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + + bazel build //cmd/migrator \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/migrator --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/migrator/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -21,7 +39,7 @@ echo "--- go build" pkg=${1:-"github.com/sourcegraph/sourcegraph/cmd/migrator"} output="$OUTPUT/$(basename "$pkg")" # shellcheck disable=SC2153 -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$output" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$output" "$pkg" echo "--- compile schema descriptions" mkdir -p "${OUTPUT}/schema-descriptions" @@ -75,7 +93,12 @@ git_versions=( v3.42.0 v3.42.1 v3.42.2 v3.43.0 v3.43.1 v3.43.2 v4.0.0 v4.0.1 - v4.1.0 v4.1.1 v4.1.2 + v4.1.0 v4.1.1 v4.1.2 v4.1.3 + v4.2.0 v4.2.1 + v4.3.0 v4.3.1 + v4.4.0 v4.4.1 v4.4.2 + v4.5.0 v4.5.1 + v5.0.0 ) for version in "${git_versions[@]}"; do echo "Persisting schemas for ${version} from Git..." @@ -86,7 +109,6 @@ done echo "--- docker build" docker build -f cmd/migrator/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/repo-updater/build-wolfi.sh b/cmd/repo-updater/build-wolfi.sh index f6a88c4948ae1..b83075978f225 100755 --- a/cmd/repo-updater/build-wolfi.sh +++ b/cmd/repo-updater/build-wolfi.sh @@ -9,8 +9,26 @@ OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) cleanup() { rm -rf "$OUTPUT" } + trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + bazel build //cmd/repo-updater \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/repo-updater --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/repo-updater/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -18,11 +36,10 @@ export GOOS=linux export CGO_ENABLED=0 for pkg in $path_to_package; do - bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename "$pkg")" "$pkg" + go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename "$pkg")" "$pkg" done docker build -f cmd/repo-updater/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/searcher/build-wolfi.sh b/cmd/searcher/build-wolfi.sh index b927f4431fc03..8914da8a54613 100755 --- a/cmd/searcher/build-wolfi.sh +++ b/cmd/searcher/build-wolfi.sh @@ -10,6 +10,23 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + bazel build //cmd/searcher \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/searcher --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/searcher/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -17,10 +34,9 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/cmd/searcher" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f cmd/searcher/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/server/build-wolfi.sh b/cmd/server/build-wolfi.sh index daba08ba23bd9..45dfd822cfded 100755 --- a/cmd/server/build-wolfi.sh +++ b/cmd/server/build-wolfi.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# TODO: This is untested and WIP - # We want to build multiple go binaries, so we use a custom build step on CI. cd "$(dirname "${BASH_SOURCE[0]}")/../.." set -eux @@ -101,7 +99,6 @@ IMAGE=sourcegraph/blobstore:server docker-images/blobstore/build-wolfi.sh echo "--- docker build" docker build -f cmd/server/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/symbols/build-wolfi.sh b/cmd/symbols/build-wolfi.sh index fd49d082d07ec..e5cb3d3e8dee3 100755 --- a/cmd/symbols/build-wolfi.sh +++ b/cmd/symbols/build-wolfi.sh @@ -7,7 +7,6 @@ set -eu echo "--- docker build symbols" docker build -f cmd/symbols/Dockerfile.wolfi -t "$IMAGE" "$(pwd)" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/cmd/worker/build-wolfi.sh b/cmd/worker/build-wolfi.sh index f339340cf3094..50eb251881cd2 100755 --- a/cmd/worker/build-wolfi.sh +++ b/cmd/worker/build-wolfi.sh @@ -11,6 +11,24 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + + bazel build //cmd/worker \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/worker --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -22,7 +40,7 @@ pkg="github.com/sourcegraph/sourcegraph/cmd/worker" go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" -docker build -f cmd/worker/Dockerfile -t "$IMAGE" "$OUTPUT" \ +docker build -f cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/batcheshelper/build-wolfi.sh b/enterprise/cmd/batcheshelper/build-wolfi.sh index d12265fec6575..cd7c935c68029 100755 --- a/enterprise/cmd/batcheshelper/build-wolfi.sh +++ b/enterprise/cmd/batcheshelper/build-wolfi.sh @@ -16,7 +16,7 @@ export GOOS=linux export CGO_ENABLED=0 pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/batcheshelper" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" docker build -f enterprise/cmd/batcheshelper/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --platform="${PLATFORM:-linux/amd64}" \ diff --git a/enterprise/cmd/frontend/build-wolfi.sh b/enterprise/cmd/frontend/build-wolfi.sh index 0a01402834b68..c716c4811b713 100755 --- a/enterprise/cmd/frontend/build-wolfi.sh +++ b/enterprise/cmd/frontend/build-wolfi.sh @@ -11,6 +11,24 @@ cleanup() { } trap cleanup EXIT +if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then + bazel build //enterprise/cmd/frontend \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ + --//:assets_bundle_type=enterprise + + out=$(bazel cquery //enterprise/cmd/frontend --output=files) + cp "$out" "$OUTPUT" + + docker build -f enterprise/cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? +fi + # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 @@ -19,11 +37,10 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/frontend/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/gitserver/build-wolfi.sh b/enterprise/cmd/gitserver/build-wolfi.sh index 420cf1c6b60e2..910ee9fd55358 100755 --- a/enterprise/cmd/gitserver/build-wolfi.sh +++ b/enterprise/cmd/gitserver/build-wolfi.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # We want to build multiple go binaries, so we use a custom build step on CI. -cd "$(dirname "${BASH_SOURCE[0]}")"/../.. +cd "$(dirname "${BASH_SOURCE[0]}")/../../.." set -ex OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) @@ -18,11 +18,10 @@ export GOARCH=amd64 export GOOS=linux export CGO_ENABLED=0 -pkg="github.com/sourcegraph/sourcegraph/cmd/gitserver" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" -docker build -f cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ +docker build -f enterprise/cmd/gitserver/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh index 3adfabdf9c76e..45a3647621293 100755 --- a/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh +++ b/enterprise/cmd/precise-code-intel-worker/build-wolfi.sh @@ -19,11 +19,10 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ diff --git a/enterprise/cmd/worker/build-wolfi.sh b/enterprise/cmd/worker/build-wolfi.sh index e9f5dadbfcc9e..e77aca857fdfe 100755 --- a/enterprise/cmd/worker/build-wolfi.sh +++ b/enterprise/cmd/worker/build-wolfi.sh @@ -19,11 +19,10 @@ export CGO_ENABLED=0 echo "--- go build" pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/worker" -bazel run @go_sdk//:bin/go -- build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" +go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" echo "--- docker build" docker build -f enterprise/cmd/worker/Dockerfile.wolfi -t "$IMAGE" "$OUTPUT" \ - --platform="${PLATFORM:-linux/amd64}" \ --progress=plain \ --build-arg COMMIT_SHA \ --build-arg DATE \ From 17718859cd144b6ff423e7a91ed801e02a25fa3c Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 12:46:24 +0100 Subject: [PATCH 44/57] shellfmt build.sh scripts --- cmd/repo-updater/build.sh | 30 ++++++++++++++---------------- cmd/searcher/build.sh | 28 ++++++++++++++-------------- cmd/worker/build.sh | 1 - enterprise/cmd/frontend/build.sh | 31 ++++++++++++++++--------------- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/cmd/repo-updater/build.sh b/cmd/repo-updater/build.sh index 406a3249ab79f..98689821a6948 100755 --- a/cmd/repo-updater/build.sh +++ b/cmd/repo-updater/build.sh @@ -12,24 +12,22 @@ cleanup() { trap cleanup EXIT if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then - bazel build //cmd/repo-updater \ - --stamp \ - --workspace_status_command=./dev/bazel_stamp_vars.sh \ - --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 - - out=$(bazel cquery //cmd/repo-updater --output=files) - cp "$out" "$OUTPUT" - - docker build -f cmd/repo-updater/Dockerfile -t "$IMAGE" "$OUTPUT" \ - --progress=plain \ - --build-arg COMMIT_SHA \ - --build-arg DATE \ - --build-arg VERSION - exit $? + bazel build //cmd/repo-updater \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/repo-updater --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/repo-updater/Dockerfile -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? fi - - # Environment for building linux binaries export GO111MODULE=on export GOARCH=amd64 diff --git a/cmd/searcher/build.sh b/cmd/searcher/build.sh index 4ab962f7e03b7..0ae24255006c3 100755 --- a/cmd/searcher/build.sh +++ b/cmd/searcher/build.sh @@ -11,20 +11,20 @@ cleanup() { trap cleanup EXIT if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then - bazel build //cmd/searcher \ - --stamp \ - --workspace_status_command=./dev/bazel_stamp_vars.sh \ - --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 - - out=$(bazel cquery //cmd/searcher --output=files) - cp "$out" "$OUTPUT" - - docker build -f cmd/searcher/Dockerfile -t "$IMAGE" "$OUTPUT" \ - --progress=plain \ - --build-arg COMMIT_SHA \ - --build-arg DATE \ - --build-arg VERSION - exit $? + bazel build //cmd/searcher \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 + + out=$(bazel cquery //cmd/searcher --output=files) + cp "$out" "$OUTPUT" + + docker build -f cmd/searcher/Dockerfile -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? fi # Environment for building linux binaries diff --git a/cmd/worker/build.sh b/cmd/worker/build.sh index 7d87169419cce..e3569d9304f86 100644 --- a/cmd/worker/build.sh +++ b/cmd/worker/build.sh @@ -11,7 +11,6 @@ cleanup() { } trap cleanup EXIT - if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then bazel build //cmd/worker \ diff --git a/enterprise/cmd/frontend/build.sh b/enterprise/cmd/frontend/build.sh index eb50f78241d87..adb4085981ebd 100755 --- a/enterprise/cmd/frontend/build.sh +++ b/enterprise/cmd/frontend/build.sh @@ -10,22 +10,23 @@ cleanup() { rm -rf "$OUTPUT" } trap cleanup EXIT + if [[ "${DOCKER_BAZEL:-false}" == "true" ]]; then - bazel build //enterprise/cmd/frontend \ - --stamp \ - --workspace_status_command=./dev/bazel_stamp_vars.sh \ - --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ - --//:assets_bundle_type=enterprise - - out=$(bazel cquery //enterprise/cmd/frontend --output=files) - cp "$out" "$OUTPUT" - - docker build -f enterprise/cmd/frontend/Dockerfile -t "$IMAGE" "$OUTPUT" \ - --progress=plain \ - --build-arg COMMIT_SHA \ - --build-arg DATE \ - --build-arg VERSION - exit $? + bazel build //enterprise/cmd/frontend \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 \ + --//:assets_bundle_type=enterprise + + out=$(bazel cquery //enterprise/cmd/frontend --output=files) + cp "$out" "$OUTPUT" + + docker build -f enterprise/cmd/frontend/Dockerfile -t "$IMAGE" "$OUTPUT" \ + --progress=plain \ + --build-arg COMMIT_SHA \ + --build-arg DATE \ + --build-arg VERSION + exit $? fi # Environment for building linux binaries From 1ed2df994224d196d21c7142b61276a7dfad929f Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 18:31:02 +0100 Subject: [PATCH 45/57] Set up proper deps for wolfi images and base images --- enterprise/dev/ci/internal/ci/pipeline.go | 26 ++++--- .../dev/ci/internal/ci/wolfi_operations.go | 69 +++++++++++++------ 2 files changed, 64 insertions(+), 31 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index 70b3965ebab2b..681f33ad6fca7 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -106,22 +106,27 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Rebuild base images if base image OR package configs have changed updateBaseImages := c.Diff.Has(changed.WolfiBaseImages) || updatePackages + var numUpdatedPackages int + var numUpdatedBaseImages int + if updatePackages { - ops.Merge(WolfiPackagesOperations(c.ChangedFiles[changed.WolfiPackages])) + var packageOps *operations.Set + packageOps, numUpdatedPackages = WolfiPackagesOperations(c.ChangedFiles[changed.WolfiPackages]) + ops.Merge(packageOps) } if updateBaseImages { - ops.Merge( - WolfiBaseImagesOperations( - c.ChangedFiles[changed.WolfiBaseImages], // TODO: If packages have changed need to update all base images. Requires a list of all base images - c.Version, - updatePackages, - ), + var baseImageOps *operations.Set + baseImageOps, numUpdatedBaseImages = WolfiBaseImagesOperations( + c.ChangedFiles[changed.WolfiBaseImages], // TODO: If packages have changed need to update all base images. Requires a list of all base images + c.Version, + (numUpdatedPackages > 0), ) + ops.Merge(baseImageOps) } // Always rebuild Wolfi images ops.Merge( // TODO: Just hardcode specific images initially - BuildWolfiOperations([]string{ + WolfiImagesOperations([]string{ "blobstore", "cadvisor", "codeinsights-db", @@ -138,7 +143,10 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { "search-indexer", "sg", "syntax-highlighter", - }, c.Version, c.candidateImageTag()), + }, c.Version, + c.candidateImageTag(), + (numUpdatedBaseImages > 0), + ), ) case runtype.PullRequest: diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 826f3aa9477a3..b78eabecc07ca 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -17,44 +17,50 @@ import ( var baseImageRegex = lazyregexp.New(`wolfi-images\/([\w-]+)[.]yaml`) var packageRegex = lazyregexp.New(`wolfi-packages\/([\w-]+)[.]yaml`) -func WolfiBaseImagesOperations(changedFiles []string, tag string, packagesChanged bool) *operations.Set { +// WolfiPackagesOperations rebuilds any packages whose configurations have changed +func WolfiPackagesOperations(changedFiles []string) (*operations.Set, int) { // TODO: Should we require the image name, or the full path to the yaml file? - ops := operations.NewNamedSet("Base image builds") - logger := log.Scoped("gen-pipeline", "generates the pipeline for ci") + ops := operations.NewNamedSet("Dependency packages") + var updatedPackages []string for _, c := range changedFiles { - match := baseImageRegex.FindStringSubmatch(c) + match := packageRegex.FindStringSubmatch(c) if len(match) == 2 { - ops.Append(buildWolfi(match[1], tag, packagesChanged)) - } else { - logger.Fatal(fmt.Sprintf("Unable to extract base image name from '%s', matches were %+v\n", c, match)) + buildFunc, key := buildPackage(match[1]) + updatedPackages = append(updatedPackages, key) + ops.Append(buildFunc) } } - return ops + ops.Append(buildRepoIndex("main", updatedPackages)) + + return ops, len(updatedPackages) } -func WolfiPackagesOperations(changedFiles []string) *operations.Set { +// WolfiBaseImagesOperations rebuilds any base images whose configurations have changed +func WolfiBaseImagesOperations(changedFiles []string, tag string, packagesChanged bool) (*operations.Set, int) { // TODO: Should we require the image name, or the full path to the yaml file? - ops := operations.NewNamedSet("Dependency packages") + ops := operations.NewNamedSet("Base image builds") + logger := log.Scoped("gen-pipeline", "generates the pipeline for ci") - var stepKeys []string + var updatedBaseImages []string for _, c := range changedFiles { - match := packageRegex.FindStringSubmatch(c) + match := baseImageRegex.FindStringSubmatch(c) if len(match) == 2 { - buildFunc, key := buildPackage(match[1]) - stepKeys = append(stepKeys, key) - ops.Append(buildFunc) + ops.Append(buildWolfiBaseImage(match[1], tag, packagesChanged)) + updatedBaseImages = append(updatedBaseImages, match[1]) + } else { + logger.Fatal(fmt.Sprintf("Unable to extract base image name from '%s', matches were %+v\n", c, match)) } } - ops.Append(buildRepoIndex("main", stepKeys)) + ops.Append(allBaseImagesBuilt(updatedBaseImages)) - return ops + return ops, len(updatedBaseImages) } -// BuildWolfiOperations builds the specified docker images, or all images if none are provided -func BuildWolfiOperations(buildImages []string, version string, tag string) *operations.Set { +// WolfiImagesOperations builds the specified docker images, or all images if none are provided +func WolfiImagesOperations(buildImages []string, version string, tag string, baseImagesChanged bool) *operations.Set { // If buildImages is not specified, rebuild all images // TODO: Maintain a list of Wolfi-based images? if len(buildImages) == 0 { @@ -66,7 +72,9 @@ func BuildWolfiOperations(buildImages []string, version string, tag string) *ope for _, dockerImage := range buildImages { // Don't upload sourcemaps // wolfiImageBuildOps.Append(buildCandidateDockerImage(dockerImage, version, tag, false)) - wolfiImageBuildOps.Append(buildCandidateWolfiDockerImage(dockerImage, version, tag, false)) + wolfiImageBuildOps.Append( + buildCandidateWolfiDockerImage(dockerImage, version, tag, false, baseImagesChanged), + ) } return wolfiImageBuildOps @@ -102,7 +110,7 @@ func buildRepoIndex(branch string, packageKeys []string) func(*bk.Pipeline) { } } -func buildWolfi(target string, tag string, dependOnPackages bool) func(*bk.Pipeline) { +func buildWolfiBaseImage(target string, tag string, dependOnPackages bool) func(*bk.Pipeline) { return func(pipeline *bk.Pipeline) { opts := []bk.StepOpt{ @@ -122,8 +130,21 @@ func buildWolfi(target string, tag string, dependOnPackages bool) func(*bk.Pipel } } +// No-op to ensure all base images are updated before building full images +func allBaseImagesBuilt(baseImageKeys []string) func(*bk.Pipeline) { + return func(pipeline *bk.Pipeline) { + pipeline.AddStep(fmt.Sprintf(":octopus: All base images built"), + // We want to run on the bazel queue, so we have a pretty minimal agent. + bk.Agent("queue", "bazel"), + // Depend on all previous package building steps + bk.DependsOn(baseImageKeys...), + bk.Key("buildAllBaseImages"), + ) + } +} + // Build a candidate Wolfi docker image -func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps bool) operations.Operation { +func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps bool, hasDependency bool) operations.Operation { return func(pipeline *bk.Pipeline) { image := strings.ReplaceAll(app, "/", "-") localImage := "sourcegraph/wolfi-" + image + ":" + version @@ -137,6 +158,10 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b bk.Agent("queue", "bazel"), } + if hasDependency { + cmds = append(cmds, bk.DependsOn("buildAllBaseImages")) + } + // Add Sentry environment variables if we are building off main branch // to enable building the webapp with source maps enabled if uploadSourcemaps { From 4ab25867d6601bc401f3e5ec45b0d9615d96c926 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 18:36:20 +0100 Subject: [PATCH 46/57] Add buildAllBaseImages no-op command --- enterprise/dev/ci/internal/ci/wolfi_operations.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index b78eabecc07ca..bf2de7fb7904a 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -133,7 +133,8 @@ func buildWolfiBaseImage(target string, tag string, dependOnPackages bool) func( // No-op to ensure all base images are updated before building full images func allBaseImagesBuilt(baseImageKeys []string) func(*bk.Pipeline) { return func(pipeline *bk.Pipeline) { - pipeline.AddStep(fmt.Sprintf(":octopus: All base images built"), + pipeline.AddStep(":octopus: All base images built", + bk.Cmd("echo 'All base images built'"), // We want to run on the bazel queue, so we have a pretty minimal agent. bk.Agent("queue", "bazel"), // Depend on all previous package building steps From 095023aa5d147e508e380369946dbbd740f7cd4b Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 18:57:16 +0100 Subject: [PATCH 47/57] Use stepKeys in wolfi base image dependencies --- .../dev/ci/internal/ci/wolfi_operations.go | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index bf2de7fb7904a..51d2afafe8924 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -22,19 +22,19 @@ func WolfiPackagesOperations(changedFiles []string) (*operations.Set, int) { // TODO: Should we require the image name, or the full path to the yaml file? ops := operations.NewNamedSet("Dependency packages") - var updatedPackages []string + var buildStepKeys []string for _, c := range changedFiles { match := packageRegex.FindStringSubmatch(c) if len(match) == 2 { buildFunc, key := buildPackage(match[1]) - updatedPackages = append(updatedPackages, key) ops.Append(buildFunc) + buildStepKeys = append(buildStepKeys, key) } } - ops.Append(buildRepoIndex("main", updatedPackages)) + ops.Append(buildRepoIndex("main", buildStepKeys)) - return ops, len(updatedPackages) + return ops, len(buildStepKeys) } // WolfiBaseImagesOperations rebuilds any base images whose configurations have changed @@ -43,20 +43,21 @@ func WolfiBaseImagesOperations(changedFiles []string, tag string, packagesChange ops := operations.NewNamedSet("Base image builds") logger := log.Scoped("gen-pipeline", "generates the pipeline for ci") - var updatedBaseImages []string + var buildStepKeys []string for _, c := range changedFiles { match := baseImageRegex.FindStringSubmatch(c) if len(match) == 2 { - ops.Append(buildWolfiBaseImage(match[1], tag, packagesChanged)) - updatedBaseImages = append(updatedBaseImages, match[1]) + buildFunc, key := buildWolfiBaseImage(match[1], tag, packagesChanged) + ops.Append(buildFunc) + buildStepKeys = append(buildStepKeys, key) } else { logger.Fatal(fmt.Sprintf("Unable to extract base image name from '%s', matches were %+v\n", c, match)) } } - ops.Append(allBaseImagesBuilt(updatedBaseImages)) + ops.Append(allBaseImagesBuilt(buildStepKeys)) - return ops, len(updatedBaseImages) + return ops, len(buildStepKeys) } // WolfiImagesOperations builds the specified docker images, or all images if none are provided @@ -84,8 +85,7 @@ func WolfiImagesOperations(buildImages []string, version string, tag string, bas // (buildPackage[1], buildPackage[2], ...) <-- buildRepoIndex <-- (buildWolfi[1], buildWolfi[2], ...) func buildPackage(target string) (func(*bk.Pipeline), string) { - // TODO: Can this be sanitised? - stepKey := fmt.Sprintf("package-dependency-%s", target) + stepKey := sanitizeStepKey(fmt.Sprintf("package-dependency-%s", target)) return func(pipeline *bk.Pipeline) { pipeline.AddStep(fmt.Sprintf(":package: Package dependency '%s'", target), @@ -110,13 +110,16 @@ func buildRepoIndex(branch string, packageKeys []string) func(*bk.Pipeline) { } } -func buildWolfiBaseImage(target string, tag string, dependOnPackages bool) func(*bk.Pipeline) { +func buildWolfiBaseImage(target string, tag string, dependOnPackages bool) (func(*bk.Pipeline), string) { + stepKey := sanitizeStepKey(fmt.Sprintf("build-base-image-%s", target)) + return func(pipeline *bk.Pipeline) { opts := []bk.StepOpt{ bk.Cmd(fmt.Sprintf("./enterprise/dev/ci/scripts/wolfi/build-base-image.sh %s %s", target, tag)), // We want to run on the bazel queue, so we have a pretty minimal agent. bk.Agent("queue", "bazel"), + bk.Key(stepKey), } // If packages have changed, wait for repo to be re-indexed as base images may depend on new packages if dependOnPackages { @@ -127,7 +130,7 @@ func buildWolfiBaseImage(target string, tag string, dependOnPackages bool) func( fmt.Sprintf(":octopus: Build Wolfi base image '%s'", target), opts..., ) - } + }, stepKey } // No-op to ensure all base images are updated before building full images From cb080dfa7965eb305b0a33c0a761d3bfc459c980 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Thu, 20 Apr 2023 18:57:30 +0100 Subject: [PATCH 48/57] Sanitize stepKeys --- enterprise/dev/ci/internal/ci/wolfi_operations.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/enterprise/dev/ci/internal/ci/wolfi_operations.go b/enterprise/dev/ci/internal/ci/wolfi_operations.go index 51d2afafe8924..611facaeb803d 100644 --- a/enterprise/dev/ci/internal/ci/wolfi_operations.go +++ b/enterprise/dev/ci/internal/ci/wolfi_operations.go @@ -232,3 +232,10 @@ func buildCandidateWolfiDockerImage(app, version, tag string, uploadSourcemaps b pipeline.AddStep(fmt.Sprintf(":octopus: :docker: :construction: Build Wolfi-based %s", app), cmds...) } } + +var reStepKeySanitizer = lazyregexp.New(`[^a-zA-Z0-9_-]+`) + +// sanitizeStepKey sanitizes BuildKite StepKeys by removing any invalid characters +func sanitizeStepKey(key string) string { + return reStepKeySanitizer.ReplaceAllString(key, "") +} From 30a478da08a2d127ad3aca472cb0f1cdbb5f8f5a Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 10:43:46 +0100 Subject: [PATCH 49/57] Remove some completed TODOs --- cmd/server/Dockerfile.wolfi | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/server/Dockerfile.wolfi b/cmd/server/Dockerfile.wolfi index eede4feb20966..decde83240f6e 100644 --- a/cmd/server/Dockerfile.wolfi +++ b/cmd/server/Dockerfile.wolfi @@ -2,10 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. -# TODO: This is untested and WIP - FROM us.gcr.io/sourcegraph-dev/wolfi-server-base:latest -# FROM --platform=x86_64 sourcegraph-wolfi/server-base:latest-amd64 # TODO(security): This container should not be running as root! # @@ -58,7 +55,6 @@ COPY --from=sourcegraph/grafana:server /sg_config_grafana/provisioning/dashboard COPY . / -# TODO: Check # symbols is cgo, ensure we have the requisite dynamic libraries RUN env SANITY_CHECK=true /usr/local/bin/symbols From d8ece3f829958204e3815c2fc9a3152d77cf0b7a Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 10:43:58 +0100 Subject: [PATCH 50/57] Remove images from wolfi pipeline --- enterprise/dev/ci/internal/ci/pipeline.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index 681f33ad6fca7..1f048304abf32 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -124,26 +124,10 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { ops.Merge(baseImageOps) } // Always rebuild Wolfi images + // Rebuild all images seems reasonable. We need a list somewhere! Maybe we can just use the standard image list though? But not all are wolfi-ified ops.Merge( // TODO: Just hardcode specific images initially - WolfiImagesOperations([]string{ - "blobstore", - "cadvisor", - "codeinsights-db", - "codeintel-db", - "indexed-searcher", - "node-exporter", - "opentelemetry-collector", - "postgres-12-alpine", - "prometheus", - "prometheus-gcp", - "redis-cache", - "redis-store", - "redis_exporter", - "search-indexer", - "sg", - "syntax-highlighter", - }, c.Version, + WolfiImagesOperations([]string{}, c.Version, c.candidateImageTag(), (numUpdatedBaseImages > 0), ), From 0985a7cb54a4bc47c35f0934e12e43aead9acc3a Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 10:54:19 +0100 Subject: [PATCH 51/57] Regenerate CI reference --- doc/dev/background-information/ci/reference.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/background-information/ci/reference.md b/doc/dev/background-information/ci/reference.md index cd5685a9556dc..1728e7ed66ef4 100644 --- a/doc/dev/background-information/ci/reference.md +++ b/doc/dev/background-information/ci/reference.md @@ -133,6 +133,7 @@ sg ci build wolfi Base pipeline (more steps might be included based on branch changes): - **Metadata**: Pipeline metadata +- **Wolfi image builds**: Build Wolfi-based alpine-3.14, Build Wolfi-based cadvisor, Build Wolfi-based codeinsights-db, Build Wolfi-based codeintel-db, Build Wolfi-based frontend, Build Wolfi-based github-proxy, Build Wolfi-based gitserver, Build Wolfi-based grafana, Build Wolfi-based indexed-searcher, Build Wolfi-based jaeger-agent, Build Wolfi-based jaeger-all-in-one, Build Wolfi-based blobstore, Build Wolfi-based blobstore2, Build Wolfi-based node-exporter, Build Wolfi-based postgres-12-alpine, Build Wolfi-based postgres_exporter, Build Wolfi-based precise-code-intel-worker, Build Wolfi-based prometheus, Build Wolfi-based prometheus-gcp, Build Wolfi-based redis-cache, Build Wolfi-based redis-store, Build Wolfi-based redis_exporter, Build Wolfi-based repo-updater, Build Wolfi-based search-indexer, Build Wolfi-based searcher, Build Wolfi-based symbols, Build Wolfi-based syntax-highlighter, Build Wolfi-based worker, Build Wolfi-based migrator, Build Wolfi-based executor, Build Wolfi-based executor-kubernetes, Build Wolfi-based executor-vm, Build Wolfi-based batcheshelper, Build Wolfi-based opentelemetry-collector, Build Wolfi-based embeddings, Build Wolfi-based dind, Build Wolfi-based bundled-executor, Build Wolfi-based server, Build Wolfi-based sg, Build Wolfi-based llm-proxy - Upload build trace ### Release branch nightly healthcheck build From 110688a068cf4a8098d5b45e7eb60117daac3f46 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 11:14:31 +0100 Subject: [PATCH 52/57] Remove symbols build flag --- cmd/symbols/go-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/symbols/go-build.sh b/cmd/symbols/go-build.sh index c209a4b282995..53861c50cbaed 100755 --- a/cmd/symbols/go-build.sh +++ b/cmd/symbols/go-build.sh @@ -17,9 +17,9 @@ echo "--- docker symbols build" # Required due to use of RUN --mount=type=cache in Dockerfile. export DOCKER_BUILDKIT=1 +# TODO: The flag --platform="${PLATFORM:-linux/amd64}" \ is required for server image to build, but will break local builds docker build -f cmd/symbols/Dockerfile.wolfi -t symbols-build "$(pwd)" \ --target=symbols-build \ - --platform="${PLATFORM:-linux/amd64}" \ # TODO(will): This is required for server image to build, but will break local builds --progress=plain \ --build-arg VERSION \ --build-arg PKG="${PKG:-github.com/sourcegraph/sourcegraph/cmd/symbols}" From 2a89c242dc7fd3d4246e12df5a60329718acdef5 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 11:21:33 +0100 Subject: [PATCH 53/57] Add hadolint ignores to use of latest with Wolfi images Images will be pinned to specific hashes when entering production --- cmd/frontend/Dockerfile.wolfi | 1 + cmd/github-proxy/Dockerfile.wolfi | 1 + cmd/gitserver/Dockerfile.wolfi | 1 + cmd/loadtest/Dockerfile.wolfi | 1 + cmd/migrator/Dockerfile.wolfi | 1 + cmd/repo-updater/Dockerfile.wolfi | 1 + cmd/searcher/Dockerfile.wolfi | 1 + cmd/server/Dockerfile.wolfi | 1 + cmd/symbols/Dockerfile.wolfi | 2 ++ cmd/worker/Dockerfile.wolfi | 1 + docker-images/blobstore/Dockerfile.wolfi | 1 + docker-images/cadvisor/Dockerfile.wolfi | 1 + docker-images/indexed-searcher/Dockerfile.wolfi | 1 + docker-images/jaeger-agent/Dockerfile.wolfi | 1 + docker-images/jaeger-all-in-one/Dockerfile.wolfi | 1 + docker-images/node-exporter/Dockerfile.wolfi | 1 + docker-images/opentelemetry-collector/Dockerfile.wolfi | 1 + docker-images/postgres-12-alpine/Dockerfile.wolfi | 1 + docker-images/postgres_exporter/Dockerfile.wolfi | 1 + docker-images/prometheus/Dockerfile.wolfi | 2 ++ docker-images/redis-cache/Dockerfile.wolfi | 1 + docker-images/redis-store/Dockerfile.wolfi | 1 + docker-images/redis_exporter/Dockerfile.wolfi | 1 + docker-images/search-indexer/Dockerfile.wolfi | 1 + docker-images/sg/Dockerfile.wolfi | 1 + docker-images/syntax-highlighter/Dockerfile.wolfi | 1 + enterprise/cmd/batcheshelper/Dockerfile.wolfi | 1 + enterprise/cmd/embeddings/Dockerfile.wolfi | 1 + enterprise/cmd/executor-kubernetes/Dockerfile.wolfi | 1 + enterprise/cmd/frontend/Dockerfile.wolfi | 1 + enterprise/cmd/gitserver/Dockerfile.wolfi | 1 + enterprise/cmd/llm-proxy/Dockerfile.wolfi | 1 + enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi | 1 + enterprise/cmd/worker/Dockerfile.wolfi | 1 + 34 files changed, 36 insertions(+) diff --git a/cmd/frontend/Dockerfile.wolfi b/cmd/frontend/Dockerfile.wolfi index cb99cb40bfa97..e8ccd3bcd5fba 100644 --- a/cmd/frontend/Dockerfile.wolfi +++ b/cmd/frontend/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/github-proxy/Dockerfile.wolfi b/cmd/github-proxy/Dockerfile.wolfi index 0630fe30b9ea0..060f19b4f6be0 100644 --- a/cmd/github-proxy/Dockerfile.wolfi +++ b/cmd/github-proxy/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/gitserver/Dockerfile.wolfi b/cmd/gitserver/Dockerfile.wolfi index d6333c8c8f8b2..0b94243249c92 100644 --- a/cmd/gitserver/Dockerfile.wolfi +++ b/cmd/gitserver/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-gitserver-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/loadtest/Dockerfile.wolfi b/cmd/loadtest/Dockerfile.wolfi index f5c90ceb1d4f2..47cb014021eef 100644 --- a/cmd/loadtest/Dockerfile.wolfi +++ b/cmd/loadtest/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/migrator/Dockerfile.wolfi b/cmd/migrator/Dockerfile.wolfi index 33bdfe3ad61d0..2f22179c28004 100644 --- a/cmd/migrator/Dockerfile.wolfi +++ b/cmd/migrator/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/repo-updater/Dockerfile.wolfi b/cmd/repo-updater/Dockerfile.wolfi index 1a1d168db9ea7..4cfd7a407068e 100644 --- a/cmd/repo-updater/Dockerfile.wolfi +++ b/cmd/repo-updater/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-repo-updater-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/searcher/Dockerfile.wolfi b/cmd/searcher/Dockerfile.wolfi index 771d0c8c54543..b9f977f224212 100644 --- a/cmd/searcher/Dockerfile.wolfi +++ b/cmd/searcher/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-searcher-base:latest ARG COMMIT_SHA="unknown" diff --git a/cmd/server/Dockerfile.wolfi b/cmd/server/Dockerfile.wolfi index decde83240f6e..90e1fd60bccd5 100644 --- a/cmd/server/Dockerfile.wolfi +++ b/cmd/server/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-server-base:latest # TODO(security): This container should not be running as root! diff --git a/cmd/symbols/Dockerfile.wolfi b/cmd/symbols/Dockerfile.wolfi index 851130ecd703a..2a5aa8b903a5e 100644 --- a/cmd/symbols/Dockerfile.wolfi +++ b/cmd/symbols/Dockerfile.wolfi @@ -3,6 +3,7 @@ # file if you change the regular Dockerfile. # TODO: See if we can switch back to cgr.dev/chainguard/go:latest +# hadolint ignore=DL3007 FROM cgr.dev/chainguard/go:latest AS symbols-build # hadolint ignore=DL3002 USER root @@ -38,6 +39,7 @@ RUN \ -o /symbols \ $PKG +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-symbols-base:latest AS symbols # TODO(security): This container should not run as root! diff --git a/cmd/worker/Dockerfile.wolfi b/cmd/worker/Dockerfile.wolfi index 4dac697458531..8cd0659bd7ef4 100644 --- a/cmd/worker/Dockerfile.wolfi +++ b/cmd/worker/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/blobstore/Dockerfile.wolfi b/docker-images/blobstore/Dockerfile.wolfi index c9e702f95ef12..38ce850ee90e7 100644 --- a/docker-images/blobstore/Dockerfile.wolfi +++ b/docker-images/blobstore/Dockerfile.wolfi @@ -13,6 +13,7 @@ RUN mvn package -DskipTests && \ cp src/main/resources/run-docker-container.sh /opt/s3proxy # Build our final Wolfi-based image +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-blobstore-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/cadvisor/Dockerfile.wolfi b/docker-images/cadvisor/Dockerfile.wolfi index d5a12137ac314..ab6470a2106d4 100644 --- a/docker-images/cadvisor/Dockerfile.wolfi +++ b/docker-images/cadvisor/Dockerfile.wolfi @@ -1,6 +1,7 @@ # TODO: Experimental cAdvisor Dockerfile. Entirely untested, and may require additional libraries # NOTE: Check the README before updating +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest LABEL com.sourcegraph.cadvisor.version=v0.47.0 diff --git a/docker-images/indexed-searcher/Dockerfile.wolfi b/docker-images/indexed-searcher/Dockerfile.wolfi index 7ffa8d90038f1..383ac8407b6a1 100644 --- a/docker-images/indexed-searcher/Dockerfile.wolfi +++ b/docker-images/indexed-searcher/Dockerfile.wolfi @@ -3,6 +3,7 @@ ARG ZOEKT_IMAGE="index.docker.io/sourcegraph/zoekt-webserver" ARG ZOEKT_VERSION FROM $ZOEKT_IMAGE:$ZOEKT_VERSION AS zoekt_upstream +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/jaeger-agent/Dockerfile.wolfi b/docker-images/jaeger-agent/Dockerfile.wolfi index 730a6e3c48cf0..3ac8edb192c8b 100644 --- a/docker-images/jaeger-agent/Dockerfile.wolfi +++ b/docker-images/jaeger-agent/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-jaeger-agent-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/jaeger-all-in-one/Dockerfile.wolfi b/docker-images/jaeger-all-in-one/Dockerfile.wolfi index 1bf1c4e0d5122..8e1737ed9561e 100644 --- a/docker-images/jaeger-all-in-one/Dockerfile.wolfi +++ b/docker-images/jaeger-all-in-one/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-jaeger-all-in-one-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/node-exporter/Dockerfile.wolfi b/docker-images/node-exporter/Dockerfile.wolfi index a484ddc3d4844..bb1779e6c0c3c 100644 --- a/docker-images/node-exporter/Dockerfile.wolfi +++ b/docker-images/node-exporter/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-node-exporter-base:latest # hadolint ignore=DL3048 LABEL com.sourcegraph.node_exporter.version=v1.5.0 diff --git a/docker-images/opentelemetry-collector/Dockerfile.wolfi b/docker-images/opentelemetry-collector/Dockerfile.wolfi index 13f7da3d44ecb..748cca73d2af0 100644 --- a/docker-images/opentelemetry-collector/Dockerfile.wolfi +++ b/docker-images/opentelemetry-collector/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-opentelemetry-collector-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/postgres-12-alpine/Dockerfile.wolfi b/docker-images/postgres-12-alpine/Dockerfile.wolfi index ac6f2708aada3..09e0fd1721d43 100644 --- a/docker-images/postgres-12-alpine/Dockerfile.wolfi +++ b/docker-images/postgres-12-alpine/Dockerfile.wolfi @@ -7,6 +7,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-postgresql-12-base:latest # To remain compatibility with codeinsights-db and codeintel-db, user and group diff --git a/docker-images/postgres_exporter/Dockerfile.wolfi b/docker-images/postgres_exporter/Dockerfile.wolfi index 289225ffabfcf..2920c564e208b 100644 --- a/docker-images/postgres_exporter/Dockerfile.wolfi +++ b/docker-images/postgres_exporter/Dockerfile.wolfi @@ -1,4 +1,5 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-postgres-exporter-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/prometheus/Dockerfile.wolfi b/docker-images/prometheus/Dockerfile.wolfi index e5fced716ab14..0ea620477e4da 100644 --- a/docker-images/prometheus/Dockerfile.wolfi +++ b/docker-images/prometheus/Dockerfile.wolfi @@ -11,6 +11,7 @@ FROM ${BASE_IMAGE} AS prom_upstream FROM prom/alertmanager:v0.25.0@sha256:db8303fa05341f5dc6b19b36a97325cd1b8307254ed9042a2c554af71f3c0284 AS am_upstream # Build monitoring definitions +# hadolint ignore=DL3007 FROM cgr.dev/chainguard/bash:latest AS monitoring_builder RUN mkdir -p '/generated/prometheus' COPY ./.bin/monitoring-generator /bin/monitoring-generator @@ -18,6 +19,7 @@ RUN PROMETHEUS_DIR='/generated/prometheus' GRAFANA_DIR='' DOCS_DIR='' NO_PRUNE=t RUN ls '/generated/prometheus' # Prepare final image +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest # Should reflect versions above diff --git a/docker-images/redis-cache/Dockerfile.wolfi b/docker-images/redis-cache/Dockerfile.wolfi index b10df6143716d..89434747ac92b 100644 --- a/docker-images/redis-cache/Dockerfile.wolfi +++ b/docker-images/redis-cache/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-redis-base:latest COPY redis.conf /etc/redis/redis.conf diff --git a/docker-images/redis-store/Dockerfile.wolfi b/docker-images/redis-store/Dockerfile.wolfi index b10df6143716d..89434747ac92b 100644 --- a/docker-images/redis-store/Dockerfile.wolfi +++ b/docker-images/redis-store/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-redis-base:latest COPY redis.conf /etc/redis/redis.conf diff --git a/docker-images/redis_exporter/Dockerfile.wolfi b/docker-images/redis_exporter/Dockerfile.wolfi index 08e5d9fff079b..c6256dbb82c6f 100644 --- a/docker-images/redis_exporter/Dockerfile.wolfi +++ b/docker-images/redis_exporter/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-redis-exporter-base:latest USER sourcegraph diff --git a/docker-images/search-indexer/Dockerfile.wolfi b/docker-images/search-indexer/Dockerfile.wolfi index 87853a2c6d925..bec7e94609516 100644 --- a/docker-images/search-indexer/Dockerfile.wolfi +++ b/docker-images/search-indexer/Dockerfile.wolfi @@ -3,6 +3,7 @@ ARG ZOEKT_IMAGE="index.docker.io/sourcegraph/zoekt-indexserver" ARG ZOEKT_VERSION FROM $ZOEKT_IMAGE:$ZOEKT_VERSION AS zoekt_upstream +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-search-indexer-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/sg/Dockerfile.wolfi b/docker-images/sg/Dockerfile.wolfi index 1e6706df7614d..50c73682e8c7f 100644 --- a/docker-images/sg/Dockerfile.wolfi +++ b/docker-images/sg/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/docker-images/syntax-highlighter/Dockerfile.wolfi b/docker-images/syntax-highlighter/Dockerfile.wolfi index 1f593228613d0..7bfc4b7e8307e 100644 --- a/docker-images/syntax-highlighter/Dockerfile.wolfi +++ b/docker-images/syntax-highlighter/Dockerfile.wolfi @@ -6,6 +6,7 @@ ####################### # Compile final image # ####################### +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-syntax-highlighter-base:latest EXPOSE 9238 diff --git a/enterprise/cmd/batcheshelper/Dockerfile.wolfi b/enterprise/cmd/batcheshelper/Dockerfile.wolfi index 06f5530851c69..efda7cf7786b3 100644 --- a/enterprise/cmd/batcheshelper/Dockerfile.wolfi +++ b/enterprise/cmd/batcheshelper/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-batcheshelper-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/embeddings/Dockerfile.wolfi b/enterprise/cmd/embeddings/Dockerfile.wolfi index 4bd3877bd6325..3899d44008b9c 100644 --- a/enterprise/cmd/embeddings/Dockerfile.wolfi +++ b/enterprise/cmd/embeddings/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi b/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi index 94dc932b8429b..e7444faff2eee 100644 --- a/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi +++ b/enterprise/cmd/executor-kubernetes/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-executor-kubernetes-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/frontend/Dockerfile.wolfi b/enterprise/cmd/frontend/Dockerfile.wolfi index 633eb6e5b761b..dc2a168bfc34f 100644 --- a/enterprise/cmd/frontend/Dockerfile.wolfi +++ b/enterprise/cmd/frontend/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/gitserver/Dockerfile.wolfi b/enterprise/cmd/gitserver/Dockerfile.wolfi index d6333c8c8f8b2..0b94243249c92 100644 --- a/enterprise/cmd/gitserver/Dockerfile.wolfi +++ b/enterprise/cmd/gitserver/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-gitserver-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/llm-proxy/Dockerfile.wolfi b/enterprise/cmd/llm-proxy/Dockerfile.wolfi index a42668bbc60c7..91baaaa9d3009 100644 --- a/enterprise/cmd/llm-proxy/Dockerfile.wolfi +++ b/enterprise/cmd/llm-proxy/Dockerfile.wolfi @@ -1,3 +1,4 @@ +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi index 791cb6cd26042..4ec03bf72bc27 100644 --- a/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi +++ b/enterprise/cmd/precise-code-intel-worker/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" diff --git a/enterprise/cmd/worker/Dockerfile.wolfi b/enterprise/cmd/worker/Dockerfile.wolfi index 4dac697458531..8cd0659bd7ef4 100644 --- a/enterprise/cmd/worker/Dockerfile.wolfi +++ b/enterprise/cmd/worker/Dockerfile.wolfi @@ -2,6 +2,7 @@ # This is currently being tested in parallel to Alpine - you don't need to update this # file if you change the regular Dockerfile. +# hadolint ignore=DL3007 FROM us.gcr.io/sourcegraph-dev/wolfi-sourcegraph-base:latest ARG COMMIT_SHA="unknown" From 55e5cb8296eb5ea8fe53f34f9a15dc5bf6d69917 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 11:24:18 +0100 Subject: [PATCH 54/57] Fix CMD formatting --- docker-images/indexed-searcher/Dockerfile.wolfi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-images/indexed-searcher/Dockerfile.wolfi b/docker-images/indexed-searcher/Dockerfile.wolfi index 383ac8407b6a1..745417ab921c8 100644 --- a/docker-images/indexed-searcher/Dockerfile.wolfi +++ b/docker-images/indexed-searcher/Dockerfile.wolfi @@ -30,4 +30,4 @@ COPY --from=zoekt_upstream /usr/local/bin/zoekt-webserver /usr/local/bin/ ENV GOGC=25 ENTRYPOINT ["/sbin/tini", "--"] -CMD zoekt-webserver -index $DATA_DIR -pprof -rpc -indexserver_proxy +CMD ["zoekt-webserver", "-index $DATA_DIR", "-pprof", "-rpc", "-indexserver_proxy"] From c1ebf0bff1e8da1ef6f1da59b3079ef146a2b43f Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 12:11:35 +0100 Subject: [PATCH 55/57] Fix symbols go-build script --- cmd/symbols/go-build-wolfi.sh | 28 ++++++++++++++++++++++++++++ cmd/symbols/go-build.sh | 3 +-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 cmd/symbols/go-build-wolfi.sh diff --git a/cmd/symbols/go-build-wolfi.sh b/cmd/symbols/go-build-wolfi.sh new file mode 100644 index 0000000000000..fec6eaaa88ea8 --- /dev/null +++ b/cmd/symbols/go-build-wolfi.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This script builds the symbols go binary. +# Requires a single argument which is the path to the target bindir. +# +# To test you can run +# +# VERSION=test ./cmd/symbols/go-build-wolfi.sh /tmp + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." +set -eu + +OUTPUT="${1:?no output path provided}" + +echo "--- docker symbols build" + +# Required due to use of RUN --mount=type=cache in Dockerfile. +export DOCKER_BUILDKIT=1 + +# TODO: The platform flag is required for server image to build, but will break local builds +docker build -f cmd/symbols/Dockerfile.wolfi -t symbols-build "$(pwd)" \ + --target=symbols-build \ + --platform="${PLATFORM:-linux/amd64}" \ + --progress=plain \ + --build-arg VERSION \ + --build-arg PKG="${PKG:-github.com/sourcegraph/sourcegraph/cmd/symbols}" + +docker cp "$(docker create --rm symbols-build)":/symbols "$OUTPUT/symbols" diff --git a/cmd/symbols/go-build.sh b/cmd/symbols/go-build.sh index 53861c50cbaed..0d0fb2aa60722 100755 --- a/cmd/symbols/go-build.sh +++ b/cmd/symbols/go-build.sh @@ -17,8 +17,7 @@ echo "--- docker symbols build" # Required due to use of RUN --mount=type=cache in Dockerfile. export DOCKER_BUILDKIT=1 -# TODO: The flag --platform="${PLATFORM:-linux/amd64}" \ is required for server image to build, but will break local builds -docker build -f cmd/symbols/Dockerfile.wolfi -t symbols-build "$(pwd)" \ +docker build -f cmd/symbols/Dockerfile -t symbols-build "$(pwd)" \ --target=symbols-build \ --progress=plain \ --build-arg VERSION \ From 7878dade15b8a3560bd0d39a3efe3aab39efa5bc Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 17:22:28 +0100 Subject: [PATCH 56/57] Specify exact Wolfi images to build --- .../background-information/ci/reference.md | 2 +- enterprise/dev/ci/internal/ci/pipeline.go | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/doc/dev/background-information/ci/reference.md b/doc/dev/background-information/ci/reference.md index 2d63351a2750e..d6b1f7106f227 100644 --- a/doc/dev/background-information/ci/reference.md +++ b/doc/dev/background-information/ci/reference.md @@ -115,7 +115,7 @@ sg ci build wolfi Base pipeline (more steps might be included based on branch changes): - **Metadata**: Pipeline metadata -- **Wolfi image builds**: Build Wolfi-based alpine-3.14, Build Wolfi-based cadvisor, Build Wolfi-based codeinsights-db, Build Wolfi-based codeintel-db, Build Wolfi-based frontend, Build Wolfi-based github-proxy, Build Wolfi-based gitserver, Build Wolfi-based grafana, Build Wolfi-based indexed-searcher, Build Wolfi-based jaeger-agent, Build Wolfi-based jaeger-all-in-one, Build Wolfi-based blobstore, Build Wolfi-based blobstore2, Build Wolfi-based node-exporter, Build Wolfi-based postgres-12-alpine, Build Wolfi-based postgres_exporter, Build Wolfi-based precise-code-intel-worker, Build Wolfi-based prometheus, Build Wolfi-based prometheus-gcp, Build Wolfi-based redis-cache, Build Wolfi-based redis-store, Build Wolfi-based redis_exporter, Build Wolfi-based repo-updater, Build Wolfi-based search-indexer, Build Wolfi-based searcher, Build Wolfi-based symbols, Build Wolfi-based syntax-highlighter, Build Wolfi-based worker, Build Wolfi-based migrator, Build Wolfi-based executor, Build Wolfi-based executor-kubernetes, Build Wolfi-based executor-vm, Build Wolfi-based batcheshelper, Build Wolfi-based opentelemetry-collector, Build Wolfi-based embeddings, Build Wolfi-based dind, Build Wolfi-based bundled-executor, Build Wolfi-based server, Build Wolfi-based sg, Build Wolfi-based llm-proxy +- **Wolfi image builds**: Build Wolfi-based postgres-12-alpine - Upload build trace ### Release branch nightly healthcheck build diff --git a/enterprise/dev/ci/internal/ci/pipeline.go b/enterprise/dev/ci/internal/ci/pipeline.go index ed0e34f076e61..9af292e4517d0 100644 --- a/enterprise/dev/ci/internal/ci/pipeline.go +++ b/enterprise/dev/ci/internal/ci/pipeline.go @@ -126,7 +126,42 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) { // Rebuild all images seems reasonable. We need a list somewhere! Maybe we can just use the standard image list though? But not all are wolfi-ified ops.Merge( // TODO: Just hardcode specific images initially - WolfiImagesOperations([]string{}, c.Version, + WolfiImagesOperations([]string{ + "batcheshelper", + "embeddings", + "executor-kubernetes", + "frontend", + "github-proxy", + "gitserver", + "llm-proxy", + "loadtest", + "migrator", + "precise-code-intel-worker", + "repo-updater", + "searcher", + "server", + "symbols", + "worker", + "blobstore", + "cadvisor", + "codeinsights-db", + "codeintel-db", + "indexed-searcher", + "jaeger-agent", + "jaeger-all-in-one", + "node-exporter", + "opentelemetry-collector", + "postgres-12-alpine", + "postgres_exporter", + "prometheus", + "prometheus-gcp", + "redis-cache", + "redis-store", + "redis_exporter", + "search-indexer", + "sg", + "syntax-highlighter", + }, c.Version, c.candidateImageTag(), (numUpdatedBaseImages > 0), ), From b4e44bb94a125f36ab07b06b066ed20a400a4f63 Mon Sep 17 00:00:00 2001 From: Will Dollman Date: Mon, 24 Apr 2023 18:05:57 +0100 Subject: [PATCH 57/57] sg generate --- doc/dev/background-information/ci/reference.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/dev/background-information/ci/reference.md b/doc/dev/background-information/ci/reference.md index d6b1f7106f227..296d6dbdcab46 100644 --- a/doc/dev/background-information/ci/reference.md +++ b/doc/dev/background-information/ci/reference.md @@ -115,8 +115,7 @@ sg ci build wolfi Base pipeline (more steps might be included based on branch changes): - **Metadata**: Pipeline metadata -- **Wolfi image builds**: Build Wolfi-based postgres-12-alpine -- Upload build trace +- **Wolfi image builds**: Build Wolfi-based batcheshelper, Build Wolfi-based embeddings, Build Wolfi-based executor-kubernetes, Build Wolfi-based frontend, Build Wolfi-based github-proxy, Build Wolfi-based gitserver, Build Wolfi-based llm-proxy, Build Wolfi-based loadtest, Build Wolfi-based migrator, Build Wolfi-based precise-code-intel-worker, Build Wolfi-based repo-updater, Build Wolfi-based searcher, Build Wolfi-based server, Build Wolfi-based symbols, Build Wolfi-based worker, Build Wolfi-based blobstore, Build Wolfi-based cadvisor, Build Wolfi-based codeinsights-db, Build Wolfi-based codeintel-db, Build Wolfi-based indexed-searcher, Build Wolfi-based jaeger-agent, Build Wolfi-based jaeger-all-in-one, Build Wolfi-based node-exporter, Build Wolfi-based opentelemetry-collector, Build Wolfi-based postgres-12-alpine, Build Wolfi-based postgres_exporter, Build Wolfi-based prometheus, Build Wolfi-based prometheus-gcp, Build Wolfi-based redis-cache, Build Wolfi-based redis-store, Build Wolfi-based redis_exporter, Build Wolfi-based search-indexer, Build Wolfi-based sg, Build Wolfi-based syntax-highlighter ### Release branch nightly healthcheck build