From 9f4e45445fda4b83d10ebba94522416d1c2da272 Mon Sep 17 00:00:00 2001 From: Spencer Judge Date: Fri, 18 Nov 2022 16:34:08 -0800 Subject: [PATCH 1/3] Add callable workflow for only building the server image --- .github/workflows/docker-build-only.yml | 96 +++++++++++++++++++++++++ admin-tools.Dockerfile | 7 +- server.Dockerfile | 11 +-- 3 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/docker-build-only.yml diff --git a/.github/workflows/docker-build-only.yml b/.github/workflows/docker-build-only.yml new file mode 100644 index 00000000..4310fb61 --- /dev/null +++ b/.github/workflows/docker-build-only.yml @@ -0,0 +1,96 @@ +# This workflow builds the server docker image and exposes it as an artifact which may be used by +# other GitHub workflow jobs. This is useful for creating docker images that you want to use to +# test a certain commit of server without publishing them. + +name: Build Docker Image as Artifact + +on: + workflow_call: + inputs: + # TODO: Shouldn't be required. If not set, use existing submodule. + temporal-server-repo-ref: + type: string + required: true + temporal-server-repo-path: + type: string + default: "temporalio/temporal" + docker-builds-repo-ref: + type: string + default: "main" + +jobs: + build-image: + # TODO: use bigger runner when available. Seems like they're stuck or not enough? + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Checkout docker builds repository + uses: actions/checkout@v3 + with: + # Must specify repo path, or it will use path of calling workflow + repository: temporalio/docker-builds + ref: ${{ inputs.docker-builds-repo-ref }} + submodules: "true" + + - name: Checkout temporal server repository + uses: actions/checkout@v3 + with: + repository: ${{ inputs.temporal-server-repo-path }} + path: temporal-server-checkedout + ref: ${{ inputs.temporal-server-repo-ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + + - name: Build Server Image + uses: docker/build-push-action@v3 + with: + context: . + file: server.Dockerfile + build-args: | + TEMPORAL_REPO_PATH=temporal-server-checkedout + GOFLAGS=-buildvcs=false + tags: localhost:5000/temporal-server:latest + push: true + + - name: Build Admin tools Image + uses: docker/build-push-action@v3 + with: + context: . + file: admin-tools.Dockerfile + build-args: | + TEMPORAL_REPO_PATH=temporal-server-checkedout + SERVER_IMAGE=localhost:5000/temporal-server:latest + GOFLAGS=-buildvcs=false + tags: localhost:5000/temporal-admin-tools:latest + push: true + + - name: Build Autosetup Image + uses: docker/build-push-action@v3 + with: + context: . + file: auto-setup.Dockerfile + build-args: | + SERVER_IMAGE=localhost:5000/temporal-server:latest + ADMIN_TOOLS_IMAGE=localhost:5000/temporal-admin-tools:latest + tags: temporal-autosetup:latest + outputs: type=docker,dest=/tmp/temporal-autosetup.tar + + # Upload-artifact has no good way to flatten paths, so we need to move the compose file + # to avoid some disgustingly long inner path inside the artifact zip. + - name: Copy compose file + run: cp ./temporal-server-checkedout/develop/docker-compose/docker-compose.yml /tmp/docker-compose.yml + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: temporal-server-docker + path: | + /tmp/temporal-autosetup.tar + /tmp/docker-compose.yml diff --git a/admin-tools.Dockerfile b/admin-tools.Dockerfile index 4ed26eaa..bf6de653 100644 --- a/admin-tools.Dockerfile +++ b/admin-tools.Dockerfile @@ -5,15 +5,18 @@ ARG GOPROXY ##### Temporal Admin Tools builder ##### FROM ${BASE_BUILDER_IMAGE} AS admin-tools-builder +ARG TEMPORAL_REPO_PATH=temporal +ARG GOFLAGS +ENV GOFLAGS ${GOFLAGS} WORKDIR /home/builder # cache Temporal packages as a docker layer -COPY ./temporal/go.mod ./temporal/go.sum ./temporal/ +COPY ./${TEMPORAL_REPO_PATH}/go.mod ./${TEMPORAL_REPO_PATH}/go.sum ./temporal/ RUN (cd ./temporal && go mod download all) # build -COPY . . +COPY ./${TEMPORAL_REPO_PATH} ./temporal RUN (cd ./temporal && make temporal-cassandra-tool temporal-sql-tool tdbg) diff --git a/server.Dockerfile b/server.Dockerfile index f51a64f1..88322d1c 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -1,15 +1,16 @@ ARG BASE_BUILDER_IMAGE=temporalio/base-builder:1.11.0 ARG BASE_SERVER_IMAGE=temporalio/base-server:1.12.0 -ARG GOPROXY ##### Builder ##### FROM ${BASE_BUILDER_IMAGE} AS temporal-builder - +ARG TEMPORAL_REPO_PATH=temporal +ARG GOFLAGS +ENV GOFLAGS ${GOFLAGS} WORKDIR /home/builder # cache Temporal packages as a docker layer -COPY ./temporal/go.mod ./temporal/go.sum ./temporal/ +COPY ./${TEMPORAL_REPO_PATH}/go.mod ./${TEMPORAL_REPO_PATH}/go.sum ./temporal/ RUN (cd ./temporal && go mod download all) # cache tctl packages as a docker layer @@ -17,12 +18,14 @@ COPY ./tctl/go.mod ./tctl/go.sum ./tctl/ RUN (cd ./tctl && go mod download all) # build -COPY . . +COPY ./tctl ./tctl +COPY ./${TEMPORAL_REPO_PATH} ./temporal RUN (cd ./temporal && make temporal-server) RUN (cd ./tctl && make build) ##### Temporal server ##### FROM ${BASE_SERVER_IMAGE} as temporal-server + WORKDIR /etc/temporal ENV TEMPORAL_HOME /etc/temporal From d2ce63bf98e72d8ec9bc6b0a3a72c85ff333d2d7 Mon Sep 17 00:00:00 2001 From: Spencer Judge Date: Tue, 6 Dec 2022 11:29:59 -0800 Subject: [PATCH 2/3] Change submodule rather than whole new checkout --- .github/workflows/docker-build-only.yml | 28 +++++++++++++++---------- admin-tools.Dockerfile | 10 ++++----- server.Dockerfile | 10 ++++----- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker-build-only.yml b/.github/workflows/docker-build-only.yml index 4310fb61..75d5ea1b 100644 --- a/.github/workflows/docker-build-only.yml +++ b/.github/workflows/docker-build-only.yml @@ -37,11 +37,22 @@ jobs: submodules: "true" - name: Checkout temporal server repository - uses: actions/checkout@v3 - with: - repository: ${{ inputs.temporal-server-repo-path }} - path: temporal-server-checkedout - ref: ${{ inputs.temporal-server-repo-ref }} + # I have no idea why all this hullabaloo with explicitly fetching the other repo's branch + # is necessary, you should be able to just `git submodule update --init` and have it work, + # but possibly something about the barebones nature of the checkout action makes that not + # work. + run: | + git submodule set-url temporal ${{ format('https://github.com/{0}', inputs.temporal-server-repo-path) }} + git submodule set-branch --branch ${{ inputs.temporal-server-repo-ref }} -- temporal + git submodule sync + cat .gitmodules + echo "Updating temporal submodule" + cd temporal + git remote show origin + git fetch --depth 1 origin ${{ inputs.temporal-server-repo-ref }}:refs/remotes/origin/${{ inputs.temporal-server-repo-ref }} + git checkout origin/${{ inputs.temporal-server-repo-ref }} + cd .. + git submodule status temporal - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -53,9 +64,6 @@ jobs: with: context: . file: server.Dockerfile - build-args: | - TEMPORAL_REPO_PATH=temporal-server-checkedout - GOFLAGS=-buildvcs=false tags: localhost:5000/temporal-server:latest push: true @@ -65,9 +73,7 @@ jobs: context: . file: admin-tools.Dockerfile build-args: | - TEMPORAL_REPO_PATH=temporal-server-checkedout SERVER_IMAGE=localhost:5000/temporal-server:latest - GOFLAGS=-buildvcs=false tags: localhost:5000/temporal-admin-tools:latest push: true @@ -85,7 +91,7 @@ jobs: # Upload-artifact has no good way to flatten paths, so we need to move the compose file # to avoid some disgustingly long inner path inside the artifact zip. - name: Copy compose file - run: cp ./temporal-server-checkedout/develop/docker-compose/docker-compose.yml /tmp/docker-compose.yml + run: cp ./temporal/develop/docker-compose/docker-compose.yml /tmp/docker-compose.yml - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/admin-tools.Dockerfile b/admin-tools.Dockerfile index bf6de653..b3d1f877 100644 --- a/admin-tools.Dockerfile +++ b/admin-tools.Dockerfile @@ -5,18 +5,18 @@ ARG GOPROXY ##### Temporal Admin Tools builder ##### FROM ${BASE_BUILDER_IMAGE} AS admin-tools-builder -ARG TEMPORAL_REPO_PATH=temporal -ARG GOFLAGS -ENV GOFLAGS ${GOFLAGS} WORKDIR /home/builder # cache Temporal packages as a docker layer -COPY ./${TEMPORAL_REPO_PATH}/go.mod ./${TEMPORAL_REPO_PATH}/go.sum ./temporal/ +COPY ./temporal/go.mod ./temporal/go.sum ./temporal/ RUN (cd ./temporal && go mod download all) # build -COPY ./${TEMPORAL_REPO_PATH} ./temporal +COPY ./temporal ./temporal +# Git info is needed for Go build to attach VCS information properly +COPY ./.git ./.git +COPY ./.gitmodules ./.gitmodules RUN (cd ./temporal && make temporal-cassandra-tool temporal-sql-tool tdbg) diff --git a/server.Dockerfile b/server.Dockerfile index 88322d1c..cd8df2e3 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -3,14 +3,11 @@ ARG BASE_SERVER_IMAGE=temporalio/base-server:1.12.0 ##### Builder ##### FROM ${BASE_BUILDER_IMAGE} AS temporal-builder -ARG TEMPORAL_REPO_PATH=temporal -ARG GOFLAGS -ENV GOFLAGS ${GOFLAGS} WORKDIR /home/builder # cache Temporal packages as a docker layer -COPY ./${TEMPORAL_REPO_PATH}/go.mod ./${TEMPORAL_REPO_PATH}/go.sum ./temporal/ +COPY ./temporal/go.mod ./temporal/go.sum ./temporal/ RUN (cd ./temporal && go mod download all) # cache tctl packages as a docker layer @@ -19,7 +16,10 @@ RUN (cd ./tctl && go mod download all) # build COPY ./tctl ./tctl -COPY ./${TEMPORAL_REPO_PATH} ./temporal +COPY ./temporal ./temporal +# Git info is needed for Go build to attach VCS information properly +COPY ./.git ./.git +COPY ./.gitmodules ./.gitmodules RUN (cd ./temporal && make temporal-server) RUN (cd ./tctl && make build) From 3342af80d22d1010c1438e9fad4a735270fab927 Mon Sep 17 00:00:00 2001 From: Spencer Judge Date: Wed, 7 Dec 2022 11:23:18 -0800 Subject: [PATCH 3/3] Address review comments --- .github/workflows/docker-build-only.yml | 8 +++++--- admin-tools.Dockerfile | 3 ++- server.Dockerfile | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build-only.yml b/.github/workflows/docker-build-only.yml index 75d5ea1b..384fb472 100644 --- a/.github/workflows/docker-build-only.yml +++ b/.github/workflows/docker-build-only.yml @@ -7,10 +7,11 @@ name: Build Docker Image as Artifact on: workflow_call: inputs: - # TODO: Shouldn't be required. If not set, use existing submodule. + # If set, change the temporal submodule to use the repo path from `temporal-server-repo-path` + # and use this ref when checking it out. If it isn't set, the submodule as defined in this + # repo will be used when building server. temporal-server-repo-ref: type: string - required: true temporal-server-repo-path: type: string default: "temporalio/temporal" @@ -31,12 +32,13 @@ jobs: - name: Checkout docker builds repository uses: actions/checkout@v3 with: - # Must specify repo path, or it will use path of calling workflow + # Must specify repo path, or it'll use path of calling workflow (since this is a reusable wf) repository: temporalio/docker-builds ref: ${{ inputs.docker-builds-repo-ref }} submodules: "true" - name: Checkout temporal server repository + if: inputs.temporal-server-repo-ref # I have no idea why all this hullabaloo with explicitly fetching the other repo's branch # is necessary, you should be able to just `git submodule update --init` and have it work, # but possibly something about the barebones nature of the checkout action makes that not diff --git a/admin-tools.Dockerfile b/admin-tools.Dockerfile index b3d1f877..77e4ba56 100644 --- a/admin-tools.Dockerfile +++ b/admin-tools.Dockerfile @@ -14,7 +14,8 @@ RUN (cd ./temporal && go mod download all) # build COPY ./temporal ./temporal -# Git info is needed for Go build to attach VCS information properly +# Git info is needed for Go build to attach VCS information properly. +# See the `buildvcs` Go flag: https://pkg.go.dev/cmd/go COPY ./.git ./.git COPY ./.gitmodules ./.gitmodules RUN (cd ./temporal && make temporal-cassandra-tool temporal-sql-tool tdbg) diff --git a/server.Dockerfile b/server.Dockerfile index cd8df2e3..84628709 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -17,7 +17,8 @@ RUN (cd ./tctl && go mod download all) # build COPY ./tctl ./tctl COPY ./temporal ./temporal -# Git info is needed for Go build to attach VCS information properly +# Git info is needed for Go build to attach VCS information properly. +# See the `buildvcs` Go flag: https://pkg.go.dev/cmd/go COPY ./.git ./.git COPY ./.gitmodules ./.gitmodules RUN (cd ./temporal && make temporal-server)