use buildx to build multi arch image (alibaba#487)

.github/workflows/build-image-and-push.yaml

@@ -65,16 +65,13 @@ jobs:
 
       - name: Build Docker Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make docker-build
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make docker-build
+          GOPROXY="https://proxy.golang.org,direct" make docker-buildx-push
           BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress"
-          docker push $BUILT_IMAGE:$GITHUB_SHA-amd64
-          docker push $BUILT_IMAGE:$GITHUB_SHA-arm64
+          docker push $BUILT_IMAGE:$GITHUB_SHA
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64
-            docker manifest push $image
+            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
           done
 
   build-pilot-image:
@@ -135,16 +132,12 @@ jobs:
 
       - name: Build Pilot-Discovery Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make build-istio
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make build-istio
+          GOPROXY="https://proxy.golang.org,direct" make build-istio
           BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/pilot"
-          docker push $BUILT_IMAGE:$GITHUB_SHA-amd64
-          docker push $BUILT_IMAGE:$GITHUB_SHA-arm64
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64
-            docker manifest push $image
+            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
           done
 
 
@@ -199,14 +192,10 @@ jobs:
           
       - name: Build Gateway Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make build-gateway
-          GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make build-gateway
+          GOPROXY="https://proxy.golang.org,direct" make build-gateway
           BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/proxyv2"
-          docker push $BUILT_IMAGE:$GITHUB_SHA-amd64
-          docker push $BUILT_IMAGE:$GITHUB_SHA-arm64
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64
-            docker manifest push $image
+            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
           done

Makefile.core.mk

@@ -69,12 +69,13 @@ build: prebuild $(OUT)
 
 .PHONY: build-linux
 build-linux: prebuild $(OUT)
-ifeq ($(BUILDX_PLATFORM), true)
+	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(HIGRESS_BINARIES)
+
+$(AMD64_OUT_LINUX)/higress: prebuild $(OUT)
 	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=amd64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/linux_amd64/ $(HIGRESS_BINARIES)
+
+$(ARM64_OUT_LINUX)/higress: prebuild $(OUT)
 	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=arm64 LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh ./out/linux_arm64/ $(HIGRESS_BINARIES)
-else
-	GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(HIGRESS_BINARIES)
-endif
 
 
 .PHONY: build-hgctl
@@ -124,6 +125,8 @@ include docker/docker.mk
 
 docker-build: docker.higress ## Build and push docker images to registry defined by $HUB and $TAG
 
+docker-buildx-push: docker.higress-buildx
+
 docker-build-base:
 	docker buildx build --no-cache --platform linux/amd64,linux/arm64 -t ${HUB}/base:${BASE_VERSION} -f docker/Dockerfile.base . --push
 
@@ -132,16 +135,24 @@ export PARENT_GIT_REVISION:=$(TAG)
 
 export ENVOY_TAR_PATH:=/home/package/envoy.tar.gz
 
-external/package/envoy-$(TARGET_ARCH).tar.gz:
+external/package/envoy-amd64.tar.gz:
+#	cd external/proxy; BUILD_WITH_CONTAINER=1  make test_release
+	cd external/package; wget "https://github.com/alibaba/higress/releases/download/v1.0.0/envoy-amd64.tar.gz"
+
+external/package/envoy-arm64.tar.gz:
 #	cd external/proxy; BUILD_WITH_CONTAINER=1  make test_release
-	cd external/package; wget "https://github.com/alibaba/higress/releases/download/v1.0.0/envoy-$(TARGET_ARCH).tar.gz"
+	cd external/package; wget "https://github.com/alibaba/higress/releases/download/v1.0.0/envoy-arm64.tar.gz"
+
+
+build-pilot:
+	cd external/istio; rm -rf out/linux_amd64; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 make build-linux
+	cd external/istio; rm -rf out/linux_arm64; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=arm64 BUILD_WITH_CONTAINER=1 make build-linux
 
-build-gateway: prebuild external/package/envoy-$(TARGET_ARCH).tar.gz
-	cp -f external/package/envoy-$(TARGET_ARCH).tar.gz external/package/envoy.tar.gz
-	cd external/istio; rm -rf out/linux_$(TARGET_ARCH); GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=$(TARGET_ARCH) BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker
+build-gateway: prebuild external/package/envoy-amd64.tar.gz external/package/envoy-arm64.tar.gz build-pilot
+	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker
 
-build-istio: prebuild
-	cd external/istio; rm -rf out/linux_$(TARGET_ARCH); GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=$(TARGET_ARCH) BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker
+build-istio: prebuild build-pilot
+	cd external/istio; BUILD_WITH_CONTAINER=1 BUILDX_PLATFORM=true DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker	
 
 build-wasmplugins: 
 	./tools/hack/build-wasm-plugins.sh
@@ -161,10 +172,10 @@ ENVOY_LATEST_IMAGE_TAG ?= 1.1.0
 ISTIO_LATEST_IMAGE_TAG ?= 1.1.1
 
 install-dev: pre-install
-	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)-$(TARGET_ARCH)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
+	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
 
 install-dev-wasmplugin: build-wasmplugins pre-install
-	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)-$(TARGET_ARCH)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true'
+	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true'
 
 uninstall:
 	helm uninstall higress -n higress-system
@@ -238,7 +249,7 @@ delete-cluster: $(tools/kind) ## Delete kind cluster.
 # docker pull registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3
 .PHONY: kube-load-image
 kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG.
-	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)-$(TARGET_ARCH)
+	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)
 	tools/hack/docker-pull-image.sh registry.cn-hangzhou.aliyuncs.com/hinsteny/dubbo-provider-demo 0.0.1
 	tools/hack/docker-pull-image.sh registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3 1.0.0-RC3
 	tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0

docker/Dockerfile.higress

@@ -6,11 +6,11 @@ ARG BASE_VERSION=latest
 
 ARG HUB
 
-ARG TARGETARCH
-
 # The following section is used as base image if BASE_DISTRIBUTION=debug
 # This base image is provided by istio, see: https://github.com/istio/istio/blob/master/docker/Dockerfile.base
-FROM ${HUB}/base:${BASE_VERSION}-${TARGETARCH}
+FROM ${HUB}/base:${BASE_VERSION}
+
+ARG TARGETARCH
 
 COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress
 

docker/docker.mk

@@ -12,11 +12,17 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 
-docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} --build-arg TARGETARCH=$(TARGET_ARCH)
+docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress: $(OUT_LINUX)/higress
 docker.higress: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_RULE)
 
+docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
+docker.higress-buildx: $(AMD64_OUT_LINUX)/higress
+docker.higress-buildx: $(ARM64_OUT_LINUX)/higress
+docker.higress-buildx: docker/Dockerfile.higress
+	$(HIGRESS_DOCKER_BUILDX_RULE)
+
 # DOCKER_BUILD_VARIANTS ?=debug distroless
 # Base images have two different forms:
 # * "debug", suffixed as -debug. This is a ubuntu based image with a bunch of debug tools
@@ -29,11 +35,6 @@ DOCKER_ALL_VARIANTS ?= debug distroless
 INCLUDE_UNTAGGED_DEFAULT ?= false
 DEFAULT_DISTRIBUTION=debug
 
-ifeq ($(BUILDX_PLATFORM), true)
-ifeq (docker buildx ls | grep -q container-builder,)
-	docker buildx create --name container-builder --use;
-endif
-HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker buildx build --no-cache --platform linux/amd64,linux/arm64 $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile$(suffix $@) . --push  ); )
-else
-HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT))-$(TARGET_ARCH) -f Dockerfile$(suffix $@) . ); )
-endif
+HIGRESS_DOCKER_BUILDX_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker buildx create --use && docker buildx build --no-cache --platform linux/amd64,linux/arm64 $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/higress:$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . --push  ); )
+HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/higress:$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . ); )
+