diff --git a/.buildkite/capi/aks-cluster/Chart.yaml b/.buildkite/capi/aks-cluster/Chart.yaml new file mode 100644 index 0000000000..c5f665dde4 --- /dev/null +++ b/.buildkite/capi/aks-cluster/Chart.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v2 +name: aks-cluster +description: A Helm chart for aks capi clusters + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/.buildkite/capi/aks-cluster/templates/NOTES.txt b/.buildkite/capi/aks-cluster/templates/NOTES.txt new file mode 100644 index 0000000000..7fd7c9f847 --- /dev/null +++ b/.buildkite/capi/aks-cluster/templates/NOTES.txt @@ -0,0 +1,26 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + + + + +Congratulations on installing {{ .Chart.Name }}! + +You can get the kubeconfig file by running: + +TODO create task file \ No newline at end of file diff --git a/.buildkite/capi/aks-cluster/templates/_helpers.tpl b/.buildkite/capi/aks-cluster/templates/_helpers.tpl new file mode 100644 index 0000000000..8801fb8af1 --- /dev/null +++ b/.buildkite/capi/aks-cluster/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "aks-cluster.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aks-cluster.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aks-cluster.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "aks-cluster.labels" -}} +helm.sh/chart: {{ include "aks-cluster.chart" . }} +clusterctl.cluster.x-k8s.io/move-hierarchy: "true" +{{ include "aks-cluster.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "aks-cluster.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aks-cluster.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "aks-cluster.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "aks-cluster.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/.buildkite/capi/aks-cluster/templates/aks.yaml b/.buildkite/capi/aks-cluster/templates/aks.yaml new file mode 100644 index 0000000000..7fde2ee089 --- /dev/null +++ b/.buildkite/capi/aks-cluster/templates/aks.yaml @@ -0,0 +1,214 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: {{ include "aks-cluster.fullname" . }} + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: {{ include "aks-cluster.fullname" . }}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: {{ include "aks-cluster.fullname" . }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: {{ include "aks-cluster.fullname" . }} + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: {{ include "aks-cluster.fullname" . }} + location: {{ .Values.location }} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: {{ include "aks-cluster.fullname" . }}-vnet + resourceGroup: {{ .Values.resourceGroup }} + subscriptionID: {{ .Values.subscriptionID }} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: {{ include "aks-cluster.fullname" . }}-control-plane + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: {{ include "aks-cluster.fullname" . }} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: {{ include "aks-cluster.fullname" . }}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: {{ printf "'{{ ds.meta_data[\"local_hostname\"] }}'" }} + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: {{ printf "'{{ ds.meta_data[\"local_hostname\"] }}'" }} + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: {{ include "aks-cluster.fullname" . }}-control-plane + replicas: {{ .Values.controlPlaneReplicas }} + version: {{ .Values.kubernetesVersion }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: {{ include "aks-cluster.fullname" . }}-control-plane + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: "" + vmSize: Standard_B2s +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: {{ include "aks-cluster.fullname" . }} + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + clusterName: {{ include "aks-cluster.fullname" . }} + replicas: {{ .Values.workerReplicas }} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ include "aks-cluster.fullname" . }}-workers + clusterName: {{ include "aks-cluster.fullname" . }} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: {{ include "aks-cluster.fullname" . }}-workers + version: {{ .Values.kubernetesVersion }} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: {{ include "aks-cluster.fullname" . }}-workers + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: "" + vmSize: Standard_B2s +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ include "aks-cluster.fullname" . }}-workers + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: {{ include "aks-cluster.fullname" . }}-workers-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: {{ printf "'{{ ds.meta_data[\"local_hostname\"] }}'" }} + preKubeadmCommands: [] +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + name: {{ include "aks-cluster.fullname" . }} + namespace: {{ .Release.namespace | default "default" }} + labels: {{ include "aks-cluster.labels" . | nindent 4 }} +spec: + allowedNamespaces: {} + clientID: {{ .Values.clientID }} + clientSecret: + name: {{ .Values.clientSecret.name }} + namespace: {{ .Values.clientSecret.namespace }} + tenantID: {{ .Values.tenantID }} + type: ServicePrincipal \ No newline at end of file diff --git a/.buildkite/capi/aks-cluster/values.yaml b/.buildkite/capi/aks-cluster/values.yaml new file mode 100644 index 0000000000..5dc87fe873 --- /dev/null +++ b/.buildkite/capi/aks-cluster/values.yaml @@ -0,0 +1,25 @@ +--- +# Default values for aks-cluster. + +# nameOverride: "" +# fullNameOverride: "" + +# Make sure to use same Kubernetes version here as building the GCE image +kubernetesVersion: 1.26.3 + +# Name of the Azure datacenter location. Change this value to your desired location +location: "westus2" + +clientID: "" +tenantID: "" + +resourceGroup: "" + +subscriptionID: "" + +controlPlaneReplicas: 1 +workerReplicas: 1 + +clientSecret: + name: "" + namespace: "" \ No newline at end of file diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index 2235f34c36..72dcb725cd 100644 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -20,4 +20,10 @@ gke() { unset CI PROJECT_ID TEST_BUCKET TEST_SA_NAME } +aks() { + cp charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml .local/artifacts/ + support-bundle -o ".local/artifacts/${CLOUD_PROVIDER}-${BUILDKITE_BUILD_NUMBER}" .buildkite/scripts/bundle.yaml + unset CI TEST_STORAGE_ACCOUNT TEST_RESOURCE_GROUP +} + $CLOUD_PROVIDER \ No newline at end of file diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 56ae8c83ce..bbba42d765 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,6 +21,7 @@ kind_dump() { local TAR="${DUMP}.tar.gz" unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY unset GCP_B64ENCODED_CREDENTIALS PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS + unset AZURE_CLIENT_SECRET kind export logs -n bootstrap "${DUMP}" tar czf "${TAR}" -C "${DUMP}" . buildkite-agent artifact upload "${TAR}" @@ -101,9 +102,78 @@ test_gke() { KUBECONFIG=$(realpath ".local/buildkite-${BUILDKITE_BUILD_NUMBER}-gke-cluster.conf") export KUBECONFIG + buildkite-agent artifact upload $KUBECONFIG + envsubst < ./charts/redpanda/ci/22-gke-tiered-storage-with-creds-values.yaml.tpl > ./charts/redpanda/ci/22-gke-tiered-storage-with-creds-values.yaml } +test_aks() { + echo ~~~ Start AKS variant + + CI=true + + TEST_STORAGE_ACCOUNT="tieredstoragetest${BUILDKITE_BUILD_NUMBER}" + TEST_STORAGE_CONTAINER="testcontainer${BUILDKITE_BUILD_NUMBER}" + TEST_RESOURCE_GROUP="k8s-team-rg-${BUILDKITE_BUILD_NUMBER}" + + # Taken from https://github.com/kubernetes-sigs/cluster-api/blob/ab50d0bc33ccaeb01bde978cd0d25e93efed95ba/docs/book/src/user/quick-start.md?plain=1#L450-L470 + export AZURE_TENANT_ID="9a95fd9e-005d-487a-9a01-d08c1eab2757" + export AZURE_CLIENT_ID="918839a6-3e96-4950-8bf2-f56d0a2e7447" + export AZURE_SUBSCRIPTION_ID="6d88ea35-36ad-4a17-88f8-7ba935e900b0" + + # Base64 encode the variables + export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" + export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" + export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" + export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" + + # Settings needed for AzureClusterIdentity used by the AzureCluster + export AZURE_CLUSTER_IDENTITY_SECRET_NAME="cluster-identity-secret" + export AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE="default" + + export TEST_STORAGE_ACCOUNT TEST_RESOURCE_GROUP TEST_STORAGE_CONTAINER + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az login --service-principal \ + --username $AZURE_CLIENT_ID \ + --password $AZURE_CLIENT_SECRET \ + --tenant $AZURE_TENANT_ID + + # the westus2 is used as buildkite is deployed in us-west-2 which is close to Azure westus2 (Maybe in the same city) + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az group create \ + --name $TEST_RESOURCE_GROUP \ + --location westus2 + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az storage account create \ + --name $TEST_STORAGE_ACCOUNT \ + --resource-group $TEST_RESOURCE_GROUP \ + --location westus2 \ + --sku Standard_RAGRS \ + --kind StorageV2 + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az storage container create \ + --name $TEST_STORAGE_CONTAINER \ + --account-name $TEST_STORAGE_ACCOUNT + + + TEST_AZURE_SHARED_KEY=$(docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 az storage account keys list -g $TEST_RESOURCE_GROUP -n $TEST_STORAGE_ACCOUNT --query '[0].value' -o tsv) + export TEST_AZURE_SHARED_KEY + + trap kind_dump EXIT + + task capi-create-aks CLUSTER_NAME="buildkite-${BUILDKITE_BUILD_NUMBER}" CLUSTER_KUBECONFIG_PATH=".local/buildkite-${BUILDKITE_BUILD_NUMBER}-aks-cluster.conf" + + KUBECONFIG=$(realpath ".local/buildkite-${BUILDKITE_BUILD_NUMBER}-aks-cluster.conf") + export KUBECONFIG + + buildkite-agent artifact upload $KUBECONFIG + + envsubst < ./charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml.tpl > ./charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml +} + test_"${CLOUD_PROVIDER}" # install cert-manager diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 906badf690..12fa9b4265 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -9,7 +9,7 @@ fi source .env -cleanup_eks(){ +cleanup_eks() { echo ~~~ Cleanup EKS variant AWS_ACCESS_KEY_ID="$HELMCHART_TEST_ACCOUNT_AWS_ACCESS_KEY_ID" @@ -25,8 +25,7 @@ cleanup_eks(){ --region "${AWS_REGION}" } - -cleanup_gke(){ +cleanup_gke() { echo ~~~ Cleanup GKE variant CI=true @@ -43,4 +42,30 @@ cleanup_gke(){ gcloud iam service-accounts delete "${TEST_SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" -q } +cleanup_aks() { + echo ~~~ Cleanup AKS variant + + CI=true + + TEST_STORAGE_ACCOUNT="tieredstoragetest${BUILDKITE_BUILD_NUMBER}" + TEST_RESOURCE_GROUP="k8s-team-rg-${BUILDKITE_BUILD_NUMBER}" + + export CI TEST_STORAGE_ACCOUNT TEST_RESOURCE_GROUP + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az login --service-principal \ + --username 918839a6-3e96-4950-8bf2-f56d0a2e7447 \ + --password $AZURE_CLIENT_SECRET \ + --tenant 9a95fd9e-005d-487a-9a01-d08c1eab2757 || true + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az storage account delete \ + --name $TEST_STORAGE_ACCOUNT \ + --resource-group $TEST_RESOURCE_GROUP \ + --yes + + docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 \ + az group delete --resource-group $TEST_RESOURCE_GROUP --yes --debug +} + cleanup_"${CLOUD_PROVIDER}" \ No newline at end of file diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index a16ecabf5c..9e33479a75 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -28,3 +28,16 @@ steps: slack_token_env_var_name: "SLACK_VBOT_TOKEN" conditions: failed: true + - label: ":cloud: test aks cloud storage" + env: + CLOUD_PROVIDER: aks + command: .buildkite/scripts/cloud-test.sh + artifact_paths: + - ".local/artifacts/**/*" + plugins: + - https://$GITHUB_API_TOKEN@github.com/redpanda-data/step-slack-notify-buildkite-plugin.git#main: + message: ":cloud: test aks cloud storage failed" + channel_name: "kubernetes-tests" + slack_token_env_var_name: "SLACK_VBOT_TOKEN" + conditions: + failed: true diff --git a/.buildkite/scripts/cloud-test.sh b/.buildkite/scripts/cloud-test.sh index daeec32fe3..2218b30a10 100755 --- a/.buildkite/scripts/cloud-test.sh +++ b/.buildkite/scripts/cloud-test.sh @@ -8,23 +8,31 @@ set PATH="$(realpath .local/bin):${PATH}" bash -O extglob -c "rm -v charts/redpanda/ci/!(2)[0-9]-*" -ct install --config .github/ct.yaml --upgrade --skip-missing-values | sed 's/>>> /--- /' - - +ct install --config .github/ct.yaml --upgrade --skip-missing-values | sed 's/>>> /~~~ /' eks() { - echo '--- testing that there is data in the s3 bucket' - if (aws s3 ls "s3://${TEST_BUCKET}" --recursive --summarize | grep 'Total Objects: 0'); then - echo "0 Objects in the bucket. Cloud-storage failed." - exit 1 - fi + echo '--- testing that there is data in the s3 bucket' + if (aws s3 ls "s3://${TEST_BUCKET}" --recursive --summarize | grep 'Total Objects: 0'); then + echo "0 Objects in the bucket. Cloud-storage test failed." + exit 1 + fi } gke() { echo '--- testing that there is data in the gcloud bucket' if (gsutil du -s gs://${TEST_BUCKET} | tail -n 1 | grep "0 gs://${TEST_BUCKET}"); then - echo "0 objects in the bucket. Cloud-storage failed." - exit 1 + echo "0 objects in the bucket. Cloud-storage test failed." + exit 1 + fi +} + +aks() { + echo '--- testing that there is data in the azure storage container' + if (docker run -v $(realpath .azure):/root/.azure mcr.microsoft.com/azure-cli:2.50.0 az storage blob list -c $TEST_STORAGE_CONTAINER --account-key $TEST_AZURE_SHARED_KEY --account-name $TEST_STORAGE_ACCOUNT --query "[].{name:name}" --output tsv | grep manifest.json ); then + echo "Manifest found. Success!" + else + echo "No manifest uploaded. Cloud-storage test failed." + exit 1 fi } diff --git a/Taskfile.yaml b/Taskfile.yaml index 24d9c9f789..9509de7369 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -127,6 +127,23 @@ tasks: KIND_FLAGS: "" - "{{.BINDIR}}/clusterctl init -i {{ .CAPI_INFRASTRUCTURE }}:{{ .GCP_PROVIDER_VERSION }} -v7 --wait-providers" + capi-bootstrap-aks: + deps: + - tool:clusterctl + env: + EXP_MACHINE_POOL: true + vars: + CAPI_INFRASTRUCTURE: '{{ default "unknown" .CAPI_INFRASTRUCTURE }}' + AKS_PROVIDER_VERSION: v1.10.0 + cmds: + - echo ~~~ Create kind cluster + - task: kind-create + vars: + KIND_CLUSTERNAME: bootstrap + KIND_FLAGS: "" + - echo ~~~ Initialize cluster API core controllers + - "{{.BINDIR}}/clusterctl init -i {{ .CAPI_INFRASTRUCTURE }}:{{ .AKS_PROVIDER_VERSION }} -v7 --wait-providers" + capi-create-eks: deps: - aws-check-login @@ -143,7 +160,7 @@ tasks: AWS_B64ENCODED_CREDENTIALS: sh: "{{.BINDIR}}/clusterawsadm bootstrap credentials encode-as-profile 2>/dev/null" - "{{.BINDIR}}/clusterawsadm bootstrap iam create-cloudformation-stack --config=.buildkite/capi/eks-bootstrap.yaml" - - helm install -n {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}} .buildkite/capi/eks-cluster --create-namespace --debug + - helm install -n {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}} .buildkite/capi/eks-cluster --create-namespace --debug - kubectl wait --for=condition=ready cluster {{.CLUSTER_NAME}}-eks-cluster --timeout=40m - kubectl wait --for=condition=Ready machinepool {{.CLUSTER_NAME}}-eks-cluster-pool-0 --timeout=20m @@ -161,6 +178,65 @@ tasks: - kubectl wait --for=condition=ready cluster {{.CLUSTER_NAME}}-gke-cluster --timeout=40m - kubectl wait --for=condition=Ready machinepool {{.CLUSTER_NAME}}-gke-cluster-mp-0 --timeout=20m + capi-create-aks: + cmds: + - task: capi-bootstrap-aks + vars: + CAPI_INFRASTRUCTURE: azure + # Create a secret to include the password of the Service Principal identity created in Azure + # This secret will be referenced by the AzureClusterIdentity used by the AzureCluster + - echo ~~~ Install Azure cluster API controllers + - kubectl create ns "{{.AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}}" || true + - kubectl delete secret "{{.AZURE_CLUSTER_IDENTITY_SECRET_NAME}}" --namespace "{{.AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}}" || true + - kubectl create secret generic "{{.AZURE_CLUSTER_IDENTITY_SECRET_NAME}}" --from-literal=clientSecret="{{.AZURE_CLIENT_SECRET}}" --namespace "{{.AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}}" + - | + helm install -n {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}} .buildkite/capi/aks-cluster \ + --create-namespace \ + --set clientID={{.AZURE_CLIENT_ID}} \ + --set tenantID={{.AZURE_TENANT_ID}} \ + --set resourceGroup={{.TEST_RESOURCE_GROUP}} \ + --set subscriptionID={{.AZURE_SUBSCRIPTION_ID}} \ + --set workerReplicas=3 \ + --set clientSecret.name={{.AZURE_CLUSTER_IDENTITY_SECRET_NAME}} \ + --set clientSecret.namespace={{.AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}} \ + --debug + - echo ~~~ Wait for Azure infra to be provisioned + - kubectl wait --for=condition=ready cluster {{.CLUSTER_NAME}}-aks-cluster --timeout=40m -n {{.CLUSTER_NAMESPACE}} + - kubectl wait --for=condition=Ready azuremachines --all --timeout=40m -n {{.CLUSTER_NAMESPACE}} + - "{{.BINDIR}}/clusterctl get kubeconfig --namespace {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}}-aks-cluster > {{.CLUSTER_KUBECONFIG_PATH}}" + - echo ~~~ Azure K8S cluster + - echo ~~~ Install Azure Cloud Controller Manager + - | + helm install \ + --kubeconfig={{.CLUSTER_KUBECONFIG_PATH}} \ + --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure \ + --generate-name \ + --set infra.clusterName={{.CLUSTER_NAME}} \ + --set cloudControllerManager.clusterCIDR="192.168.0.0/16" \ + --debug + - echo "~~~ Install project calico - CNI" + - | + helm repo add projectcalico https://docs.tigera.io/calico/charts \ + --kubeconfig={{.CLUSTER_KUBECONFIG_PATH}} + - | + helm install calico projectcalico/tigera-operator \ + --kubeconfig={{.CLUSTER_KUBECONFIG_PATH}} \ + -f https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/addons/calico/values.yaml \ + --namespace tigera-operator --create-namespace + - echo ~~~ Installing azure disk csi driver + - curl -skSLo install-driver.sh https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/v1.28.1/deploy/install-driver.sh + - chmod +x ./install-driver.sh + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} ./install-driver.sh v1.28.1 snapshot + - rm ./install-driver.sh + - echo ~~~ Installing azure storage class + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} kubectl create -f https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/deploy/example/storageclass-azuredisk-csi.yaml + - echo ~~~ Wait for all nodes to become ready + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} kubectl get pod -A + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} kubectl get nodes + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} kubectl wait --for=condition=ready nodes --all --timeout=10m + - KUBECONFIG={{.CLUSTER_KUBECONFIG_PATH}} kubectl get nodes + - echo ~~~ Azure K8S cluster created + capi-delete: internal: true cmds: @@ -184,6 +260,12 @@ tasks: - kubectl delete cluster {{.CLUSTER_NAME}}-gke-cluster --timeout=20m - helm uninstall --wait -n {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}} --timeout 10m --debug + capi-delete-aks: + deps: + - tool:helm + cmds: + - helm uninstall --wait -n {{.CLUSTER_NAMESPACE}} {{.CLUSTER_NAME}} --timeout 60m --debug + # This is if you are running locally and not in CI gke-auth-login: deps: diff --git a/charts/redpanda/Chart.yaml b/charts/redpanda/Chart.yaml index 48a3557cb3..99e030a175 100644 --- a/charts/redpanda/Chart.yaml +++ b/charts/redpanda/Chart.yaml @@ -23,7 +23,7 @@ type: application # The chart version and the app version are not the same and will not track # together. The chart version is a semver representation of changes to this # chart. -version: 4.0.56 +version: 4.0.57 # The app version is the default version of Redpanda to install. appVersion: v23.1.13 diff --git a/charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml.tpl b/charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml.tpl new file mode 100644 index 0000000000..07cbe67a24 --- /dev/null +++ b/charts/redpanda/ci/23-aks-tiered-storage-with-creds-values.yaml.tpl @@ -0,0 +1,21 @@ +storage: + persistentVolume: + storageClass: managed-csi + tieredConfig: + cloud_storage_enabled: true + cloud_storage_credentials_source: config_file + cloud_storage_segment_max_upload_interval_sec: 1 + cloud_storage_azure_storage_account: ${TEST_STORAGE_ACCOUNT} + cloud_storage_azure_container: ${TEST_STORAGE_CONTAINER} + cloud_storage_azure_shared_key: ${TEST_AZURE_SHARED_KEY} +license_key: "${REDPANDA_SAMPLE_LICENSE}" + +resources: + cpu: + cores: 400m + memory: + container: + max: 2.0Gi + redpanda: + memory: 1Gi + reserveMemory: 100Mi \ No newline at end of file diff --git a/tasks/ToolTasks.yaml b/tasks/ToolTasks.yaml index b666c51dfd..cffb6286da 100644 --- a/tasks/ToolTasks.yaml +++ b/tasks/ToolTasks.yaml @@ -28,8 +28,7 @@ tasks: deps: [mklocal] cmds: - echo "~~~ Installing clusterawsadm" - - mkdir -p {{.BINDIR}} - - curl -Ls https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/{{.CLUSTERAWSADM_VERSION}}/clusterawsadm-linux-amd64 -o {{.BINDIR}}/clusterawsadm-{{.CLUSTERAWSADM_VERSION}} + - curl -Ls https://github.com/kubernetes-sigs/cluster-api-provider-aws/releases/download/{{.CLUSTERAWSADM_VERSION}}/clusterawsadm-{{OS}}-{{ARCH}} -o {{.BINDIR}}/clusterawsadm-{{.CLUSTERAWSADM_VERSION}} - chmod 0775 {{.BINDIR}}/clusterawsadm-{{.CLUSTERAWSADM_VERSION}} - ln -sf clusterawsadm-{{.CLUSTERAWSADM_VERSION}} {{.BINDIR}}/clusterawsadm status: @@ -39,8 +38,7 @@ tasks: deps: [mklocal] cmds: - echo "~~~ Installing clusterctl" - - mkdir -p {{.BINDIR}} - - curl -Ls https://github.com/kubernetes-sigs/cluster-api/releases/download/{{.CLUSTERCTL_VERSION}}/clusterctl-linux-amd64 -o {{.BINDIR}}/clusterctl-{{.CLUSTERCTL_VERSION}} + - curl -Ls https://github.com/kubernetes-sigs/cluster-api/releases/download/{{.CLUSTERCTL_VERSION}}/clusterctl-{{OS}}-{{ARCH}} -o {{.BINDIR}}/clusterctl-{{.CLUSTERCTL_VERSION}} - chmod 0775 {{.BINDIR}}/clusterctl-{{.CLUSTERCTL_VERSION}} - ln -sf clusterctl-{{.CLUSTERCTL_VERSION}} {{.BINDIR}}/clusterctl status: @@ -50,7 +48,7 @@ tasks: deps: [mklocal] cmds: - echo '~~~ Installing helm' - - curl -Ls https://get.helm.sh/helm-{{.HELM_VERSION}}-linux-amd64.tar.gz | tar xz -C {{.BINDIR}} linux-amd64/helm --strip 1 --transform 's@$@-{{.HELM_VERSION}}@' + - curl -Ls https://get.helm.sh/helm-{{.HELM_VERSION}}-{{OS}}-{{ARCH}}.tar.gz | tar xz -C {{.BINDIR}} {{OS}}-{{ARCH}}/helm --strip 1 --transform 's@$@-{{.HELM_VERSION}}@' - ln -s helm-{{.HELM_VERSION}} {{.BINDIR}}/helm status: - test -f {{.BINDIR}}/helm-{{.HELM_VERSION}} @@ -59,7 +57,7 @@ tasks: deps: [mklocal] cmds: - echo '~~~ Installing kind' - - curl -Ls https://github.com/kubernetes-sigs/kind/releases/download/{{.KIND_VERSION}}/kind-linux-amd64 -o {{.BINDIR}}/kind-{{.KIND_VERSION}} + - curl -Ls https://github.com/kubernetes-sigs/kind/releases/download/{{.KIND_VERSION}}/kind-{{OS}}-{{ARCH}} -o {{.BINDIR}}/kind-{{.KIND_VERSION}} - chmod +x {{.BINDIR}}/kind-{{.KIND_VERSION}} - ln -s kind-{{.KIND_VERSION}} {{.BINDIR}}/kind status: @@ -72,8 +70,8 @@ tasks: sh: curl -Ls -s https://dl.k8s.io/release/stable.txt cmds: - echo '~~~ Installing kubectl' - - curl -Ls https://dl.k8s.io/{{.KUBECTL_VERSION}}/bin/linux/amd64/kubectl -o {{.BINDIR}}/kubectl-{{.KUBECTL_VERSION}} - - echo "$(curl -Ls https://dl.k8s.io/{{.KUBECTL_VERSION}}/bin/linux/amd64/kubectl.sha256) {{.BINDIR}}/kubectl-{{.KUBECTL_VERSION}}" | sha256sum --check + - curl -Ls https://dl.k8s.io/{{.KUBECTL_VERSION}}/bin/{{OS}}/{{ARCH}}/kubectl -o {{.BINDIR}}/kubectl-{{.KUBECTL_VERSION}} + - echo "$(curl -Ls https://dl.k8s.io/{{.KUBECTL_VERSION}}/bin/{{OS}}/{{ARCH}}/kubectl.sha256) {{.BINDIR}}/kubectl-{{.KUBECTL_VERSION}}" | sha256sum --check - chmod +x {{.BINDIR}}/kubectl-{{.KUBECTL_VERSION}} - ln -s kubectl-{{.KUBECTL_VERSION}} {{.BINDIR}}/kubectl status: @@ -83,7 +81,7 @@ tasks: deps: [mklocal] cmds: - echo '~~~ Installing chart testing' - - curl -Ls https://github.com/joejulian/chart-testing/releases/download/v{{.CT_VERSION}}/chart-testing_{{.CT_VERSION}}_linux_amd64.tar.gz | tar xz -C {{.BINDIR}} ct --transform 's@$@-{{.CT_VERSION}}@' + - curl -Ls https://github.com/joejulian/chart-testing/releases/download/v{{.CT_VERSION}}/chart-testing_{{.CT_VERSION}}_{{OS}}_{{ARCH}}.tar.gz | tar xz -C {{.BINDIR}} ct --transform 's@$@-{{.CT_VERSION}}@' - ln -s ct-{{.CT_VERSION}} {{.BINDIR}}/ct status: - test -f {{.BINDIR}}/ct-{{.CT_VERSION}} @@ -95,7 +93,7 @@ tasks: sh: mktemp -d cmds: - echo '~~~ Installing aws cli' - - curl -Ls "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" + - curl -Ls "https://awscli.amazonaws.com/awscli-exe-{{OS}}-x86_64.zip" -o "/tmp/awscliv2.zip" - unzip -q -d "$TEMPDIR" /tmp/awscliv2.zip - "${TEMPDIR}/aws/install --bin-dir $(realpath .local/bin) --install-dir $(realpath .local/aws-cli)" status: @@ -105,7 +103,7 @@ tasks: deps: [mklocal] cmds: - echo '~~~ Installing support bundle' - - curl -Ls https://github.com/replicatedhq/troubleshoot/releases/download/{{.SUPPORT_BUNDLE_VERSION}}/support-bundle_linux_amd64.tar.gz | tar xz -C {{.BINDIR}} support-bundle --transform 's@$@-{{.SUPPORT_BUNDLE_VERSION}}@' + - curl -Ls https://github.com/replicatedhq/troubleshoot/releases/download/{{.SUPPORT_BUNDLE_VERSION}}/support-bundle_{{OS}}_{{ARCH}}.tar.gz | tar xz -C {{.BINDIR}} support-bundle --transform 's@$@-{{.SUPPORT_BUNDLE_VERSION}}@' - ln -s support-bundle-{{.SUPPORT_BUNDLE_VERSION}} {{.BINDIR}}/support-bundle status: - test -f {{.BINDIR}}/support-bundle-{{.SUPPORT_BUNDLE_VERSION}} @@ -113,8 +111,8 @@ tasks: gcloud: cmds: - echo '~~~ Installing gcloud' - - curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-433.0.0-linux-x86_64.tar.gz - - tar -xf google-cloud-cli-433.0.0-linux-x86_64.tar.gz + - curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-433.0.0-{{OS}}-x86_64.tar.gz + - tar -xf google-cloud-cli-433.0.0-{{OS}}-x86_64.tar.gz - ./google-cloud-sdk/install.sh --usage-reporting=false --quiet --path-update=true - "ln -s $(realpath ./google-cloud-sdk/bin/gcloud) {{.BINDIR}}/gcloud" - "ln -s $(realpath ./google-cloud-sdk/bin/gsutil) {{.BINDIR}}/gsutil"