feat: implements outbound proxy support for arc extension (#695)

* feat: implements outbound proxy support for arc extension

Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com>

.pipelines/e2e-job-azure.yaml

@@ -28,3 +28,5 @@ jobs:
       osTypes:
       - "linux"
       - "windows"
+  # this will ensure any changes to provider works on arc extension too.
+  - template: templates/arc/e2e-extension-test.yaml

.pipelines/nightly.yaml

@@ -40,4 +40,4 @@ jobs:
       clusterConfigs:
       - "csi-secrets-store-soak-linux-aks"
       - "csi-secrets-store-soak-win-aks"
-  - template: templates/e2e-test-kind-arc.yaml
+  - template: templates/arc/e2e-test-kind.yaml

.pipelines/templates/arc/cluster-connect.yaml

@@ -0,0 +1,23 @@
+steps:
+  - script: |
+      az group create -n ${AZURE_CLUSTER_NAME} -l $(AZURE_CANARY_LOCATION)
+      az connectedk8s connect -n ${AZURE_CLUSTER_NAME} -g ${AZURE_CLUSTER_NAME} --no-wait
+      # It takes time for Arc pods to come up. Sometimes, in such cases helm might report unable to install helm release, but in fact Arc operators get installed and can connect to the cluster. Also, az connectedk8s connect will go through different phases (Connecting, Connected etc.) of installation. So to address both, we are checking the status later without waiting.
+      echo "verifying cluster connectivity..."
+      for i in $(seq 1 25); do
+        provisioningState=$(az connectedk8s list --resource-group ${AZURE_CLUSTER_NAME} --query "[*].provisioningState" -otsv)
+        connectivityStatus=$(az connectedk8s list --resource-group ${AZURE_CLUSTER_NAME} --query "[*].connectivityStatus" -otsv)
+        if [ "$provisioningState" == "Succeeded" ]  && [ "$connectivityStatus" == "Connected" ]; then
+          echo "KinD cluster is 'Connected'"
+          break
+        else
+          echo "Provisioning state - $provisioningState, Connectivity status - $connectivityStatus"
+          sleep 1
+        fi
+      done
+      if [ "$connectivityStatus" != "Connected" ]; then
+        echo "failed to connect to the cluster."
+        exit 1
+      fi
+    displayName: "connect KinD cluster"
+    condition: succeeded()

.pipelines/templates/arc/e2e-extension-test.yaml

@@ -0,0 +1,63 @@
+jobs:
+  - job: e2e_arc_test
+    variables:
+    - name: AZURE_ENVIRONMENT_FILEPATH
+      value: /etc/kubernetes/custom_environment.json
+    - name: VOLUME_NAME
+      value: cloudenvfile-vol
+    - group: csi-secrets-store-e2e
+    steps:
+      - template: ../build-images.yaml
+        parameters:
+          registry: e2e
+          ciKindCluster: true
+      - template: setup.yaml
+      - script: |
+          # version should always be greater than 0.0.1 as this is the minimum version configured for reconciliation.
+          patchVersion="$(date +%s)"
+          version="0.0.$patchVersion"
+          echo "##vso[task.setvariable variable=EXT_VERSION]$version"
+
+          helm dependency update manifest_staging/charts/csi-secrets-store-provider-azure
+          helm package manifest_staging/charts/csi-secrets-store-provider-azure --version $version
+
+          # echo "Authenticating..."
+          az acr login -n $(STAGING_REGISTRY_NAME)
+
+          # echo 'Pushing chart...'
+          oras push $(STAGING_REGISTRY):$version ./csi-secrets-store-provider-azure-$version.tgz:application/tar+gzip --debug
+
+          registry=$(STAGING_REGISTRY)
+          respository=${registry#*/}
+          echo "##vso[task.setvariable variable=REGISTRY_REPO]$respository"
+        displayName: 'Push OCI helm chart to ACR'
+        condition: succeeded()
+      - template: cluster-connect.yaml
+      - template: extension-create.yaml
+        parameters:
+          azureClusterName: $(AZURE_CLUSTER_NAME)
+          extensionVersion: $(EXT_VERSION)
+          releaseTrain: dev
+          configurationSettings: "'secrets-store-csi-driver.enableSecretRotation=true' \
+          'secrets-store-csi-driver.rotationPollInterval=30s' \
+          'linux.image.tag=$(IMAGE_VERSION)' \
+          'linux.image.repository=$(REGISTRY)/provider-azure' \
+          'secrets-store-csi-driver.syncSecret.enabled=true' \
+          'linux.volumes[0].name=$(VOLUME_NAME)' \
+          'linux.volumes[0].hostPath.path=$(AZURE_ENVIRONMENT_FILEPATH)' \
+          'linux.volumes[0].hostPath.type=File' \
+          'linux.volumeMounts[0].name=$(VOLUME_NAME)' \
+          'linux.volumeMounts[0].mountPath=$(AZURE_ENVIRONMENT_FILEPATH)'"
+      - template: ../e2e-test.yaml
+        parameters:
+          testName: "arc extension e2e test"
+          ciKindCluster: true
+          isArcTest: true
+      - template: ../teardown.yaml
+      - template: ../cleanup-images.yaml
+        parameters:
+          imageVersion: $(EXT_VERSION)
+          registryRepo: $(REGISTRY_REPO)
+          subscriptionId: $(SUBSCRIPTION_ID)
+          registryName: $(STAGING_REGISTRY_NAME)
+          isMultiArch: false

.pipelines/templates/arc/e2e-test-kind.yaml

@@ -0,0 +1,34 @@
+jobs:
+  - job: e2e_arc_kind
+    variables:
+    - name: AZURE_ENVIRONMENT_FILEPATH
+      value: /etc/kubernetes/custom_environment.json
+    - name: VOLUME_NAME
+      value: cloudenvfile-vol
+    - group: csi-secrets-store-e2e-kind
+    steps:
+    - template: ../az-login.yaml
+    - template: setup.yaml
+    - script: |
+        make install-helm install-kubectl setup-kind
+      displayName: "install dependencies and setup kind"
+      condition: succeeded()
+    - template: cluster-connect.yaml
+    - template: extension-create.yaml
+      parameters:
+        azureClusterName: $(AZURE_CLUSTER_NAME)
+        releaseTrain: preview
+        configurationSettings: "'secrets-store-csi-driver.enableSecretRotation=true' \
+        'secrets-store-csi-driver.rotationPollInterval=30s' \
+        'secrets-store-csi-driver.syncSecret.enabled=true' \
+        'linux.volumes[0].name=$(VOLUME_NAME)' \
+        'linux.volumes[0].hostPath.path=$(AZURE_ENVIRONMENT_FILEPATH)' \
+        'linux.volumes[0].hostPath.type=File' \
+        'linux.volumeMounts[0].name=$(VOLUME_NAME)' \
+        'linux.volumeMounts[0].mountPath=$(AZURE_ENVIRONMENT_FILEPATH)'"
+    - template: ../e2e-test.yaml
+      parameters:
+        testName: "arc extension e2e test"
+        ciKindCluster: true
+        isArcTest: true
+    - template: ../teardown.yaml

.pipelines/templates/arc/extension-create.yaml

@@ -0,0 +1,53 @@
+parameters:
+  - name: azureClusterName
+    type: string
+  - name: extensionVersion
+    type: string
+    default: ""
+  - name: releaseTrain
+    type: string
+  - name: configurationSettings
+    type: string
+
+steps:
+  - script: |
+      echo "Installing extension..."
+      echo "version - '${{ parameters.extensionVersion }}'"
+      if [[ "${{ parameters.extensionVersion }}" != "" ]]; then
+        EXTRA_ARGS="--version ${{ parameters.extensionVersion }}"
+      fi
+      az k8s-extension create \
+      --name ${{ parameters.azureClusterName }} \
+      --extension-type Microsoft.AzureKeyVaultSecretsProvider \
+      --scope cluster \
+      --cluster-name ${{ parameters.azureClusterName }} \
+      --resource-group ${{ parameters.azureClusterName }} \
+      --cluster-type connectedClusters \
+      --release-train ${{ parameters.releaseTrain }} \
+      --release-namespace kube-system \
+      --configuration-settings ${{ parameters.configurationSettings }} \
+      $EXTRA_ARGS
+
+      # Arc extensions will go through different phases (Pending, Installed etc.) of installation. We want to make sure extension is 'Installed' before running e2e tests.
+      echo "verifying extension install status..."
+      for i in $(seq 1 30); do
+        provisioningState=$(az k8s-extension show -c ${{ parameters.azureClusterName }} -t connectedClusters -n ${{ parameters.azureClusterName }} -g ${{ parameters.azureClusterName }} --query "provisioningState" -otsv)
+        if [ "$provisioningState" == "Succeeded" ]; then
+          echo "AzureKeyVaultSecretsProvider extension is 'Installed'"
+          break
+        else
+          echo "Provisioning state: '$provisioningState'"
+          sleep 2
+        fi
+      done
+      if [ "$provisioningState" != "Succeeded" ]; then
+        echo "failed to install extension."
+        exit 1
+      fi
+
+      helm ls -A
+      helm get values ${{ parameters.azureClusterName }} -n kube-system
+      kubectl get pods -n kube-system
+      kubectl get pods -n azure-arc
+    displayName: "install AzureKeyVaultSecretsProvider extension"
+    condition: succeeded()

.pipelines/templates/arc/setup.yaml

@@ -0,0 +1,31 @@
+parameters:
+  version: 0.12.0
+
+steps:
+  - script: |
+      az extension add --name connectedk8s
+      az extension add --name k8s-extension
+      echo "az version:"
+      az version
+    displayName: "add cli extensions"
+    condition: succeeded()
+  - script: |
+      clusterName=sscd-arc-e2e-$(openssl rand -hex 6)
+      echo "##vso[task.setvariable variable=AZURE_CLUSTER_NAME]$clusterName"
+      echo "cluster name is set to - $clusterName"
+    displayName: "set cluster name"
+    condition: succeeded()
+  - bash: |
+      mkdir -p oras/
+      curl -LO https://github.com/deislabs/oras/releases/download/v${{ parameters.version }}/oras_${{ parameters.version }}_linux_amd64.tar.gz
+      tar xvzf oras_${{ parameters.version }}_linux_amd64.tar.gz -C oras/
+    displayName: Install ORAS
+    workingDirectory: $(Pipeline.Workspace)
+    condition: succeeded()
+  - bash: |
+        tree $(Pipeline.Workspace)
+        echo "##vso[task.setvariable variable=PATH]${PATH}:$(Pipeline.Workspace)/oras"
+    displayName: Add oras to PATH
+  - bash: oras version
+    displayName: Print oras version
+    condition: succeeded()

.pipelines/templates/az-login.yaml

@@ -0,0 +1,6 @@
+steps:
+  - script: |
+      az login -i > /dev/null
+      az account set -s=$(SUBSCRIPTION_ID)
+    displayName: "az login"
+    condition: succeeded()

.pipelines/templates/build-images.yaml

@@ -1,21 +1,36 @@
+parameters:
+  - name: registry
+    type: string
+    default: ""
+  - name: ciKindCluster
+    type: boolean
+    default: false
+
 steps:
+  - template: az-login.yaml
   - script: |
-      az login -i > /dev/null
-      az account set -s=$(SUBSCRIPTION_ID)
-    displayName: "az login"
+      if [[ ${{ parameters.ciKindCluster }} == True ]]; then
+        export CI_KIND_CLUSTER=true
+      fi
+
+      if [[ -n "${{ parameters.registry }}" ]]; then
+        export REGISTRY=${{ parameters.registry }}
+        echo "##vso[task.setvariable variable=REGISTRY]${REGISTRY}"
+      fi
 
-  - script: |
       # Generate image version
-      if [[ -n "${CLUSTER_CONFIG:-}" ]]; then
+      if [[ ${{ parameters.ciKindCluster }} == True ]]; then
+        IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-e2e"
+      elif [[ -n "${CLUSTER_CONFIG:-}" ]]; then
         IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-$(CLUSTER_TYPE)-$(OS_TYPE)"
       else
         IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-$(CLUSTER_TYPE)-load"
       fi
       echo "Image version: ${IMAGE_VERSION}"
-
       export IMAGE_VERSION="${IMAGE_VERSION}"
       echo "##vso[task.setvariable variable=IMAGE_VERSION]${IMAGE_VERSION}"
-
+      
       az acr login -n $(REGISTRY_NAME)
       make e2e-bootstrap
     displayName: "Build and push azure keyvault provider image"
+    condition: succeeded()

.pipelines/templates/cleanup-images.yaml

@@ -1,17 +1,33 @@
+parameters:
+  - name: imageVersion
+    type: string
+  - name: registryRepo
+    type: string
+  - name: subscriptionId
+    type: string
+  - name: registryName
+    type: string
+  - name: isMultiArch
+    type: boolean
+    default: true
+
 steps:
   - script: |
       # an empty tag will result in deleting the whole repo.
-      if [[ -n "${IMAGE_VERSION:-}" ]]; then
+      if [[ -n "${{ parameters.imageVersion }}" ]]; then
         # Allow errors in case the images do not exist
         set +e
-        az account set -s=$(SUBSCRIPTION_ID)
-        az acr login -n $(REGISTRY_NAME)
+        az account set -s=${{ parameters.subscriptionId }}
+        az acr login -n ${{ parameters.registryName }}
 
-        for suffix in linux-amd64 linux-arm64 windows-1809-amd64 windows-1903-amd64 windows-1909-amd64 windows-2004-amd64; do
-          az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-$suffix -y || true
-        done
+        if [[ ${{ parameters.isMultiArch }} == True ]]; then
+          for suffix in linux-amd64 linux-arm64 windows-1809-amd64 windows-1903-amd64 windows-1909-amd64 windows-2004-amd64 windows-ltsc2022-amd64; do
+            az acr repository delete --name ${{ parameters.registryName }} --image ${{ parameters.registryRepo }}:${{ parameters.imageVersion }}-$suffix -y || true
+          done
+        fi
 
-        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION} -y || true
+        echo "deleting image: ${{ parameters.registryRepo }}:${{ parameters.imageVersion }}"
+        az acr repository delete --name ${{ parameters.registryName }} --image ${{ parameters.registryRepo }}:${{ parameters.imageVersion }} -y || true
       fi
     condition: always()
     displayName: "Cleanup"

.pipelines/templates/e2e-test-azure.yaml

@@ -72,6 +72,7 @@ jobs:
             isBackwardCompatibilityTest: true
             testReleasedVersion: true
             testWithGPU: ${{ parameters.testWithGPU }}
+            osType: $(OS_TYPE)
 
         # Run e2e tests with New Version
         - template: e2e-test.yaml
@@ -81,6 +82,7 @@ jobs:
             isBackwardCompatibilityTest: true
             setRegistry: true
             testWithGPU: ${{ parameters.testWithGPU }}
+            osType: $(OS_TYPE)
 
         - ${{ if and(parameters.testClusterUpgrade, eq(clusterType, 'aks')) }}:
           - template: aks-upgrade.yaml
@@ -92,6 +94,7 @@ jobs:
               testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
               setRegistry: true
               testWithGPU: ${{ parameters.testWithGPU }}
+              osType: $(OS_TYPE)
 
           - template: aks-upgrade.yaml
 
@@ -101,7 +104,13 @@ jobs:
               testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
               setRegistry: true
               testWithGPU: ${{ parameters.testWithGPU }}
+              osType: $(OS_TYPE)
 
         - template: cleanup-role-assignments.yaml
         - template: teardown.yaml
         - template: cleanup-images.yaml
+          parameters:
+            imageVersion: $(IMAGE_VERSION)
+            registryRepo: $(CI_PROVIDER_REPOSITORY)
+            subscriptionId: $(SUBSCRIPTION_ID)
+            registryName: $(REGISTRY_NAME)