-
Notifications
You must be signed in to change notification settings - Fork 191
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implements outbound proxy support for arc extension (#695)
* feat: implements outbound proxy support for arc extension Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com>
- Loading branch information
Showing
18 changed files
with
351 additions
and
118 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
steps: | ||
- script: | | ||
az group create -n ${AZURE_CLUSTER_NAME} -l $(AZURE_CANARY_LOCATION) | ||
az connectedk8s connect -n ${AZURE_CLUSTER_NAME} -g ${AZURE_CLUSTER_NAME} --no-wait | ||
# It takes time for Arc pods to come up. Sometimes, in such cases helm might report unable to install helm release, but in fact Arc operators get installed and can connect to the cluster. Also, az connectedk8s connect will go through different phases (Connecting, Connected etc.) of installation. So to address both, we are checking the status later without waiting. | ||
echo "verifying cluster connectivity..." | ||
for i in $(seq 1 25); do | ||
provisioningState=$(az connectedk8s list --resource-group ${AZURE_CLUSTER_NAME} --query "[*].provisioningState" -otsv) | ||
connectivityStatus=$(az connectedk8s list --resource-group ${AZURE_CLUSTER_NAME} --query "[*].connectivityStatus" -otsv) | ||
if [ "$provisioningState" == "Succeeded" ] && [ "$connectivityStatus" == "Connected" ]; then | ||
echo "KinD cluster is 'Connected'" | ||
break | ||
else | ||
echo "Provisioning state - $provisioningState, Connectivity status - $connectivityStatus" | ||
sleep 1 | ||
fi | ||
done | ||
if [ "$connectivityStatus" != "Connected" ]; then | ||
echo "failed to connect to the cluster." | ||
exit 1 | ||
fi | ||
displayName: "connect KinD cluster" | ||
condition: succeeded() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
jobs: | ||
- job: e2e_arc_test | ||
variables: | ||
- name: AZURE_ENVIRONMENT_FILEPATH | ||
value: /etc/kubernetes/custom_environment.json | ||
- name: VOLUME_NAME | ||
value: cloudenvfile-vol | ||
- group: csi-secrets-store-e2e | ||
steps: | ||
- template: ../build-images.yaml | ||
parameters: | ||
registry: e2e | ||
ciKindCluster: true | ||
- template: setup.yaml | ||
- script: | | ||
# version should always be greater than 0.0.1 as this is the minimum version configured for reconciliation. | ||
patchVersion="$(date +%s)" | ||
version="0.0.$patchVersion" | ||
echo "##vso[task.setvariable variable=EXT_VERSION]$version" | ||
helm dependency update manifest_staging/charts/csi-secrets-store-provider-azure | ||
helm package manifest_staging/charts/csi-secrets-store-provider-azure --version $version | ||
# echo "Authenticating..." | ||
az acr login -n $(STAGING_REGISTRY_NAME) | ||
# echo 'Pushing chart...' | ||
oras push $(STAGING_REGISTRY):$version ./csi-secrets-store-provider-azure-$version.tgz:application/tar+gzip --debug | ||
registry=$(STAGING_REGISTRY) | ||
respository=${registry#*/} | ||
echo "##vso[task.setvariable variable=REGISTRY_REPO]$respository" | ||
displayName: 'Push OCI helm chart to ACR' | ||
condition: succeeded() | ||
- template: cluster-connect.yaml | ||
- template: extension-create.yaml | ||
parameters: | ||
azureClusterName: $(AZURE_CLUSTER_NAME) | ||
extensionVersion: $(EXT_VERSION) | ||
releaseTrain: dev | ||
configurationSettings: "'secrets-store-csi-driver.enableSecretRotation=true' \ | ||
'secrets-store-csi-driver.rotationPollInterval=30s' \ | ||
'linux.image.tag=$(IMAGE_VERSION)' \ | ||
'linux.image.repository=$(REGISTRY)/provider-azure' \ | ||
'secrets-store-csi-driver.syncSecret.enabled=true' \ | ||
'linux.volumes[0].name=$(VOLUME_NAME)' \ | ||
'linux.volumes[0].hostPath.path=$(AZURE_ENVIRONMENT_FILEPATH)' \ | ||
'linux.volumes[0].hostPath.type=File' \ | ||
'linux.volumeMounts[0].name=$(VOLUME_NAME)' \ | ||
'linux.volumeMounts[0].mountPath=$(AZURE_ENVIRONMENT_FILEPATH)'" | ||
- template: ../e2e-test.yaml | ||
parameters: | ||
testName: "arc extension e2e test" | ||
ciKindCluster: true | ||
isArcTest: true | ||
- template: ../teardown.yaml | ||
- template: ../cleanup-images.yaml | ||
parameters: | ||
imageVersion: $(EXT_VERSION) | ||
registryRepo: $(REGISTRY_REPO) | ||
subscriptionId: $(SUBSCRIPTION_ID) | ||
registryName: $(STAGING_REGISTRY_NAME) | ||
isMultiArch: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
jobs: | ||
- job: e2e_arc_kind | ||
variables: | ||
- name: AZURE_ENVIRONMENT_FILEPATH | ||
value: /etc/kubernetes/custom_environment.json | ||
- name: VOLUME_NAME | ||
value: cloudenvfile-vol | ||
- group: csi-secrets-store-e2e-kind | ||
steps: | ||
- template: ../az-login.yaml | ||
- template: setup.yaml | ||
- script: | | ||
make install-helm install-kubectl setup-kind | ||
displayName: "install dependencies and setup kind" | ||
condition: succeeded() | ||
- template: cluster-connect.yaml | ||
- template: extension-create.yaml | ||
parameters: | ||
azureClusterName: $(AZURE_CLUSTER_NAME) | ||
releaseTrain: preview | ||
configurationSettings: "'secrets-store-csi-driver.enableSecretRotation=true' \ | ||
'secrets-store-csi-driver.rotationPollInterval=30s' \ | ||
'secrets-store-csi-driver.syncSecret.enabled=true' \ | ||
'linux.volumes[0].name=$(VOLUME_NAME)' \ | ||
'linux.volumes[0].hostPath.path=$(AZURE_ENVIRONMENT_FILEPATH)' \ | ||
'linux.volumes[0].hostPath.type=File' \ | ||
'linux.volumeMounts[0].name=$(VOLUME_NAME)' \ | ||
'linux.volumeMounts[0].mountPath=$(AZURE_ENVIRONMENT_FILEPATH)'" | ||
- template: ../e2e-test.yaml | ||
parameters: | ||
testName: "arc extension e2e test" | ||
ciKindCluster: true | ||
isArcTest: true | ||
- template: ../teardown.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
parameters: | ||
- name: azureClusterName | ||
type: string | ||
- name: extensionVersion | ||
type: string | ||
default: "" | ||
- name: releaseTrain | ||
type: string | ||
- name: configurationSettings | ||
type: string | ||
|
||
steps: | ||
- script: | | ||
echo "Installing extension..." | ||
echo "version - '${{ parameters.extensionVersion }}'" | ||
if [[ "${{ parameters.extensionVersion }}" != "" ]]; then | ||
EXTRA_ARGS="--version ${{ parameters.extensionVersion }}" | ||
fi | ||
az k8s-extension create \ | ||
--name ${{ parameters.azureClusterName }} \ | ||
--extension-type Microsoft.AzureKeyVaultSecretsProvider \ | ||
--scope cluster \ | ||
--cluster-name ${{ parameters.azureClusterName }} \ | ||
--resource-group ${{ parameters.azureClusterName }} \ | ||
--cluster-type connectedClusters \ | ||
--release-train ${{ parameters.releaseTrain }} \ | ||
--release-namespace kube-system \ | ||
--configuration-settings ${{ parameters.configurationSettings }} \ | ||
$EXTRA_ARGS | ||
# Arc extensions will go through different phases (Pending, Installed etc.) of installation. We want to make sure extension is 'Installed' before running e2e tests. | ||
echo "verifying extension install status..." | ||
for i in $(seq 1 30); do | ||
provisioningState=$(az k8s-extension show -c ${{ parameters.azureClusterName }} -t connectedClusters -n ${{ parameters.azureClusterName }} -g ${{ parameters.azureClusterName }} --query "provisioningState" -otsv) | ||
if [ "$provisioningState" == "Succeeded" ]; then | ||
echo "AzureKeyVaultSecretsProvider extension is 'Installed'" | ||
break | ||
else | ||
echo "Provisioning state: '$provisioningState'" | ||
sleep 2 | ||
fi | ||
done | ||
if [ "$provisioningState" != "Succeeded" ]; then | ||
echo "failed to install extension." | ||
exit 1 | ||
fi | ||
helm ls -A | ||
helm get values ${{ parameters.azureClusterName }} -n kube-system | ||
kubectl get pods -n kube-system | ||
kubectl get pods -n azure-arc | ||
displayName: "install AzureKeyVaultSecretsProvider extension" | ||
condition: succeeded() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
parameters: | ||
version: 0.12.0 | ||
|
||
steps: | ||
- script: | | ||
az extension add --name connectedk8s | ||
az extension add --name k8s-extension | ||
echo "az version:" | ||
az version | ||
displayName: "add cli extensions" | ||
condition: succeeded() | ||
- script: | | ||
clusterName=sscd-arc-e2e-$(openssl rand -hex 6) | ||
echo "##vso[task.setvariable variable=AZURE_CLUSTER_NAME]$clusterName" | ||
echo "cluster name is set to - $clusterName" | ||
displayName: "set cluster name" | ||
condition: succeeded() | ||
- bash: | | ||
mkdir -p oras/ | ||
curl -LO https://github.com/deislabs/oras/releases/download/v${{ parameters.version }}/oras_${{ parameters.version }}_linux_amd64.tar.gz | ||
tar xvzf oras_${{ parameters.version }}_linux_amd64.tar.gz -C oras/ | ||
displayName: Install ORAS | ||
workingDirectory: $(Pipeline.Workspace) | ||
condition: succeeded() | ||
- bash: | | ||
tree $(Pipeline.Workspace) | ||
echo "##vso[task.setvariable variable=PATH]${PATH}:$(Pipeline.Workspace)/oras" | ||
displayName: Add oras to PATH | ||
- bash: oras version | ||
displayName: Print oras version | ||
condition: succeeded() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
steps: | ||
- script: | | ||
az login -i > /dev/null | ||
az account set -s=$(SUBSCRIPTION_ID) | ||
displayName: "az login" | ||
condition: succeeded() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,21 +1,36 @@ | ||
parameters: | ||
- name: registry | ||
type: string | ||
default: "" | ||
- name: ciKindCluster | ||
type: boolean | ||
default: false | ||
|
||
steps: | ||
- template: az-login.yaml | ||
- script: | | ||
az login -i > /dev/null | ||
az account set -s=$(SUBSCRIPTION_ID) | ||
displayName: "az login" | ||
if [[ ${{ parameters.ciKindCluster }} == True ]]; then | ||
export CI_KIND_CLUSTER=true | ||
fi | ||
if [[ -n "${{ parameters.registry }}" ]]; then | ||
export REGISTRY=${{ parameters.registry }} | ||
echo "##vso[task.setvariable variable=REGISTRY]${REGISTRY}" | ||
fi | ||
- script: | | ||
# Generate image version | ||
if [[ -n "${CLUSTER_CONFIG:-}" ]]; then | ||
if [[ ${{ parameters.ciKindCluster }} == True ]]; then | ||
IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-e2e" | ||
elif [[ -n "${CLUSTER_CONFIG:-}" ]]; then | ||
IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-$(CLUSTER_TYPE)-$(OS_TYPE)" | ||
else | ||
IMAGE_VERSION="$(git describe --tags $(git rev-list --tags --max-count=1))-$(git rev-parse --short HEAD)-$(CLUSTER_TYPE)-load" | ||
fi | ||
echo "Image version: ${IMAGE_VERSION}" | ||
export IMAGE_VERSION="${IMAGE_VERSION}" | ||
echo "##vso[task.setvariable variable=IMAGE_VERSION]${IMAGE_VERSION}" | ||
az acr login -n $(REGISTRY_NAME) | ||
make e2e-bootstrap | ||
displayName: "Build and push azure keyvault provider image" | ||
condition: succeeded() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,33 @@ | ||
parameters: | ||
- name: imageVersion | ||
type: string | ||
- name: registryRepo | ||
type: string | ||
- name: subscriptionId | ||
type: string | ||
- name: registryName | ||
type: string | ||
- name: isMultiArch | ||
type: boolean | ||
default: true | ||
|
||
steps: | ||
- script: | | ||
# an empty tag will result in deleting the whole repo. | ||
if [[ -n "${IMAGE_VERSION:-}" ]]; then | ||
if [[ -n "${{ parameters.imageVersion }}" ]]; then | ||
# Allow errors in case the images do not exist | ||
set +e | ||
az account set -s=$(SUBSCRIPTION_ID) | ||
az acr login -n $(REGISTRY_NAME) | ||
az account set -s=${{ parameters.subscriptionId }} | ||
az acr login -n ${{ parameters.registryName }} | ||
for suffix in linux-amd64 linux-arm64 windows-1809-amd64 windows-1903-amd64 windows-1909-amd64 windows-2004-amd64; do | ||
az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-$suffix -y || true | ||
done | ||
if [[ ${{ parameters.isMultiArch }} == True ]]; then | ||
for suffix in linux-amd64 linux-arm64 windows-1809-amd64 windows-1903-amd64 windows-1909-amd64 windows-2004-amd64 windows-ltsc2022-amd64; do | ||
az acr repository delete --name ${{ parameters.registryName }} --image ${{ parameters.registryRepo }}:${{ parameters.imageVersion }}-$suffix -y || true | ||
done | ||
fi | ||
az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION} -y || true | ||
echo "deleting image: ${{ parameters.registryRepo }}:${{ parameters.imageVersion }}" | ||
az acr repository delete --name ${{ parameters.registryName }} --image ${{ parameters.registryRepo }}:${{ parameters.imageVersion }} -y || true | ||
fi | ||
condition: always() | ||
displayName: "Cleanup" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.