Skip to content

Commit

Permalink
Merge pull request #624 from RafalKorepta/rk/gh-cloudv2-7259/connecto…
Browse files Browse the repository at this point in the history
…rs-test

Connectors chart fixes and e2e test
  • Loading branch information
Rafal Korepta authored Jul 28, 2023
2 parents e31a4e6 + 03c83aa commit 0a0a2dc
Show file tree
Hide file tree
Showing 9 changed files with 373 additions and 14 deletions.
22 changes: 22 additions & 0 deletions .github/ct-connectors.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
debug: true
remote: origin
target-branch: main
helm-extra-args: --timeout 900s
chart-repos:
- redpanda=https://charts.redpanda.com
charts:
- charts/connectors
149 changes: 149 additions & 0 deletions .github/workflows/pull_requests_connectors.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Lint and Test Connectors Chart

on:
pull_request:
branches:
- "**"
paths:
- .github/*.sh
- .github/ct.yaml
- .github/kind.yaml
- .github/workflows/pull_requests_connectors.yaml
- 'charts/connectors/**'
- '!**/*.md'
jobs:
lint:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- run: |
git checkout main
git checkout -
- name: Run CI file name checker
run: .github/check-ci-files.sh charts/connectors/ci

- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.12.2

- name: Set up chart-testing
uses: joejulian/chart-testing-action@v2.3.1-1
with:
version: v3.8.0-1

- name: Run chart-testing (lint)
run: ct lint --config .github/ct-connectors.yaml --github-groups

check-values:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- run: |
git checkout main
git checkout -
- name: install dyff
run: curl -Ls https://github.com/homeport/dyff/releases/download/v1.5.6/dyff_1.5.6_linux_amd64.tar.gz | tar xzv dyff
- name: compare connectors values with main
run: ./dyff --color=off -k between -s <(git show 'origin/main:charts/connectors/values.yaml') charts/connectors/values.yaml

test:
needs: lint
name: Run ct tests for connectors chart
strategy:
fail-fast: true
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- run: |
git checkout main
git checkout -
- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.11.1

- name: Set up chart-testing
uses: joejulian/chart-testing-action@v2.3.1-1
with:
version: v3.8.0-1

- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --config .github/ct-connectors.yaml)
echo $changed
if [[ -n "$changed" ]]; then
echo changed="true" >> "$GITHUB_OUTPUT"
fi
- name: Create kind cluster
uses: helm/kind-action@v1.5.0
if: steps.list-changed.outputs.changed == 'true'
with:
config: .github/kind.yaml

- name: Check kind config worked
if: steps.list-changed.outputs.changed == 'true'
run: kubectl get nodes

- name: install cert-manager
if: steps.list-changed.outputs.changed == 'true'
run: |
helm repo add jetstack https://charts.jetstack.io &&
helm install cert-manager --namespace cert-manager \
--create-namespace --version v1.11.0 jetstack/cert-manager \
--set installCRDs=true --wait --wait-for-jobs
- name: install Redpanda
if: steps.list-changed.outputs.changed == 'true'
run: |
helm dependency build charts/redpanda
helm install --namespace redpanda --create-namespace redpanda charts/redpanda --wait --wait-for-jobs
- name: Copy Redpanda tls cert to connectors chart
if: steps.list-changed.outputs.changed == 'true'
run: |
kubectl -n redpanda wait --for=condition=Ready --timeout=10m certificates/redpanda-default-cert
mkdir -p charts/connectors/templates/hidden-only-for-ci
kubectl -n redpanda get secret -o yaml redpanda-default-cert | \
sed -e '/namespace/d' | \
sed -e '/resourceVersion/d' | \
sed -e '/uid/d' > charts/connectors/templates/hidden-only-for-ci/redpanda-tls.yaml
- name: Run chart-testing (install and upgrade)
if: steps.list-changed.outputs.changed == 'true'
run: |
ct install \
--github-groups \
--upgrade \
--config .github/ct-connectors.yaml \
--skip-missing-values \
--target-branch ${{ github.event.repository.default_branch }}
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ charts/redpanda/templates/external-service.yaml
charts/redpanda/templates/external-tls-secret.yaml
charts/redpanda/templates/some-users-updated.yaml
charts/kminion/templates/redpanda-tls.yaml
charts/connectors/templates/hidden-only-for-ci/

ct_previous_*
6 changes: 4 additions & 2 deletions charts/connectors/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type: application
# The chart version and the app version are not the same and will not track
# together. The chart version is a semver representation of changes to this
# chart.
version: 0.1.1
version: 0.1.2

# The app version is the default version of Redpanda Connectors to install.
appVersion: v1.0.2
Expand All @@ -43,5 +43,7 @@ annotations:
- name: "Helm (>= 3.6.0)"
url: https://helm.sh/docs/intro/install/
artifacthub.io/images: |
- name: redpanda
- name: connectors
image: docker.redpanda.com/redpandadata/connectors:v1.0.2
- name: rpk
image: docker.redpanda.com/redpandadata/redpanda:latest
21 changes: 21 additions & 0 deletions charts/connectors/ci/01-default-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
connectors:
bootstrapServers: "redpanda-0.redpanda.redpanda.svc.cluster.local.:9093,redpanda-1.redpanda.redpanda.svc.cluster.local.:9093,redpanda-2.redpanda.redpanda.svc.cluster.local.:9093"
brokerTLS:
enabled: true
ca:
secretRef: redpanda-default-cert
66 changes: 57 additions & 9 deletions charts/connectors/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ template "connectors.fullname" . }}
name: {{ include "connectors.fullname" . }}
labels:
{{- with include "full.labels" . }}
{{- . | nindent 4 }}
Expand All @@ -31,11 +31,7 @@ spec:
revisionHistoryLimit: {{ .Values.deployment.revisionHistoryLimit }}
selector:
matchLabels: {{ (include "connectors-pod-labels" .) | nindent 6 }}
strategy:
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
type: RollingUpdate
strategy: {{- toYaml .Values.deployment.strategy | nindent 4 }}
template:
metadata:
{{- with $.Values.deployment.annotations }}
Expand Down Expand Up @@ -74,10 +70,10 @@ spec:
{{- toYaml .Values.deployment.podAntiAffinity.custom | nindent 10 }}
{{- end }}
{{- end }}
serviceAccountName: {{ template "connectors.fullname" . }}
serviceAccountName: {{ include "connectors.serviceAccountName" . }}
containers:
- name: connectors-cluster
image: {{ .Values.image.repository }}:{{ template "connectors.tag" . }}
image: {{ .Values.image.repository }}:{{ include "connectors.tag" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- with .Values.container.securityContext }}
Expand All @@ -86,6 +82,8 @@ spec:
env:
- name: CONNECT_CONFIGURATION
value: |
rest.advertised.port={{ .Values.connectors.restPort }}
rest.port={{ .Values.connectors.restPort }}
key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
value.converter=org.apache.kafka.connect.converters.ByteArrayConverter
group.id={{ .Values.connectors.groupID }}
Expand Down Expand Up @@ -132,6 +130,20 @@ spec:
- name: CONNECT_SASL_PASSWORD_FILE
value: rc-credentials/password
{{- end }}
- name: CONNECT_TLS_ENABLED
value: {{ .Values.connectors.brokerTLS.enabled | quote }}
{{- if .Values.connectors.brokerTLS.ca.secretRef }}
- name: CONNECT_TRUSTED_CERTS
value: {{ printf "ca/%s" (default "ca.crt" .Values.connectors.brokerTLS.ca.secretNameOverwrite) }}
{{- end }}
{{- if .Values.connectors.brokerTLS.cert.secretRef }}
- name: CONNECT_TLS_AUTH_CERT
value: {{ printf "cert/%s" (default "tls.crt" .Values.connectors.brokerTLS.cert.secretNameOverwrite) }}
{{- end }}
{{- if .Values.connectors.brokerTLS.key.secretRef }}
- name: CONNECT_TLS_AUTH_KEY
value: {{ printf "key/%s" (default "tls.key" .Values.connectors.brokerTLS.key.secretNameOverwrite) }}
{{- end }}
livenessProbe:
httpGet:
path: /
Expand All @@ -153,6 +165,9 @@ spec:
successThreshold: {{ .Values.deployment.readinessProbe.successThreshold }}
timeoutSeconds: {{ .Values.deployment.readinessProbe.timeoutSeconds }}
ports:
- containerPort: {{ .Values.connectors.restPort }}
name: rest-api
protocol: TCP
{{- range $port := .Values.service.ports }}
- containerPort: {{ $port.port }}
name: {{ $port.name }}
Expand All @@ -168,6 +183,21 @@ spec:
- mountPath: /opt/kafka/connect-password/rc-credentials
name: rc-credentials
{{- end }}
{{- if .Values.connectors.brokerTLS.ca.secretRef }}
- name: truststore
# The /opt/kafka/connect-certs is fixed path within Connectors
mountPath: /opt/kafka/connect-certs/ca
{{- end }}
{{- if .Values.connectors.brokerTLS.cert.secretRef }}
- name: cert
# The /opt/kafka/connect-certs is fixed path within Connectors
mountPath: /opt/kafka/connect-certs/cert
{{- end }}
{{- if .Values.connectors.brokerTLS.key.secretRef }}
- name: key
# The /opt/kafka/connect-certs is fixed path within Connectors
mountPath: /opt/kafka/connect-certs/key
{{- end }}
{{- toYaml .Values.storage.volumeMounts | nindent 12 }}
dnsPolicy: ClusterFirst
restartPolicy: {{ .Values.deployment.restartPolicy }}
Expand Down Expand Up @@ -195,10 +225,28 @@ spec:
whenUnsatisfiable: {{ $spread.whenUnsatisfiable }}
{{- end }}
volumes:
{{- if .Values.connectors.brokerTLS.ca.secretRef }}
- name: truststore
secret:
defaultMode: 0o444
secretName: {{ .Values.connectors.brokerTLS.ca.secretRef }}
{{- end }}
{{- if .Values.connectors.brokerTLS.cert.secretRef }}
- name: truststore
secret:
defaultMode: 0o444
secretName: {{ .Values.connectors.brokerTLS.cert.secretRef }}
{{- end }}
{{- if .Values.connectors.brokerTLS.key.secretRef }}
- name: truststore
secret:
defaultMode: 0o444
secretName: {{ .Values.connectors.brokerTLS.key.secretRef }}
{{- end }}
{{- if and .Values.auth.sasl.userName .Values.auth.sasl.mechanism .Values.auth.sasl.secretRef }}
- name: rc-credentials
secret:
defaultMode: 0444
defaultMode: 0o444
secretName: {{ .Values.auth.sasl.secretRef }}
{{- end }}
{{- toYaml .Values.storage.volume | nindent 8 }}
6 changes: 5 additions & 1 deletion charts/connectors/templates/service.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
apiVersion: v1
kind: Service
metadata:
name: {{ template "connectors.serviceName" . }}
name: {{ include "connectors.serviceName" . }}
labels:
{{- with include "full.labels" . }}
{{- . | nindent 4 }}
Expand All @@ -30,6 +30,10 @@ spec:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- name: rest-api
port: {{ .Values.connectors.restPort }}
targetPort: {{ .Values.connectors.restPort }}
protocol: TCP
{{- range $port := .Values.service.ports }}
- name: {{ $port.name }}
port: {{ $port.port }}
Expand Down
Loading

0 comments on commit 0a0a2dc

Please sign in to comment.