refactor k8s resources to use kustomize.

sillsdev · Nov 8, 2023 · 05c4242 · 05c4242
1 parent be66019
commit 05c4242
Show file tree

Hide file tree

Showing 19 changed files with 182 additions and 98 deletions.
diff --git a/docker/deployment/Makefile b/docker/deployment/Makefile
@@ -44,34 +44,11 @@ init-secrets:
 create-new-deployment-mail:
 	kubectl create deployment mail --image=juanluisbaptiste/postfix:1.0.0 --dry-run=client -o yaml > mail-deployment-new.yaml
 
-deploy-staging: deploy-db deploy-mail-staging deploy-app-staging deploy-lfmerge-staging deploy-next-proxy-staging deploy-next-app-staging
-deploy-mail-staging:
-	sed -e s/{{SERVER_HOSTNAME}}/staging.languageforge.org/ mail-deployment.yaml | kubectl apply -f -
-deploy-app-staging:
-	sed -e s/{{WEBSITE}}/staging.languageforge.org/ app-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
-deploy-lfmerge-staging:
-	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
-deploy-next-proxy-staging:
-	sed -e s/{{WEBSITE}}/staging.languageforge.org/ next-proxy-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
-deploy-next-app-staging:
-	sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f -
-deploy-prod: deploy-db deploy-mail-prod deploy-app-prod deploy-lfmerge-prod deploy-next-proxy-prod deploy-next-app-prod
-deploy-mail-prod:
-	sed -e s/{{SERVER_HOSTNAME}}/languageforge.org/ mail-deployment.yaml | kubectl apply -f -
-deploy-app-prod:
-	sed -e s/{{WEBSITE}}/languageforge.org/ app-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f -
-deploy-lfmerge-prod:
-	sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f -
-deploy-next-proxy-prod:
-	sed -e s/{{WEBSITE}}/languageforge.org/ next-proxy-deployment.yaml \
-  | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f -
-deploy-next-app-prod:
-	sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f -
-deploy-db:
-	kubectl apply -f db-deployment.yaml
+deploy-staging:
+	kubectl --context dallas-rke apply -k staging/
+deploy-prod:
+	kubectl --context aws-rke apply -k prod/
+
 
 delete: delete-app delete-lfmerge delete-mail delete-db delete-next-proxy delete-next-app
 delete-db: # does NOT delete the volume, i.e., the data in the database

diff --git a/docker/deployment/base/app-config.yaml b/docker/deployment/base/app-config.yaml
@@ -4,4 +4,4 @@ metadata:
   name: app-config
 data:
   website: "localhost"
-
+  hg-domain: "hg-staging.languagforge.org"
diff --git a/docker/deployment/base/app-deployment.yaml b/docker/deployment/base/app-deployment.yaml
@@ -18,35 +18,6 @@ spec:
 
 ---
 
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lf-project-assets
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lfmerge-sendreceive-data
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 50Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
 
 # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
 apiVersion: apps/v1
@@ -93,7 +64,7 @@ spec:
             name: sendreceive-data
       containers:
       - name: app
-        image: sillsdev/web-languageforge:{{VERSION}}
+        image: sillsdev/web-languageforge:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:
@@ -113,7 +84,10 @@ spec:
           - name: ENVIRONMENT
             value: production
           - name: WEBSITE
-            value: {{WEBSITE}}
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: website
           - name: MAIL_HOST
             value: mail
           - name: LFMERGE_LOGGING_DEST
@@ -143,6 +117,7 @@ spec:
               secretKeyRef:
                 key: LEX_BOX_HOST
                 name: ld-api
+                optional: true
           - name: FACEBOOK_CLIENT_ID
             valueFrom:
               secretKeyRef:

diff --git a/docker/deployment/base/ingress-config.yaml b/docker/deployment/base/ingress-config.yaml
@@ -0,0 +1,9 @@
+# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file
+
+- op: replace
+  path: /spec/rules/0/host
+  value: localhost
+- op: replace
+  path: /spec/tls/0/hosts
+  value:
+    - localhost
diff --git a/docker/deployment/base/kustomization.yaml b/docker/deployment/base/kustomization.yaml
@@ -6,6 +6,8 @@ resources:
   - app-deployment.yaml
   - db-deployment.yaml
   - lfmerge-deployment.yaml
+  - lfmerge-pvcs.yaml
   - mail-deployment.yaml
   - next-app-deployment.yaml
   - next-proxy-deployment.yaml
+  - app-config.yaml
diff --git a/docker/deployment/base/lfmerge-deployment.yaml b/docker/deployment/base/lfmerge-deployment.yaml
@@ -18,36 +18,6 @@ spec:
 
 ---
 
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lf-project-assets
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
-# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lfmerge-sendreceive-data
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 50Gi
-  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
-
----
-
 # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec
 apiVersion: apps/v1
 kind: Deployment
@@ -100,7 +70,7 @@ spec:
             name: sendreceive-data
       containers:
       - name: lfmerge
-        image: ghcr.io/sillsdev/lfmerge:{{VERSION_LFMERGE}}
+        image: ghcr.io/sillsdev/lfmerge:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:
@@ -133,6 +103,11 @@ spec:
             value: sf_
           - name: LFMERGE_VERBOSE_PROGRESS
             value: "true"
+          - name: LFMERGE_LANGUAGE_DEPOT_HG_PUBLIC_HOSTNAME
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: hg-domain
           - name: LANGUAGE_DEPOT_TRUST_TOKEN
             valueFrom:
               secretKeyRef:

diff --git a/docker/deployment/base/lfmerge-pvcs.yaml b/docker/deployment/base/lfmerge-pvcs.yaml
@@ -0,0 +1,28 @@
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lf-project-assets
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
+
+---
+
+# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lfmerge-sendreceive-data
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: weekly-snapshots-retain-4 # provided by LTOps
diff --git a/docker/deployment/base/mail-deployment.yaml b/docker/deployment/base/mail-deployment.yaml
@@ -41,7 +41,10 @@ spec:
             memory: 100Mi
         env:
           - name: SERVER_HOSTNAME
-            value: {{SERVER_HOSTNAME}}
+            valueFrom:
+              configMapKeyRef:
+                name: app-config
+                key: website
           - name: SMTP_SERVER
             valueFrom:
               secretKeyRef:

diff --git a/docker/deployment/base/next-app-deployment.yaml b/docker/deployment/base/next-app-deployment.yaml
@@ -37,7 +37,9 @@ spec:
     spec:
       containers:
       - name: next-app
-        image: sillsdev/web-languageforge:{{VERSION}}
+        # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number.
+        # this is a workaround because kustomize is designed to work with image names
+        image: sillsdev/web-languageforge-next-app:latest
         imagePullPolicy: Always
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         resources:

diff --git a/docker/deployment/base/next-proxy-deployment.yaml b/docker/deployment/base/next-proxy-deployment.yaml
@@ -9,7 +9,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-body-size: 60M
 spec:
   rules:
-  - host: {{WEBSITE}}
+  - host: localhost
     http:
       paths:
       - path: /
@@ -21,7 +21,7 @@ spec:
               number: 80
   tls:
   - hosts:
-    - {{WEBSITE}}
+    - localhost
     secretName: languageforge-tls
 
 ---
@@ -64,7 +64,9 @@ spec:
     spec:
       containers:
       - name: next-proxy
-        image: sillsdev/web-languageforge:{{VERSION}}
+        # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number.
+        # this is a workaround because kustomize is designed to work with image names
+        image: sillsdev/web-languageforge-next-proxy:latest
         # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
         imagePullPolicy: Always
         resources:

diff --git a/docker/deployment/prod/app-config-patch.yaml b/docker/deployment/prod/app-config-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  website: "languageforge.org"
+  hg-domain: "hg-public.languagforge.org"
+
diff --git a/docker/deployment/prod/ingress-config-patch.yaml b/docker/deployment/prod/ingress-config-patch.yaml
@@ -0,0 +1,9 @@
+# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file
+
+- op: replace
+  path: /spec/rules/0/host
+  value: languageforge.org
+- op: replace
+  path: /spec/tls/0/hosts
+  value:
+    - languageforge.org
diff --git a/docker/deployment/prod/kustomization.yaml b/docker/deployment/prod/kustomization.yaml
@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: languageforge
+
+resources:
+  - ../base
+
+# both next app and next proxy require their own images, but the image name web-languageforge was reused for them
+# with a different tag to select the correct image instead of making a new image name
+# because of that we have to do a bit of a workaround to allow setting the image tag for the correct container
+images:
+  - name: sillsdev/web-languageforge
+    newTag: 2023-09-21
+  - name: sillsdev/web-languageforge-next-proxy
+    newName: sillsdev/web-languageforge
+    newTag: next-proxy-2023-09-21
+  - name: sillsdev/web-languageforge-next-app
+    newName: sillsdev/web-languageforge
+    newTag: next-app-2023-09-21
+  - name: ghcr.io/sillsdev/lfmerge
+    newTag: 2.0.135
+
+patches:
+  - path: app-config-patch.yaml
+  - path: lfmerge-pvcs-patch.yaml
+  - path: ingress-config-patch.yaml
+    target:
+      kind: Ingress
+      name: languageforge-app
+      namespace: languageforge
diff --git a/docker/deployment/prod/lfmerge-pvcs-patch.yaml b/docker/deployment/prod/lfmerge-pvcs-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lfmerge-sendreceive-data
+spec:
+  resources:
+    requests:
+      storage: 70Gi
diff --git a/docker/deployment/staging/app-config-patch.yaml b/docker/deployment/staging/app-config-patch.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+data:
+  website: "staging.languageforge.org"
+
diff --git a/docker/deployment/staging/ingress-config-patch.yaml b/docker/deployment/staging/ingress-config-patch.yaml
@@ -0,0 +1,9 @@
+# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file
+
+- op: replace
+  path: /spec/rules/0/host
+  value: staging.languageforge.org
+- op: replace
+  path: /spec/tls/0/hosts
+  value:
+    - staging.languageforge.org
diff --git a/docker/deployment/staging/kustomization.yaml b/docker/deployment/staging/kustomization.yaml
@@ -0,0 +1,30 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: languageforge
+
+resources:
+  - ../base
+
+# both next app and next proxy require their own images, but the image name web-languageforge was reused for them
+# with a different tag to select the correct image instead of making a new image name
+# because of that we have to do a bit of a workaround to allow setting the image tag for the correct container
+images:
+  - name: sillsdev/web-languageforge
+    newTag: develop-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d
+  - name: sillsdev/web-languageforge-next-proxy
+    newName: sillsdev/web-languageforge
+    newTag: develop-next-proxy-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d
+  - name: sillsdev/web-languageforge-next-app
+    newName: sillsdev/web-languageforge
+    newTag: develop-next-app-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d
+  - name: ghcr.io/sillsdev/lfmerge
+    newTag: 2.0.135
+
+patches:
+  - path: app-config-patch.yaml
+  - path: mongo-pvc-patch.yaml
+  - path: ingress-config-patch.yaml
+    target:
+      kind: Ingress
+      name: languageforge-app
+      namespace: languageforge
diff --git a/docker/deployment/staging/mongo-pvc-patch.yaml b/docker/deployment/staging/mongo-pvc-patch.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lf-mongo-data
+spec:
+  storageClassName: xfs-weekly-snapshots-retain-4 # provided by LTOps