From bacbce17fbe74ef7a966b5e5e3db705bd3916766 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Fri, 4 Nov 2022 19:49:12 +0100
Subject: [PATCH 01/15] [kibana] create a secret with es token

---
 kibana/templates/configmap-helm-scripts.yaml  | 135 ++++++++++++++++--
 kibana/templates/deployment.yaml              |  39 +----
 .../{job.yaml => post-delete-job.yaml}        |   2 +-
 kibana/templates/pre-install-job.yaml         |  54 +++++++
 kibana/templates/pre-install-role.yaml        |  20 +++
 kibana/templates/pre-install-rolebinding.yaml |  20 +++
 .../templates/pre-install-serviceaccount.yaml |  12 ++
 7 files changed, 237 insertions(+), 45 deletions(-)
 rename kibana/templates/{job.yaml => post-delete-job.yaml} (96%)
 create mode 100644 kibana/templates/pre-install-job.yaml
 create mode 100644 kibana/templates/pre-install-role.yaml
 create mode 100644 kibana/templates/pre-install-rolebinding.yaml
 create mode 100644 kibana/templates/pre-install-serviceaccount.yaml

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 120094161..5fdad2c6c 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -4,18 +4,131 @@ kind: ConfigMap
 metadata:
   name: {{ template "kibana.fullname" . }}-helm-scripts
   labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
 data:
-  kibana-entrypoint.sh: |
-    #!/bin/bash
-    set -euo pipefail
+  get-token.js: |
+    const https = require('https');
+    const fs = require('fs');
 
-    echo "export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$({{ template "kibana.home_dir" . }}/node/bin/node {{ template "kibana.home_dir" . }}/helm-scripts/parse-token.js {{ template "kibana.home_dir" . }}/config/tokens/{{ template "kibana.fullname" . }}.json)" > $HOME/.elasticsearch-serviceaccounttoken
-    source $HOME/.elasticsearch-serviceaccounttoken
+    // Elasticsearch API
+    //TODO: remove hardcoded host
+    const esHostname = "elasticsearch-master";
+    const esPort = process.env.ELASTICSEARCH_MASTER_SERVICE_PORT;
+    const esPath = '_security/service/elastic/kibana/credential/token/kb-kibana';
+    const esUsername = process.env.ELASTICSEARCH_USERNAME;
+    const esPassword = process.env.ELASTICSEARCH_PASSWORD;
+    const esAuth = esUsername + ':' + esPassword;
+    const esCaFile = process.env.ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES;
+    const esCa = fs.readFileSync(esCaFile);
 
-    # https://github.com/elastic/dockerfiles/blob/a405a4d692031b72cefcb8523bd464aa3221ec47/kibana/Dockerfile#L131
-    exec /bin/tini -- /usr/local/bin/kibana-docker "$@"
+    // Kubernetes API
+    const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
+    const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
+    const k8sPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
+    const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
+    const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
 
-  parse-token.js: |
-    let dataFile = process.argv[2];
-    let dataContent = require(dataFile.toString());
-    console.log(dataContent.token.value);
+    // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
+    function requestPromise(options, payload) {
+      return new Promise((resolve, reject) => {
+        const request = https.request(options, response => {
+
+          console.log('statusCode:', response.statusCode);
+          console.log('headers:', response.headers);
+
+          // TODO: remove 404 and handle it during esToken deletion
+          const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
+
+          let data = '';
+          response.on('data', chunk => data += chunk); // accumulate data
+          response.once('end', () => isSuccess ? resolve(data) : reject(data));  // resolve promise here
+        });
+
+        request.once('error', err => {
+          // This won't log anything for e.g. an HTTP 404 or 500 response,
+          // since from HTTP's point-of-view we successfully received a
+          // response.
+          console.log(`${options.method} ${options.path} failed: `, err.message || err);
+          reject(err);  // if promise is not already resolved, then we can reject it here
+        });
+
+        if (payload) {
+          request.write(payload);
+        }
+        request.end();
+      });
+    }
+
+    // delete kb-kibana token
+    const esTokenDeleteOptions = {
+      hostname: esHostname,
+      port: esPort,
+      path: esPath,
+      method: 'DELETE',
+      auth: esAuth,
+      ca: esCa,
+    };
+
+    // create new kb-kibana token
+    const esTokenCreateOptions = {
+      hostname: esHostname,
+      port: esPort,
+      path: esPath,
+      method: 'POST',
+      auth: esAuth,
+      ca: esCa,
+    };
+
+    // create new k8s secret
+    const secretCreateOptions = {
+      hostname: k8sHostname,
+      port: k8sPort,
+      path: k8sPath,
+      method: 'POST',
+      ca: k8sCa,
+      headers: {
+        'Authorization': 'Bearer' + k8sBearer,
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      }
+    };
+
+    // Chaining requests
+    requestPromise(esTokenDeleteOptions).then(() =>
+      requestPromise(esTokenCreateOptions).then(response => {
+        const body = JSON.parse(response);
+        const token = body.token.value
+        console.log('Token:', token);
+
+        // encode the token in base 64
+        const base64Token = Buffer.from(token, 'utf8').toString('base64')
+        console.log('Base64 Token:', base64Token);
+
+        // prepare the k8s secret
+        secretData = {
+          "apiVersion":"v1",
+          "kind" :"Secret",
+          "metadata" :{
+            "namespace" :"{{ .Release.Namespace }}",
+            "name":"{{ template "kibana.fullname" . }}-token"
+          },
+          "type": "Opaque",
+          "data": {
+            "token": base64Token,
+          }
+        }
+
+        // create the k8s secret
+        requestPromise(secretCreateOptions, secretData.toString())
+
+        return;
+      })
+    ).catch(err => {
+      console.error(err);
+    });
diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml
index 193d6dc41..c76737a57 100644
--- a/kibana/templates/deployment.yaml
+++ b/kibana/templates/deployment.yaml
@@ -49,10 +49,6 @@ spec:
       volumes:
         - name: kibana-tokens
           emptyDir: {}
-        - name: kibana-helm-scripts
-          configMap:
-            name: {{ template "kibana.fullname" . }}-helm-scripts
-            defaultMode: 0755
         - name: elasticsearch-certs
           secret:
             secretName: {{ .Values.elasticsearchCertificateSecret }}
@@ -86,31 +82,6 @@ spec:
 {{ toYaml .Values.imagePullSecrets | indent 8 }}
     {{- end }}
       initContainers:
-      - name: configure-kibana-token
-        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
-        imagePullPolicy: "{{ .Values.imagePullPolicy }}"
-        # TODO add retries
-        command:
-          - sh
-          - -c
-          - curl --output {{ template "kibana.home_dir" . }}/config/tokens/{{ template "kibana.fullname" . }}.json --fail -XPOST --cacert {{ template "kibana.home_dir" . }}/config/certs/{{ .Values.elasticsearchCertificateAuthoritiesFile }} -u "$(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD)" "{{ .Values.elasticsearchHosts }}/_security/service/elastic/kibana/credential/token/{{ template "kibana.fullname" . }}?pretty"
-        env:
-          - name: "ELASTICSEARCH_USERNAME"
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.elasticsearchCredentialSecret }}
-                key: username
-          - name: "ELASTICSEARCH_PASSWORD"
-            valueFrom:
-              secretKeyRef:
-                name: {{ .Values.elasticsearchCredentialSecret }}
-                key: password
-        volumeMounts:
-          - name: elasticsearch-certs
-            mountPath: {{ template "kibana.home_dir" . }}/config/certs
-            readOnly: true
-          - name: kibana-tokens
-            mountPath: {{ template "kibana.home_dir" . }}/config/tokens
       {{- if .Values.extraInitContainers }}
       # Currently some extra blocks accept strings
       # to continue with backwards compatibility this is being kept
@@ -127,8 +98,6 @@ spec:
 {{ toYaml .Values.securityContext | indent 10 }}
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: "{{ .Values.imagePullPolicy }}"
-        command: ["/bin/bash"]
-        args: ["-c","{{ template "kibana.home_dir" . }}/helm-scripts/kibana-entrypoint.sh"]
         env:
           {{- if .Values.elasticsearchURL }}
           - name: ELASTICSEARCH_URL
@@ -141,6 +110,12 @@ spec:
             value: "{{ template "kibana.home_dir" . }}/config/certs/{{ .Values.elasticsearchCertificateAuthoritiesFile }}"
           - name: SERVER_HOST
             value: "{{ .Values.serverHost }}"
+          - name: ELASTICSEARCH_SERVICEACCOUNTTOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ template "kibana.fullname" . }}-es-token
+                key: token
+                optional: false
 {{- if .Values.extraEnvs }}
 {{ toYaml .Values.extraEnvs | indent 10 }}
 {{- end }}
@@ -194,8 +169,6 @@ spec:
           - name: kibana-tokens
             mountPath: {{ template "kibana.home_dir" . }}/config/tokens
             readOnly: true
-          - name: kibana-helm-scripts
-            mountPath: {{ template "kibana.home_dir" . }}/helm-scripts
           {{- range .Values.secretMounts }}
           - name: {{ .name }}
             mountPath: {{ .path }}
diff --git a/kibana/templates/job.yaml b/kibana/templates/post-delete-job.yaml
similarity index 96%
rename from kibana/templates/job.yaml
rename to kibana/templates/post-delete-job.yaml
index 4bf011f1e..ad3427402 100644
--- a/kibana/templates/job.yaml
+++ b/kibana/templates/post-delete-job.yaml
@@ -1,7 +1,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ template "kibana.fullname" . }}-post-delete
+  name: post-delete-{{ template "kibana.fullname" . }}
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": post-delete
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
new file mode 100644
index 000000000..99261790b
--- /dev/null
+++ b/kibana/templates/pre-install-job.yaml
@@ -0,0 +1,54 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pre-install-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: create-kibana-token
+          image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+          imagePullPolicy: "{{ .Values.imagePullPolicy }}"
+          command: ["{{ template "kibana.home_dir" . }}/node/bin/node"]
+          args:
+            - {{ template "kibana.home_dir" . }}/helm-scripts/get-token.js
+          #command: ["sleep"]
+          #args:
+          #  - "600"
+          env:
+            - name: "ELASTICSEARCH_USERNAME"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.elasticsearchCredentialSecret }}
+                  key: username
+            - name: "ELASTICSEARCH_PASSWORD"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.elasticsearchCredentialSecret }}
+                  key: password
+            - name: ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES
+              value: "{{ template "kibana.home_dir" . }}/config/certs/{{ .Values.elasticsearchCertificateAuthoritiesFile }}"
+          volumeMounts:
+            - name: elasticsearch-certs
+              mountPath: {{ template "kibana.home_dir" . }}/config/certs
+              readOnly: true
+            - name: kibana-helm-scripts
+              mountPath: {{ template "kibana.home_dir" . }}/helm-scripts
+      volumes:
+        - name: elasticsearch-certs
+          secret:
+            secretName: {{ .Values.elasticsearchCertificateSecret }}
+        - name: kibana-helm-scripts
+          configMap:
+            name: {{ template "kibana.fullname" . }}-helm-scripts
+            defaultMode: 0755
diff --git a/kibana/templates/pre-install-role.yaml b/kibana/templates/pre-install-role.yaml
new file mode 100644
index 000000000..4f4e48217
--- /dev/null
+++ b/kibana/templates/pre-install-role.yaml
@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pre-install-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+      - update
diff --git a/kibana/templates/pre-install-rolebinding.yaml b/kibana/templates/pre-install-rolebinding.yaml
new file mode 100644
index 000000000..e516f6b8c
--- /dev/null
+++ b/kibana/templates/pre-install-rolebinding.yaml
@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pre-install-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+subjects:
+  - kind: ServiceAccount
+    name: pre-install-{{ template "kibana.fullname" . }}
+    namespace: {{ .Release.Namespace | quote }}
+roleRef:
+  kind: Role
+  name: pre-install-{{ template "kibana.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
diff --git a/kibana/templates/pre-install-serviceaccount.yaml b/kibana/templates/pre-install-serviceaccount.yaml
new file mode 100644
index 000000000..52908d075
--- /dev/null
+++ b/kibana/templates/pre-install-serviceaccount.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pre-install-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}

From 2b7524bb9f385431182b91f17f2900c6f51e5841 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Mon, 7 Nov 2022 19:13:07 +0100
Subject: [PATCH 02/15] fix pre-post-install jobs

---
 kibana/templates/configmap-helm-scripts.yaml | 150 ++++++++++++++-----
 kibana/templates/post-delete-job.yaml        |  19 ++-
 kibana/templates/pre-install-job.yaml        |   6 +-
 3 files changed, 126 insertions(+), 49 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 5fdad2c6c..b955bb71b 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -17,10 +17,8 @@ data:
     const fs = require('fs');
 
     // Elasticsearch API
-    //TODO: remove hardcoded host
-    const esHostname = "elasticsearch-master";
-    const esPort = process.env.ELASTICSEARCH_MASTER_SERVICE_PORT;
     const esPath = '_security/service/elastic/kibana/credential/token/kb-kibana';
+    const esUrl = '{{ .Values.elasticsearchHosts }}' + '/' + esPath
     const esUsername = process.env.ELASTICSEARCH_USERNAME;
     const esPassword = process.env.ELASTICSEARCH_PASSWORD;
     const esAuth = esUsername + ':' + esPassword;
@@ -31,16 +29,17 @@ data:
     const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
     const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
     const k8sPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
+    const k8sUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPath
     const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
     const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
 
     // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
-    function requestPromise(options, payload) {
+    function requestPromise(url, options, payload) {
       return new Promise((resolve, reject) => {
-        const request = https.request(options, response => {
+        const request = https.request(url, options, response => {
 
           console.log('statusCode:', response.statusCode);
-          console.log('headers:', response.headers);
+          // console.log('headers:', response.headers);
 
           // TODO: remove 404 and handle it during esToken deletion
           const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
@@ -65,70 +64,147 @@ data:
       });
     }
 
-    // delete kb-kibana token
+    // Delete kb-kibana token
     const esTokenDeleteOptions = {
-      hostname: esHostname,
-      port: esPort,
-      path: esPath,
       method: 'DELETE',
       auth: esAuth,
       ca: esCa,
     };
 
-    // create new kb-kibana token
+    // Create new kb-kibana token
     const esTokenCreateOptions = {
-      hostname: esHostname,
-      port: esPort,
-      path: esPath,
       method: 'POST',
       auth: esAuth,
       ca: esCa,
     };
 
-    // create new k8s secret
+    // Create new k8s secret
     const secretCreateOptions = {
-      hostname: k8sHostname,
-      port: k8sPort,
-      path: k8sPath,
       method: 'POST',
       ca: k8sCa,
       headers: {
-        'Authorization': 'Bearer' + k8sBearer,
+        'Authorization': 'Bearer ' + k8sBearer,
         'Accept': 'application/json',
         'Content-Type': 'application/json',
       }
     };
 
     // Chaining requests
-    requestPromise(esTokenDeleteOptions).then(() =>
-      requestPromise(esTokenCreateOptions).then(response => {
+    console.log('Cleaning previous token');
+    requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+      console.log('Creating new token');
+      requestPromise(esUrl, esTokenCreateOptions).then(response => {
         const body = JSON.parse(response);
         const token = body.token.value
-        console.log('Token:', token);
-
-        // encode the token in base 64
-        const base64Token = Buffer.from(token, 'utf8').toString('base64')
-        console.log('Base64 Token:', base64Token);
-
-        // prepare the k8s secret
-        secretData = {
-          "apiVersion":"v1",
-          "kind" :"Secret",
-          "metadata" :{
-            "namespace" :"{{ .Release.Namespace }}",
-            "name":"{{ template "kibana.fullname" . }}-token"
+
+        // Encode the token in base64
+        const base64Token = Buffer.from(token, 'utf8').toString('base64');
+
+        // Prepare the k8s secret
+        secretData = JSON.stringify({
+          "apiVersion": "v1",
+          "kind": "Secret",
+          "metadata": {
+            "namespace": "{{ .Release.Namespace }}",
+            "name": "{{ template "kibana.fullname" . }}-es-token",
           },
           "type": "Opaque",
           "data": {
             "token": base64Token,
           }
+        })
+
+        // Create the k8s secret
+        console.log('Creating K8S secret');
+        requestPromise(k8sUrl, secretCreateOptions, secretData).then().catch(err => {
+          console.error(err)
+        });
+        return;
+      })
+    }).catch(err => {
+      console.error(err);
+    });
+
+  clean-token.js: |
+    const https = require('https');
+    const fs = require('fs');
+
+    // Elasticsearch API
+    const esPath = '_security/service/elastic/kibana/credential/token/kb-kibana';
+    const esUrl = '{{ .Values.elasticsearchHosts }}' + '/' + esPath
+    const esUsername = process.env.ELASTICSEARCH_USERNAME;
+    const esPassword = process.env.ELASTICSEARCH_PASSWORD;
+    const esAuth = esUsername + ':' + esPassword;
+    const esCaFile = process.env.ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES;
+    const esCa = fs.readFileSync(esCaFile);
+
+    // Kubernetes API
+    const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
+    const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
+    const k8sPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets/{{ template "kibana.fullname" . }}-es-token';
+    const k8sUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPath
+    const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
+    const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
+
+    // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
+    function requestPromise(url, options, payload) {
+      return new Promise((resolve, reject) => {
+        const request = https.request(url, options, response => {
+
+          console.log('statusCode:', response.statusCode);
+          // console.log('headers:', response.headers);
+
+          // TODO: remove 404 and handle it during esToken deletion
+          const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
+
+          let data = '';
+          response.on('data', chunk => data += chunk); // accumulate data
+          response.once('end', () => isSuccess ? resolve(data) : reject(data));  // resolve promise here
+        });
+
+        request.once('error', err => {
+          // This won't log anything for e.g. an HTTP 404 or 500 response,
+          // since from HTTP's point-of-view we successfully received a
+          // response.
+          console.log(`${options.method} ${options.path} failed: `, err.message || err);
+          reject(err);  // if promise is not already resolved, then we can reject it here
+        });
+
+        if (payload) {
+          request.write(payload);
         }
+        request.end();
+      });
+    }
+
+    // Delete kb-kibana token
+    const esTokenDeleteOptions = {
+      method: 'DELETE',
+      auth: esAuth,
+      ca: esCa,
+    };
 
-        // create the k8s secret
-        requestPromise(secretCreateOptions, secretData.toString())
+    // Create new k8s secret
+    const secretDeleteOptions = {
+      method: 'DELETE',
+      ca: k8sCa,
+      headers: {
+        'Authorization': 'Bearer ' + k8sBearer,
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      }
+    };
 
+    // Chaining requests
+    console.log('Cleaning token');
+    requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+      // Create the k8s secret
+      console.log('Delete K8S secret');
+      requestPromise(k8sUrl, secretDeleteOptions).then().catch(err => {
+          console.error(err)
+        });
         return;
       })
-    ).catch(err => {
+    }).catch(err => {
       console.error(err);
     });
diff --git a/kibana/templates/post-delete-job.yaml b/kibana/templates/post-delete-job.yaml
index ad3427402..af508c4cd 100644
--- a/kibana/templates/post-delete-job.yaml
+++ b/kibana/templates/post-delete-job.yaml
@@ -5,7 +5,6 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": post-delete
-    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
@@ -20,14 +19,9 @@ spec:
         - name: clean-kibana-token
           image: "{{ .Values.image }}:{{ .Values.imageTag }}"
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
-          command: ["curl"]
+          command: ["{{ template "kibana.home_dir" . }}/node/bin/node"]
           args:
-            - -XDELETE
-            - --cacert
-            - {{ template "kibana.home_dir" . }}/config/certs/{{ .Values.elasticsearchCertificateAuthoritiesFile }}
-            - -u
-            - "$(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD)"
-            - "{{ .Values.elasticsearchHosts }}/_security/service/elastic/kibana/credential/token/{{ template "kibana.fullname" . }}"
+           - {{ template "kibana.home_dir" . }}/helm-scripts/clean-token.js
           env:
             - name: "ELASTICSEARCH_USERNAME"
               valueFrom:
@@ -39,11 +33,20 @@ spec:
                 secretKeyRef:
                   name: {{ .Values.elasticsearchCredentialSecret }}
                   key: password
+            - name: ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES
+              value: "{{ template "kibana.home_dir" . }}/config/certs/{{ .Values.elasticsearchCertificateAuthoritiesFile }}"
           volumeMounts:
             - name: elasticsearch-certs
               mountPath: {{ template "kibana.home_dir" . }}/config/certs
               readOnly: true
+            - name: kibana-helm-scripts
+              mountPath: {{ template "kibana.home_dir" . }}/helm-scripts
+      serviceAccount: pre-install-{{ template "kibana.fullname" . }}
       volumes:
         - name: elasticsearch-certs
           secret:
             secretName: {{ .Values.elasticsearchCertificateSecret }}
+        - name: kibana-helm-scripts
+          configMap:
+            name: {{ template "kibana.fullname" . }}-helm-scripts
+            defaultMode: 0755
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
index 99261790b..68babc110 100644
--- a/kibana/templates/pre-install-job.yaml
+++ b/kibana/templates/pre-install-job.yaml
@@ -21,10 +21,7 @@ spec:
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           command: ["{{ template "kibana.home_dir" . }}/node/bin/node"]
           args:
-            - {{ template "kibana.home_dir" . }}/helm-scripts/get-token.js
-          #command: ["sleep"]
-          #args:
-          #  - "600"
+           - {{ template "kibana.home_dir" . }}/helm-scripts/get-token.js
           env:
             - name: "ELASTICSEARCH_USERNAME"
               valueFrom:
@@ -44,6 +41,7 @@ spec:
               readOnly: true
             - name: kibana-helm-scripts
               mountPath: {{ template "kibana.home_dir" . }}/helm-scripts
+      serviceAccount: pre-install-{{ template "kibana.fullname" . }}
       volumes:
         - name: elasticsearch-certs
           secret:

From 827ddb30e4bd9e6ed6b5f30da5776ea5d2054116 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Tue, 8 Nov 2022 18:42:56 +0100
Subject: [PATCH 03/15] fix clean command + optimize js script

---
 kibana/templates/configmap-helm-scripts.yaml  | 204 +++++++-----------
 kibana/templates/post-delete-job.yaml         |   6 +-
 kibana/templates/post-delete-role.yaml        |  20 ++
 kibana/templates/post-delete-rolebinding.yaml |  21 ++
 .../templates/post-delete-serviceaccount.yaml |  13 ++
 kibana/templates/pre-install-job.yaml         |   3 +
 kibana/templates/pre-install-role.yaml        |   1 +
 kibana/templates/pre-install-rolebinding.yaml |   1 +
 .../templates/pre-install-serviceaccount.yaml |   1 +
 9 files changed, 143 insertions(+), 127 deletions(-)
 create mode 100644 kibana/templates/post-delete-role.yaml
 create mode 100644 kibana/templates/post-delete-rolebinding.yaml
 create mode 100644 kibana/templates/post-delete-serviceaccount.yaml

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index b955bb71b..fc653afda 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -5,14 +5,15 @@ metadata:
   name: {{ template "kibana.fullname" . }}-helm-scripts
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,post-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
     {{- end }}
     {{- end }}
 data:
-  get-token.js: |
+  manage-es-token.js: |
     const https = require('https');
     const fs = require('fs');
 
@@ -28,57 +29,24 @@ data:
     // Kubernetes API
     const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
     const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
-    const k8sPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
-    const k8sUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPath
+    const k8sPostSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
+    const k8sDeleteSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets/{{ template "kibana.fullname" . }}-es-token';
+    const k8sPostSecretUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPostSecretPath;
+    const k8sDeleteSecretUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sDeleteSecretPath;
     const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
     const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
 
-    // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
-    function requestPromise(url, options, payload) {
-      return new Promise((resolve, reject) => {
-        const request = https.request(url, options, response => {
-
-          console.log('statusCode:', response.statusCode);
-          // console.log('headers:', response.headers);
-
-          // TODO: remove 404 and handle it during esToken deletion
-          const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
-
-          let data = '';
-          response.on('data', chunk => data += chunk); // accumulate data
-          response.once('end', () => isSuccess ? resolve(data) : reject(data));  // resolve promise here
-        });
-
-        request.once('error', err => {
-          // This won't log anything for e.g. an HTTP 404 or 500 response,
-          // since from HTTP's point-of-view we successfully received a
-          // response.
-          console.log(`${options.method} ${options.path} failed: `, err.message || err);
-          reject(err);  // if promise is not already resolved, then we can reject it here
-        });
-
-        if (payload) {
-          request.write(payload);
-        }
-        request.end();
-      });
-    }
-
-    // Delete kb-kibana token
+    // Post Data
     const esTokenDeleteOptions = {
       method: 'DELETE',
       auth: esAuth,
       ca: esCa,
     };
-
-    // Create new kb-kibana token
     const esTokenCreateOptions = {
       method: 'POST',
       auth: esAuth,
       ca: esCa,
     };
-
-    // Create new k8s secret
     const secretCreateOptions = {
       method: 'POST',
       ca: k8sCa,
@@ -88,63 +56,15 @@ data:
         'Content-Type': 'application/json',
       }
     };
-
-    // Chaining requests
-    console.log('Cleaning previous token');
-    requestPromise(esUrl, esTokenDeleteOptions).then(() => {
-      console.log('Creating new token');
-      requestPromise(esUrl, esTokenCreateOptions).then(response => {
-        const body = JSON.parse(response);
-        const token = body.token.value
-
-        // Encode the token in base64
-        const base64Token = Buffer.from(token, 'utf8').toString('base64');
-
-        // Prepare the k8s secret
-        secretData = JSON.stringify({
-          "apiVersion": "v1",
-          "kind": "Secret",
-          "metadata": {
-            "namespace": "{{ .Release.Namespace }}",
-            "name": "{{ template "kibana.fullname" . }}-es-token",
-          },
-          "type": "Opaque",
-          "data": {
-            "token": base64Token,
-          }
-        })
-
-        // Create the k8s secret
-        console.log('Creating K8S secret');
-        requestPromise(k8sUrl, secretCreateOptions, secretData).then().catch(err => {
-          console.error(err)
-        });
-        return;
-      })
-    }).catch(err => {
-      console.error(err);
-    });
-
-  clean-token.js: |
-    const https = require('https');
-    const fs = require('fs');
-
-    // Elasticsearch API
-    const esPath = '_security/service/elastic/kibana/credential/token/kb-kibana';
-    const esUrl = '{{ .Values.elasticsearchHosts }}' + '/' + esPath
-    const esUsername = process.env.ELASTICSEARCH_USERNAME;
-    const esPassword = process.env.ELASTICSEARCH_PASSWORD;
-    const esAuth = esUsername + ':' + esPassword;
-    const esCaFile = process.env.ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES;
-    const esCa = fs.readFileSync(esCaFile);
-
-    // Kubernetes API
-    const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
-    const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
-    const k8sPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets/{{ template "kibana.fullname" . }}-es-token';
-    const k8sUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPath
-    const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
-    const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
+    const secretDeleteOptions = {
+      method: 'DELETE',
+      ca: k8sCa,
+      headers: {
+        'Authorization': 'Bearer ' + k8sBearer,
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+      }
+    };
 
     // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
     function requestPromise(url, options, payload) {
@@ -152,7 +72,6 @@ data:
         const request = https.request(url, options, response => {
 
           console.log('statusCode:', response.statusCode);
-          // console.log('headers:', response.headers);
 
           // TODO: remove 404 and handle it during esToken deletion
           const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
@@ -177,34 +96,69 @@ data:
       });
     }
 
-    // Delete kb-kibana token
-    const esTokenDeleteOptions = {
-      method: 'DELETE',
-      auth: esAuth,
-      ca: esCa,
-    };
-
-    // Create new k8s secret
-    const secretDeleteOptions = {
-      method: 'DELETE',
-      ca: k8sCa,
-      headers: {
-        'Authorization': 'Bearer ' + k8sBearer,
-        'Accept': 'application/json',
-        'Content-Type': 'application/json',
-      }
-    };
+    function createEsToken() {
+      // Chaining requests
+      console.log('Cleaning previous token');
+      requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+        console.log('Creating new token');
+        requestPromise(esUrl, esTokenCreateOptions).then(response => {
+          const body = JSON.parse(response);
+          const token = body.token.value
+
+          // Encode the token in base64
+          const base64Token = Buffer.from(token, 'utf8').toString('base64');
+
+          // Prepare the k8s secret
+          secretData = JSON.stringify({
+            "apiVersion": "v1",
+            "kind": "Secret",
+            "metadata": {
+              "namespace": "{{ .Release.Namespace }}",
+              "name": "{{ template "kibana.fullname" . }}-es-token",
+            },
+            "type": "Opaque",
+            "data": {
+              "token": base64Token,
+            }
+          })
+
+          // Create the k8s secret
+          console.log('Creating K8S secret');
+          requestPromise(k8sPostSecretUrl, secretCreateOptions, secretData).then().catch(err => {
+            console.error(err)
+          });
+          return;
+        })
+      }).catch(err => {
+        console.error(err);
+      });
+    }
 
-    // Chaining requests
-    console.log('Cleaning token');
-    requestPromise(esUrl, esTokenDeleteOptions).then(() => {
-      // Create the k8s secret
-      console.log('Delete K8S secret');
-      requestPromise(k8sUrl, secretDeleteOptions).then().catch(err => {
+    function cleanEsToken() {
+      // Chaining requests
+      console.log('Cleaning token');
+      requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+        // Create the k8s secret
+        console.log('Delete K8S secret');
+        requestPromise(k8sDeleteSecretUrl, secretDeleteOptions).then().catch(err => {
           console.error(err)
         });
         return;
-      })
-    }).catch(err => {
-      console.error(err);
-    });
+      }).catch(err => {
+        console.error(err);
+      });
+    }
+
+    const command = process.argv[2];
+    switch (command) {
+      case 'create':
+        console.log('Creating a new Elasticsearch token for Kibana')
+        createEsToken();
+        break;
+      case 'clean':
+        console.log('Cleaning the Kibana Elasticsearch token')
+        cleanEsToken();
+        break;
+      default:
+        console.log('Unknown command');
+    }
diff --git a/kibana/templates/post-delete-job.yaml b/kibana/templates/post-delete-job.yaml
index af508c4cd..c123ccca4 100644
--- a/kibana/templates/post-delete-job.yaml
+++ b/kibana/templates/post-delete-job.yaml
@@ -5,6 +5,7 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
@@ -21,7 +22,8 @@ spec:
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           command: ["{{ template "kibana.home_dir" . }}/node/bin/node"]
           args:
-           - {{ template "kibana.home_dir" . }}/helm-scripts/clean-token.js
+           - {{ template "kibana.home_dir" . }}/helm-scripts/manage-es-token.js
+           - clean
           env:
             - name: "ELASTICSEARCH_USERNAME"
               valueFrom:
@@ -41,7 +43,7 @@ spec:
               readOnly: true
             - name: kibana-helm-scripts
               mountPath: {{ template "kibana.home_dir" . }}/helm-scripts
-      serviceAccount: pre-install-{{ template "kibana.fullname" . }}
+      serviceAccount: post-delete-{{ template "kibana.fullname" . }}
       volumes:
         - name: elasticsearch-certs
           secret:
diff --git a/kibana/templates/post-delete-role.yaml b/kibana/templates/post-delete-role.yaml
new file mode 100644
index 000000000..f3ef8aa76
--- /dev/null
+++ b/kibana/templates/post-delete-role.yaml
@@ -0,0 +1,20 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: post-delete-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - delete
diff --git a/kibana/templates/post-delete-rolebinding.yaml b/kibana/templates/post-delete-rolebinding.yaml
new file mode 100644
index 000000000..fa161e52c
--- /dev/null
+++ b/kibana/templates/post-delete-rolebinding.yaml
@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: post-delete-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+subjects:
+  - kind: ServiceAccount
+    name: post-delete-{{ template "kibana.fullname" . }}
+    namespace: {{ .Release.Namespace | quote }}
+roleRef:
+  kind: Role
+  name: post-delete-{{ template "kibana.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
diff --git a/kibana/templates/post-delete-serviceaccount.yaml b/kibana/templates/post-delete-serviceaccount.yaml
new file mode 100644
index 000000000..fa842641c
--- /dev/null
+++ b/kibana/templates/post-delete-serviceaccount.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: post-delete-{{ template "kibana.fullname" . }}
+  labels: {{ include "kibana.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-delete-policy": hook-succeeded
+    {{- if .Values.annotations }}
+    {{- range $key, $value := .Values.annotations }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
index 68babc110..25bec5208 100644
--- a/kibana/templates/pre-install-job.yaml
+++ b/kibana/templates/pre-install-job.yaml
@@ -5,6 +5,7 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": pre-install
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
@@ -21,6 +22,8 @@ spec:
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           command: ["{{ template "kibana.home_dir" . }}/node/bin/node"]
           args:
+           - {{ template "kibana.home_dir" . }}/helm-scripts/manage-es-token.js
+           - create
            - {{ template "kibana.home_dir" . }}/helm-scripts/get-token.js
           env:
             - name: "ELASTICSEARCH_USERNAME"
diff --git a/kibana/templates/pre-install-role.yaml b/kibana/templates/pre-install-role.yaml
index 4f4e48217..8334d537f 100644
--- a/kibana/templates/pre-install-role.yaml
+++ b/kibana/templates/pre-install-role.yaml
@@ -5,6 +5,7 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": pre-install
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
diff --git a/kibana/templates/pre-install-rolebinding.yaml b/kibana/templates/pre-install-rolebinding.yaml
index e516f6b8c..9810bbbe8 100644
--- a/kibana/templates/pre-install-rolebinding.yaml
+++ b/kibana/templates/pre-install-rolebinding.yaml
@@ -5,6 +5,7 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": pre-install
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}
diff --git a/kibana/templates/pre-install-serviceaccount.yaml b/kibana/templates/pre-install-serviceaccount.yaml
index 52908d075..dc0295532 100644
--- a/kibana/templates/pre-install-serviceaccount.yaml
+++ b/kibana/templates/pre-install-serviceaccount.yaml
@@ -5,6 +5,7 @@ metadata:
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
     "helm.sh/hook": pre-install
+    "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
     {{ $key }}: {{ $value | quote }}

From 753facdaa97dc3c0601666cfaa08e343f8aa9a03 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Tue, 8 Nov 2022 19:28:23 +0100
Subject: [PATCH 04/15] fix upgrade test

---
 kibana/templates/configmap-helm-scripts.yaml     | 2 +-
 kibana/templates/pre-install-job.yaml            | 2 +-
 kibana/templates/pre-install-role.yaml           | 2 +-
 kibana/templates/pre-install-rolebinding.yaml    | 2 +-
 kibana/templates/pre-install-serviceaccount.yaml | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index fc653afda..49a02c3cd 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -5,7 +5,7 @@ metadata:
   name: {{ template "kibana.fullname" . }}-helm-scripts
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install,post-delete
+    "helm.sh/hook": pre-install,pre-upgrade,post-delete
     "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
index 25bec5208..b15b9d456 100644
--- a/kibana/templates/pre-install-job.yaml
+++ b/kibana/templates/pre-install-job.yaml
@@ -4,7 +4,7 @@ metadata:
   name: pre-install-{{ template "kibana.fullname" . }}
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
diff --git a/kibana/templates/pre-install-role.yaml b/kibana/templates/pre-install-role.yaml
index 8334d537f..a3439a235 100644
--- a/kibana/templates/pre-install-role.yaml
+++ b/kibana/templates/pre-install-role.yaml
@@ -4,7 +4,7 @@ metadata:
   name: pre-install-{{ template "kibana.fullname" . }}
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
diff --git a/kibana/templates/pre-install-rolebinding.yaml b/kibana/templates/pre-install-rolebinding.yaml
index 9810bbbe8..a0c0747c3 100644
--- a/kibana/templates/pre-install-rolebinding.yaml
+++ b/kibana/templates/pre-install-rolebinding.yaml
@@ -4,7 +4,7 @@ metadata:
   name: pre-install-{{ template "kibana.fullname" . }}
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}
diff --git a/kibana/templates/pre-install-serviceaccount.yaml b/kibana/templates/pre-install-serviceaccount.yaml
index dc0295532..2548fd4fb 100644
--- a/kibana/templates/pre-install-serviceaccount.yaml
+++ b/kibana/templates/pre-install-serviceaccount.yaml
@@ -4,7 +4,7 @@ metadata:
   name: pre-install-{{ template "kibana.fullname" . }}
   labels: {{ include "kibana.labels" . | nindent 4 }}
   annotations:
-    "helm.sh/hook": pre-install
+    "helm.sh/hook": pre-install,pre-upgrade
     "helm.sh/hook-delete-policy": hook-succeeded
     {{- if .Values.annotations }}
     {{- range $key, $value := .Values.annotations }}

From 547a349df403f6c86c0f6a3219d0ac43bddd570e Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Tue, 8 Nov 2022 22:05:29 +0100
Subject: [PATCH 05/15] fix a few bugs

---
 kibana/examples/upgrade/Makefile             | 1 +
 kibana/templates/configmap-helm-scripts.yaml | 2 +-
 kibana/templates/pre-install-job.yaml        | 1 -
 3 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/kibana/examples/upgrade/Makefile b/kibana/examples/upgrade/Makefile
index 882a3e289..a51caeccd 100644
--- a/kibana/examples/upgrade/Makefile
+++ b/kibana/examples/upgrade/Makefile
@@ -23,6 +23,7 @@ upgrade-es:
 	kubectl delete pod --selector=app=$(ES_CLUSTER)-master
 
 upgrade-kb:
+	sleep 120 # wait that ES pods are started
 	helm upgrade $(RELEASE) ../../ --wait --values values.yaml
 	kubectl rollout status deployment $(RELEASE)-kibana
 
diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 49a02c3cd..14fb48e1d 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -18,7 +18,7 @@ data:
     const fs = require('fs');
 
     // Elasticsearch API
-    const esPath = '_security/service/elastic/kibana/credential/token/kb-kibana';
+    const esPath = '_security/service/elastic/kibana/credential/token/{{ template "kibana.fullname" . }}';
     const esUrl = '{{ .Values.elasticsearchHosts }}' + '/' + esPath
     const esUsername = process.env.ELASTICSEARCH_USERNAME;
     const esPassword = process.env.ELASTICSEARCH_PASSWORD;
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
index b15b9d456..7edf406dd 100644
--- a/kibana/templates/pre-install-job.yaml
+++ b/kibana/templates/pre-install-job.yaml
@@ -24,7 +24,6 @@ spec:
           args:
            - {{ template "kibana.home_dir" . }}/helm-scripts/manage-es-token.js
            - create
-           - {{ template "kibana.home_dir" . }}/helm-scripts/get-token.js
           env:
             - name: "ELASTICSEARCH_USERNAME"
               valueFrom:

From 81ce165618d2b7602bff2e814e9825800c087aa0 Mon Sep 17 00:00:00 2001
From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 13:01:31 +0100
Subject: [PATCH 06/15] Update kibana/templates/configmap-helm-scripts.yaml

Co-authored-by: Rory Hunter <pugnascotia@users.noreply.github.com>
---
 kibana/templates/configmap-helm-scripts.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 14fb48e1d..83e54c8c7 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -31,8 +31,8 @@ data:
     const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
     const k8sPostSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
     const k8sDeleteSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets/{{ template "kibana.fullname" . }}-es-token';
-    const k8sPostSecretUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sPostSecretPath;
-    const k8sDeleteSecretUrl = 'https://' + k8sHostname + ':' + k8sPort + '/' + k8sDeleteSecretPath;
+    const k8sPostSecretUrl = `https://${k8sHostname}:${k8sPort}/${k8sPostSecretPath}`;
+    const k8sDeleteSecretUrl = `https://${k8sHostname}:${k8sPort}/${k8sDeleteSecretPath}`;
     const k8sBearer = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/token');
     const k8sCa = fs.readFileSync('/run/secrets/kubernetes.io/serviceaccount/ca.crt');
 

From ff4a1c60b02b3553120dc97454519a1917e90085 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 15:14:45 +0100
Subject: [PATCH 07/15] check if vars exist

---
 kibana/templates/configmap-helm-scripts.yaml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 83e54c8c7..19b94dd11 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -17,18 +17,27 @@ data:
     const https = require('https');
     const fs = require('fs');
 
+    // Read environment variables
+    function getEnvVar(name) {
+      if (name in process.env) {
+        return process.env[name]
+      } else {
+        throw new Error(name + ' environment variable is missing')
+      }
+    }
+
     // Elasticsearch API
     const esPath = '_security/service/elastic/kibana/credential/token/{{ template "kibana.fullname" . }}';
     const esUrl = '{{ .Values.elasticsearchHosts }}' + '/' + esPath
-    const esUsername = process.env.ELASTICSEARCH_USERNAME;
-    const esPassword = process.env.ELASTICSEARCH_PASSWORD;
+    const esUsername = getEnvVar('ELASTICSEARCH_USERNAME');
+    const esPassword = getEnvVar('ELASTICSEARCH_PASSWORD');
     const esAuth = esUsername + ':' + esPassword;
-    const esCaFile = process.env.ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES;
+    const esCaFile = getEnvVar('ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES');
     const esCa = fs.readFileSync(esCaFile);
 
     // Kubernetes API
-    const k8sHostname = process.env.KUBERNETES_SERVICE_HOST;
-    const k8sPort = process.env.KUBERNETES_SERVICE_PORT_HTTPS;
+    const k8sHostname = getEnvVar('KUBERNETES_SERVICE_HOST');
+    const k8sPort = getEnvVar('KUBERNETES_SERVICE_PORT_HTTPS');
     const k8sPostSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets';
     const k8sDeleteSecretPath = 'api/v1/namespaces/{{ .Release.Namespace }}/secrets/{{ template "kibana.fullname" . }}-es-token';
     const k8sPostSecretUrl = `https://${k8sHostname}:${k8sPort}/${k8sPostSecretPath}`;

From 43c08a13100b29023ec7640094d6623532aa067e Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 16:59:56 +0100
Subject: [PATCH 08/15] handle both payload and extra status code

---
 kibana/templates/configmap-helm-scripts.yaml | 24 ++++++++++++--------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 19b94dd11..075125b53 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -76,14 +76,19 @@ data:
     };
 
     // With thanks to https://stackoverflow.com/questions/57332374/how-to-chain-http-request
-    function requestPromise(url, options, payload) {
+    function requestPromise(url, httpsOptions, extraOptions = {}) {
       return new Promise((resolve, reject) => {
-        const request = https.request(url, options, response => {
+        const request = https.request(url, httpsOptions, response => {
 
           console.log('statusCode:', response.statusCode);
 
-          // TODO: remove 404 and handle it during esToken deletion
-          const isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == 404;
+          let isSuccess = undefined;
+
+          if (typeof(extraOptions.extraStatusCode) != "undefined" && extraOptions.extraStatusCode != null) {
+            isSuccess = response.statusCode >= 200 && response.statusCode < 300 || response.statusCode == extraOptions.extraStatusCode;
+          } else {
+            isSuccess = response.statusCode >= 200 && response.statusCode < 300;
+          }
 
           let data = '';
           response.on('data', chunk => data += chunk); // accumulate data
@@ -94,12 +99,12 @@ data:
           // This won't log anything for e.g. an HTTP 404 or 500 response,
           // since from HTTP's point-of-view we successfully received a
           // response.
-          console.log(`${options.method} ${options.path} failed: `, err.message || err);
+          console.log(`${httpsOptions.method} ${httpsOptions.path} failed: `, err.message || err);
           reject(err);  // if promise is not already resolved, then we can reject it here
         });
 
-        if (payload) {
-          request.write(payload);
+        if (typeof(extraOptions.payload) != "undefined") {
+          request.write(extraOptions.payload);
         }
         request.end();
       });
@@ -108,7 +113,8 @@ data:
     function createEsToken() {
       // Chaining requests
       console.log('Cleaning previous token');
-      requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+      // 404 status code is accepted if there is no previous token to clean
+      return requestPromise(esUrl, esTokenDeleteOptions, {extraStatusCode: 404}).then(() => {
         console.log('Creating new token');
         requestPromise(esUrl, esTokenCreateOptions).then(response => {
           const body = JSON.parse(response);
@@ -133,7 +139,7 @@ data:
 
           // Create the k8s secret
           console.log('Creating K8S secret');
-          requestPromise(k8sPostSecretUrl, secretCreateOptions, secretData).then().catch(err => {
+          requestPromise(k8sPostSecretUrl, secretCreateOptions, {payload: secretData}).then().catch(err => {
             console.error(err)
           });
           return;

From e3ef0c1620c5953b623d232babf63489b67e9bb5 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:16:37 +0100
Subject: [PATCH 09/15] return promises

---
 kibana/templates/configmap-helm-scripts.yaml | 26 ++++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index 075125b53..fbb77a468 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -139,28 +139,18 @@ data:
 
           // Create the k8s secret
           console.log('Creating K8S secret');
-          requestPromise(k8sPostSecretUrl, secretCreateOptions, {payload: secretData}).then().catch(err => {
-            console.error(err)
-          });
-          return;
-        })
-      }).catch(err => {
-        console.error(err);
+          requestPromise(k8sPostSecretUrl, secretCreateOptions, {payload: secretData})
+        });
       });
     }
 
     function cleanEsToken() {
       // Chaining requests
       console.log('Cleaning token');
-      requestPromise(esUrl, esTokenDeleteOptions).then(() => {
+      return requestPromise(esUrl, esTokenDeleteOptions).then(() => {
         // Create the k8s secret
         console.log('Delete K8S secret');
-        requestPromise(k8sDeleteSecretUrl, secretDeleteOptions).then().catch(err => {
-          console.error(err)
-        });
-        return;
-      }).catch(err => {
-        console.error(err);
+        requestPromise(k8sDeleteSecretUrl, secretDeleteOptions)
       });
     }
 
@@ -168,11 +158,15 @@ data:
     switch (command) {
       case 'create':
         console.log('Creating a new Elasticsearch token for Kibana')
-        createEsToken();
+        createEsToken().catch(err => {
+          console.error(err);
+        });
         break;
       case 'clean':
         console.log('Cleaning the Kibana Elasticsearch token')
-        cleanEsToken();
+        cleanEsToken().catch(err => {
+          console.error(err);
+        });
         break;
       default:
         console.log('Unknown command');

From 5b812e5b5374d2ed1220a010afeeea59b6f4f16d Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:37:47 +0100
Subject: [PATCH 10/15] add a note to retrieve the kibana token

---
 kibana/templates/NOTES.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kibana/templates/NOTES.txt b/kibana/templates/NOTES.txt
index 1182a7db0..b2627e128 100644
--- a/kibana/templates/NOTES.txt
+++ b/kibana/templates/NOTES.txt
@@ -2,3 +2,5 @@
   $ kubectl get pods --namespace={{ .Release.Namespace }} -l release={{ .Release.Name }} -w
 2. Retrieve the elastic user's password.
   $ kubectl get secrets --namespace={{ .Release.Namespace }} {{ .Values.elasticsearchCredentialSecret }} -ojsonpath='{.data.password}' | base64 -d
+3. Retrieve the kibana service account token.
+  $ kubectl get secrets --namespace={{ .Release.Namespace }} {{ template "kibana.fullname" . }}-es-token -ojsonpath='{.data.password}' | base64 -d

From 86386f088893b75313de30c92dc15b73131e3b88 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:52:29 +0100
Subject: [PATCH 11/15] exit when catching an error

---
 kibana/templates/configmap-helm-scripts.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index fbb77a468..edfa24966 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -160,12 +160,14 @@ data:
         console.log('Creating a new Elasticsearch token for Kibana')
         createEsToken().catch(err => {
           console.error(err);
+          process.exit(1);
         });
         break;
       case 'clean':
         console.log('Cleaning the Kibana Elasticsearch token')
         cleanEsToken().catch(err => {
           console.error(err);
+          process.exit(1);
         });
         break;
       default:

From cb0d6bbc6c4946e78940387c794acdf86b680b50 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:53:58 +0100
Subject: [PATCH 12/15] add more retries instead of sleep

---
 kibana/examples/upgrade/Makefile      | 1 -
 kibana/templates/pre-install-job.yaml | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/kibana/examples/upgrade/Makefile b/kibana/examples/upgrade/Makefile
index a51caeccd..882a3e289 100644
--- a/kibana/examples/upgrade/Makefile
+++ b/kibana/examples/upgrade/Makefile
@@ -23,7 +23,6 @@ upgrade-es:
 	kubectl delete pod --selector=app=$(ES_CLUSTER)-master
 
 upgrade-kb:
-	sleep 120 # wait that ES pods are started
 	helm upgrade $(RELEASE) ../../ --wait --values values.yaml
 	kubectl rollout status deployment $(RELEASE)-kibana
 
diff --git a/kibana/templates/pre-install-job.yaml b/kibana/templates/pre-install-job.yaml
index 7edf406dd..077cb3e6f 100644
--- a/kibana/templates/pre-install-job.yaml
+++ b/kibana/templates/pre-install-job.yaml
@@ -12,7 +12,7 @@ metadata:
     {{- end }}
     {{- end }}
 spec:
-  backoffLimit: 3
+  backoffLimit: 20
   template:
     spec:
       restartPolicy: Never

From c08720a2c0207104a1cc6aa859bf53a4a2157024 Mon Sep 17 00:00:00 2001
From: jmlrt <8582351+jmlrt@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:57:38 +0100
Subject: [PATCH 13/15] fixup! add a note to retrieve the kibana token

---
 kibana/templates/NOTES.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kibana/templates/NOTES.txt b/kibana/templates/NOTES.txt
index b2627e128..98997b306 100644
--- a/kibana/templates/NOTES.txt
+++ b/kibana/templates/NOTES.txt
@@ -3,4 +3,4 @@
 2. Retrieve the elastic user's password.
   $ kubectl get secrets --namespace={{ .Release.Namespace }} {{ .Values.elasticsearchCredentialSecret }} -ojsonpath='{.data.password}' | base64 -d
 3. Retrieve the kibana service account token.
-  $ kubectl get secrets --namespace={{ .Release.Namespace }} {{ template "kibana.fullname" . }}-es-token -ojsonpath='{.data.password}' | base64 -d
+  $ kubectl get secrets --namespace={{ .Release.Namespace }} {{ template "kibana.fullname" . }}-es-token -ojsonpath='{.data.token}' | base64 -d

From 47ebb41bb962b2ab62fa1ef922ca610f22ee822f Mon Sep 17 00:00:00 2001
From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com>
Date: Thu, 10 Nov 2022 13:15:58 +0100
Subject: [PATCH 14/15] Update kibana/templates/configmap-helm-scripts.yaml

Co-authored-by: Rory Hunter <pugnascotia@users.noreply.github.com>
---
 kibana/templates/configmap-helm-scripts.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index edfa24966..b290e92f1 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -19,11 +19,10 @@ data:
 
     // Read environment variables
     function getEnvVar(name) {
-      if (name in process.env) {
-        return process.env[name]
-      } else {
+      if (!process.env[name]) {
         throw new Error(name + ' environment variable is missing')
       }
+      return process.env[name]
     }
 
     // Elasticsearch API

From 84a955d3f2eb56beccd073d001faa59bec84b48c Mon Sep 17 00:00:00 2001
From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com>
Date: Thu, 10 Nov 2022 13:16:26 +0100
Subject: [PATCH 15/15] Update kibana/templates/configmap-helm-scripts.yaml

Co-authored-by: Rory Hunter <pugnascotia@users.noreply.github.com>
---
 kibana/templates/configmap-helm-scripts.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kibana/templates/configmap-helm-scripts.yaml b/kibana/templates/configmap-helm-scripts.yaml
index b290e92f1..61e55735f 100644
--- a/kibana/templates/configmap-helm-scripts.yaml
+++ b/kibana/templates/configmap-helm-scripts.yaml
@@ -171,4 +171,5 @@ data:
         break;
       default:
         console.log('Unknown command');
+        process.exit(1);
     }