Add custom terraform and bash script to the setup

deploy/docker/Dockerfile

@@ -71,8 +71,8 @@ RUN apt-get update \
         libsnappy-dev
 
 # Create directory for terraform configuration
-RUN mkdir -p /terraform \
- && chown -R fluent /terraform
+RUN mkdir -p /terraform /scripts \
+ && chown -R fluent /terraform /scripts
 
 COPY --from=builder /bin/terraform /bin/terraform
 COPY --from=builder /usr/local/bundle /usr/local/bundle

deploy/helm/sumologic/conf/setup/custom.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# This script copies files from /customer-scripts to /scripts/<dirname> basing on the filename
+#
+# Example file structure:
+#
+# /customer-scripts
+# ├── dir1_main.tf
+# ├── dir1_setup.sh
+# ├── dir2_list.txt
+# └── dir2_setup.sh
+#
+# Expected structure:
+#
+# /scripts
+# ├── dir1
+# │   ├── main.tf
+# │   └── setup.sh
+# └── dir2
+#     ├── list.txt
+#     └── setup.sh
+#
+# shellcheck disable=SC2010
+# extract target directory names from the file names using _ as separator
+for dir in $(ls -1 /customer-scripts | grep _ | grep -oE '^.*?_' | sed 's/_//g' | sort | uniq); do
+  target="/scripts/${dir}"
+  mkdir "${target}"
+  # shellcheck disable=SC2010
+  # Get files for given directory and take only filename part (after first _)
+  for file in $(ls -1 "/customer-scripts/${dir}_"* | grep -oE '_.*' | sed 's/_//g'); do
+    cp "/customer-scripts/${dir}_${file}" "${target}/${file}"
+  done
+  cd "${target}" && bash setup.sh
+done

deploy/helm/sumologic/conf/setup/locals.tf

@@ -3,8 +3,10 @@ locals {
 {{- range $type, $sources := .Values.sumologic.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
+{{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
   {{ template "terraform.sources.local" (dict "Name" (include "terraform.sources.name" (dict "Name" $key "Type" $type)) "Value" $source.name) }}
 {{- end }}
 {{- end }}
 {{- end }}
+{{- end }}
 }

deploy/helm/sumologic/conf/setup/resources.tf

@@ -13,10 +13,12 @@ resource "sumologic_collector" "collector" {
 {{- range $type, $sources := .Values.sumologic.sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
 {{- range $key, $source := $sources }}
+{{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
 {{ include "terraform.sources.resource" (dict "Name" (include "terraform.sources.name" (dict "Name" $key "Type" $type)) "Source" $source "Context" $ctx) | nindent 2 }}
 {{- end }}
 {{- end }}
 {{- end }}
+{{- end }}
 
 resource "kubernetes_secret" "sumologic_collection_secret" {
   metadata {
@@ -29,10 +31,12 @@ resource "kubernetes_secret" "sumologic_collection_secret" {
     {{- range $type, $sources := .Values.sumologic.sources }}
     {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
     {{- range $key, $source := $sources }}
+    {{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
     {{ include "terraform.sources.data" (dict "Endpoint" (include "terraform.sources.config-map-variable" (dict "Type" $type "Context" $ctx "Name" $key)) "Name" (include "terraform.sources.name" (dict "Name" $key "Type" $type))) }}
     {{- end }}
     {{- end }}
     {{- end }}
+    {{- end }}
   }
 
   type = "Opaque"

deploy/helm/sumologic/conf/setup/setup.sh

@@ -1,5 +1,5 @@
-#!/bin/sh
-cp /etc/terraform/*.tf /terraform
+#!/bin/bash
+cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
 cd /terraform
 
 # Fix URL to remove "v1" or "v1/"
@@ -18,15 +18,23 @@ terraform init
 terraform import sumologic_collector.collector "$COLLECTOR_NAME"
 {{- $ctx := .Values -}}
 {{- range $type, $sources := .Values.sumologic.sources }}
-{{- range $key, $source := $sources }}
 {{- if eq (include "terraform.sources.component_enabled" (dict "Context" $ctx "Type" $type)) "true" }}
+{{- range $key, $source := $sources }}
+{{- if eq (include "terraform.sources.to_create" (dict "Context" $ctx "Type" $type "Name" $key)) "true" }}
 terraform import sumologic_http_source.{{ template "terraform.sources.name" (dict "Name" $key "Type" $type) }} "$COLLECTOR_NAME/{{ $source.name }}"
 {{- end }}
 {{- end }}
 {{- end }}
-
+{{- end }}
 
 # Kubernetes Secret
 terraform import kubernetes_secret.sumologic_collection_secret {{ .Release.Namespace }}/sumologic
 
-terraform apply -auto-approve
+terraform apply -auto-approve
+
+# Cleanup env variables
+export SUMOLOGIC_BASE_URL=
+export SUMOLOGIC_ACCESSKEY=
+export SUMOLOGIC_ACCESSID=
+
+bash /etc/terraform/custom.sh

deploy/helm/sumologic/templates/_helpers.tpl

@@ -151,6 +151,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- template "sumologic.labels.app.setup" . }}
 {{- end -}}
 
+{{- define "sumologic.labels.app.setup.configmap-custom" -}}
+{{- template "sumologic.labels.app.setup" . }}
+{{- end -}}
+
 {{- define "sumologic.labels.app.setup.roles.clusterrole" -}}
 {{- template "sumologic.labels.app.setup" . }}
 {{- end -}}
@@ -305,6 +309,10 @@ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 {{ template "sumologic.metadata.name.setup" . }}
 {{- end -}}
 
+{{- define "sumologic.metadata.name.setup.configmap-custom" -}}
+{{ template "sumologic.metadata.name.setup" . }}-custom
+{{- end -}}
+
 {{- define "sumologic.metadata.name.setup.configmap" -}}
 {{ template "sumologic.metadata.name.setup" . }}
 {{- end -}}
@@ -728,6 +736,26 @@ Example Usage:
 {{ $value }}
 {{- end -}}
 
+{{/*
+Check if particular source is enabled or not
+
+Example Usage:
+{{- if eq (include "terraform.sources.to_create" (dict "Context" .Values "Type" "metrics" .Name "default" )) "true" }}
+
+*/}}
+{{- define "terraform.sources.to_create" -}}
+{{- $type := .Type -}}
+{{- $ctx := .Context -}}
+{{- $name := .Name -}}
+{{- $value := true -}}
+{{- if and (hasKey $ctx.sumologic.sources $type) (hasKey (index $ctx.sumologic.sources $type) $name) (hasKey (index $ctx.sumologic.sources $type $name) "create") -}}
+{{- if not (index $ctx.sumologic.sources $type $name "create") -}}
+{{- $value = false -}}
+{{- end -}}
+{{- end -}}
+{{ $value }}
+{{- end -}}
+
 {{/*
 Generate fluentd envs for given source type:
 

deploy/helm/sumologic/templates/setup/setup-configmap.yaml

@@ -10,5 +10,4 @@ metadata:
     {{- include "sumologic.labels.common" . | nindent 4 }}
 data:
   {{- (tpl (.Files.Glob "conf/setup/*").AsConfig .) | nindent 2 }}
-
 {{- end }}

deploy/helm/sumologic/templates/setup/setup-custom-configmap.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.sumologic.setup.additionalFiles }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name:  {{ template "sumologic.metadata.name.setup.configmap-custom" . }}
+  annotations:
+{{ include "sumologic.annotations.app.setup.helmsh" "2" | indent 4 }}
+  labels:
+    app: {{ template "sumologic.labels.app.setup.configmap-custom" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}
+data:
+  {{- range $directory, $files := .Values.sumologic.setup.additionalFiles }}
+  {{- range $filename, $file := $files }}
+  {{ printf "%s_%s: |-" $directory $filename }}
+    {{- $file | nindent 4 }}
+  {{- end }}
+  {{- end }}
+{{- end }}

deploy/helm/sumologic/templates/setup/setup-job.yaml

@@ -38,6 +38,10 @@ spec:
         configMap:
           name: {{ template "sumologic.metadata.name.setup.configmap" . }}
           defaultMode: 0777
+      - name: custom
+        configMap:
+          name: {{ template "sumologic.metadata.name.setup.configmap-custom" . }}
+          defaultMode: 0777
       containers:
       - name: setup
         image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
@@ -48,6 +52,8 @@ spec:
         volumeMounts:
         - name: setup
           mountPath: /etc/terraform
+        - name: custom
+          mountPath: /customer-scripts
         {{- if .Values.sumologic.envFromSecret }}
         envFrom:
         - secretRef:

deploy/kubernetes/setup-sumologic.yaml.tmpl

@@ -14,6 +14,41 @@ metadata:
     app: collection-sumologic
 
 data:
+  custom.sh: |
+    #!/bin/bash
+    #
+    # This script copies files from /customer-scripts to /scripts/<dirname> basing on the filename
+    #
+    # Example file structure:
+    #
+    # /customer-scripts
+    # ├── dir1_main.tf
+    # ├── dir1_setup.sh
+    # ├── dir2_list.txt
+    # └── dir2_setup.sh
+    #
+    # Expected structure:
+    #
+    # /scripts
+    # ├── dir1
+    # │   ├── main.tf
+    # │   └── setup.sh
+    # └── dir2
+    #     ├── list.txt
+    #     └── setup.sh
+    #
+    # shellcheck disable=SC2010
+    # extract target directory names from the file names using _ as separator
+    for dir in $(ls -1 /customer-scripts | grep _ | grep -oE '^.*?_' | sed 's/_//g' | sort | uniq); do
+      target="/scripts/${dir}"
+      mkdir "${target}"
+      # shellcheck disable=SC2010
+      # Get files for given directory and take only filename part (after first _)
+      for file in $(ls -1 "/customer-scripts/${dir}_"* | grep -oE '_.*' | sed 's/_//g'); do
+        cp "/customer-scripts/${dir}_${file}" "${target}/${file}"
+      done
+      cd "${target}" && bash setup.sh
+    done
   locals.tf: |
     locals {
       default_events_source                       = "events"
@@ -124,9 +159,9 @@ data:
 
       type = "Opaque"
     }
-  setup.sh: |-
-    #!/bin/sh
-    cp /etc/terraform/*.tf /terraform
+  setup.sh: |
+    #!/bin/bash
+    cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
     cd /terraform
 
     # Fix URL to remove "v1" or "v1/"
@@ -154,11 +189,17 @@ data:
     terraform import sumologic_http_source.scheduler_metrics_source "$COLLECTOR_NAME/kube-scheduler-metrics"
     terraform import sumologic_http_source.state_metrics_source "$COLLECTOR_NAME/kube-state-metrics"
 
-
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret $NAMESPACE/sumologic
 
     terraform apply -auto-approve
+
+    # Cleanup env variables
+    export SUMOLOGIC_BASE_URL=
+    export SUMOLOGIC_ACCESSKEY=
+    export SUMOLOGIC_ACCESSID=
+
+    bash /etc/terraform/custom.sh
   variables.tf: |-
     variable "cluster_name" {
       type  = string
@@ -264,6 +305,10 @@ spec:
         configMap:
           name: collection-sumologic-setup
           defaultMode: 0777
+      - name: custom
+        configMap:
+          name: collection-sumologic-setup-custom
+          defaultMode: 0777
       containers:
       - name: setup
         image: sumologic/kubernetes-fluentd:1.3.0
@@ -280,6 +325,8 @@ spec:
         volumeMounts:
         - name: setup
           mountPath: /etc/terraform
+        - name: custom
+          mountPath: /customer-scripts
         env:
         - name: SUMOLOGIC_ACCESSID
           value: $SUMOLOGIC_ACCESSID
@@ -297,6 +344,10 @@ spec:
       securityContext:
         runAsUser: 999
 
+---
+# Source: sumologic/templates/setup/setup-custom-configmap.yaml
+
+
 ---
 # Source: sumologic/templates/setup/setup-scc.yaml
 

tests/terraform/static/all_fields.output.yaml

@@ -14,6 +14,41 @@ metadata:
     release: "RELEASE-NAME"
     heritage: "Helm"
 data:
+  custom.sh: |
+    #!/bin/bash
+    #
+    # This script copies files from /customer-scripts to /scripts/<dirname> basing on the filename
+    #
+    # Example file structure:
+    #
+    # /customer-scripts
+    # ├── dir1_main.tf
+    # ├── dir1_setup.sh
+    # ├── dir2_list.txt
+    # └── dir2_setup.sh
+    #
+    # Expected structure:
+    #
+    # /scripts
+    # ├── dir1
+    # │   ├── main.tf
+    # │   └── setup.sh
+    # └── dir2
+    #     ├── list.txt
+    #     └── setup.sh
+    #
+    # shellcheck disable=SC2010
+    # extract target directory names from the file names using _ as separator
+    for dir in $(ls -1 /customer-scripts | grep _ | grep -oE '^.*?_' | sed 's/_//g' | sort | uniq); do
+      target="/scripts/${dir}"
+      mkdir "${target}"
+      # shellcheck disable=SC2010
+      # Get files for given directory and take only filename part (after first _)
+      for file in $(ls -1 "/customer-scripts/${dir}_"* | grep -oE '_.*' | sed 's/_//g'); do
+        cp "/customer-scripts/${dir}_${file}" "${target}/${file}"
+      done
+      cd "${target}" && bash setup.sh
+    done
   locals.tf: |
     locals {
       default_events_source                       = "events"
@@ -171,9 +206,9 @@ data:
 
       type = "Opaque"
     }
-  setup.sh: |-
-    #!/bin/sh
-    cp /etc/terraform/*.tf /terraform
+  setup.sh: |
+    #!/bin/bash
+    cp /etc/terraform/{locals,main,providers,resources,variables}.tf /terraform
     cd /terraform
 
     # Fix URL to remove "v1" or "v1/"
@@ -202,11 +237,17 @@ data:
     terraform import sumologic_http_source.state_metrics_source "$COLLECTOR_NAME/kube-state-metrics"
     terraform import sumologic_http_source.test_source_metrics_source "$COLLECTOR_NAME/(Test source)"
 
-
     # Kubernetes Secret
     terraform import kubernetes_secret.sumologic_collection_secret sumologic/sumologic
 
     terraform apply -auto-approve
+
+    # Cleanup env variables
+    export SUMOLOGIC_BASE_URL=
+    export SUMOLOGIC_ACCESSKEY=
+    export SUMOLOGIC_ACCESSID=
+
+    bash /etc/terraform/custom.sh
   variables.tf: |-
     variable "cluster_name" {
       type  = string