Add HorizontalPodAutoscaler for fluentd

ci/build.sh

@@ -109,6 +109,16 @@ if [ -n "$GITHUB_TOKEN" ] && [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then
   # Generate override yaml files for chart dependencies to determine if changes are made to overrides yaml files
   echo "Generating overrides files..."
 
+  metrics_server_start=`grep -n "metrics-server:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1`
+  metrics_server_start=$(($metrics_server_start + 2))
+  metrics_server_end=`grep -n "## Configure fluent-bit" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1`
+  metrics_server_end=$(($metrics_server_end - 1))
+
+  echo "Copy 'values.yaml' from line $metrics_server_start to line $metrics_server_end to 'metrics-server-overrides.yaml'"
+  echo "# This file is auto-generated." > deploy/helm/metrics-server-overrides.yaml
+  # Copy lines of metrics_server section and remove indention from values.yaml
+  sed -n "$metrics_server_start,${metrics_server_end}p" deploy/helm/sumologic/values.yaml | sed 's/  //' >> deploy/helm/metrics-server-overrides.yaml
+
   fluent_bit_start=`grep -n "fluent-bit:" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1`
   fluent_bit_start=$(($fluent_bit_start + 2))
   fluent_bit_end=`grep -n "## Configure prometheus-operator" deploy/helm/sumologic/values.yaml | head -n 1 | cut -d: -f1`
@@ -149,7 +159,7 @@ if [ -n "$GITHUB_TOKEN" ] && [ "$TRAVIS_EVENT_TYPE" == "pull_request" ]; then
   # Copy lines of falco section and remove indention from values.yaml
   sed -n "$falco_start,$ p" deploy/helm/sumologic/values.yaml | sed 's/  //' >> deploy/helm/falco-overrides.yaml
 
-  if [ "$(git diff deploy/helm/fluent-bit-overrides.yaml)" ] || [ "$(git diff deploy/helm/prometheus-overrides.yaml)" ] || [ "$(git diff deploy/helm/falco-overrides.yaml)" ] || [ "$(git diff deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet)" ]; then
+  if [ "$(git diff deploy/helm/metrics-server-overrides.yaml)" ] || [ "$(git diff deploy/helm/fluent-bit-overrides.yaml)" ] || [ "$(git diff deploy/helm/prometheus-overrides.yaml)" ] || [ "$(git diff deploy/helm/falco-overrides.yaml)" ] || [ "$(git diff deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet)" ]; then
     echo "Detected changes in 'fluent-bit-overrides.yaml', 'prometheus-overrides.yaml', 'falco-overrides.yaml', or 'kube-prometheus-sumo-logic-mixin.libsonnet', committing the updated version to $TRAVIS_PULL_REQUEST_BRANCH..."
     git add deploy/helm/*-overrides.yaml
     git add deploy/kubernetes/kube-prometheus-sumo-logic-mixin.libsonnet

deploy/helm/metrics-server-overrides.yaml

@@ -0,0 +1,5 @@
+# This file is auto-generated.
+args:
+  - --kubelet-insecure-tls
+  - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+

deploy/helm/sumologic/requirements.yaml

@@ -11,3 +11,7 @@ dependencies:
     version: 1.0.9
     repository: https://kubernetes-charts.storage.googleapis.com/
     condition: falco.enabled
+  - name: metrics-server
+    version: 2.7.0
+    repository: https://kubernetes-charts.storage.googleapis.com/
+    condition: metrics-server.enabled

deploy/helm/sumologic/templates/hpa.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.sumologic.fluentd.autoscaling.enabled}}
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "sumologic.fullname" . }}
+  labels:
+    app: {{ template "sumologic.labels.app" . }}
+    {{- include "sumologic.labels.common" . | nindent 4 }}  
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ template "sumologic.fullname" . }}
+  minReplicas: {{ .Values.sumologic.fluentd.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.sumologic.fluentd.autoscaling.maxReplicas }}
+  targetCPUUtilizationPercentage: {{ .Values.sumologic.fluentd.autoscaling.targetCPUUtilizationPercentage }}
+{{- end -}}

deploy/helm/sumologic/values.yaml

@@ -164,6 +164,12 @@ sumologic:
   fluentd:
     ## Option to specify the Fluentd buffer as file/memory.
     buffer: "memory"
+    ## Option to turn autoscaling on for fluentd and specify metrics for HPA.
+    autoscaling:
+      enabled: false
+      minReplicas: 3
+      maxReplicas: 10
+      targetCPUUtilizationPercentage: 50
 
   k8sMetadataFilter:
     ## Option to control the enabling of metadata filter plugin watch.
@@ -181,6 +187,15 @@ sumologic:
     ## ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration
     bearerCacheTtl: "3600"
 
+
+## Configure metrics-server
+## ref: https://github.com/helm/charts/blob/master/stable/metrics-server/values.yaml
+metrics-server:
+  enabled: true
+  args:
+    - --kubelet-insecure-tls
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+
 ## Configure fluent-bit 
 ## ref: https://github.com/helm/charts/blob/master/stable/fluent-bit/values.yaml
 fluent-bit:

deploy/kubernetes/fluentd-sumologic.yaml.tmpl

@@ -656,3 +656,6 @@ spec:
           value: "100k"
         - name: TOTAL_LIMIT_SIZE
           value: "128m"
+---
+# Source: sumologic/templates/hpa.yaml
+