Add Fluentd plugin extensibility points for logs

SumoLogic · Jan 28, 2021 · 621d308 · 621d308
1 parent 11762e9
commit 621d308
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 0 deletions.
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -110,7 +110,10 @@ Parameter | Description | Default
 `fluentd.logs.containers.k8sMetadataFilter.clientKey` | Path to a client key file to authenticate to the API server. | `Nil`
 `fluentd.logs.containers.k8sMetadataFilter.bearerTokenFile` | Path to a file containing the bearer token to use for authentication. | `Nil`
 `fluentd.logs.containers.extraFilterPluginConf` | To use additional filter plugins. | `Nil`
+`fluentd.logs.containers.extraOutputPluginConf` | To use additional output plugins. | `Nil`
 `fluentd.logs.kubelet.enabled` | Collect kubelet logs. | `true`
+`fluentd.logs.kubelet.extraFilterPluginConf` | To use additional filter plugins. | `Nil`
+`fluentd.logs.kubelet.extraOutputPluginConf` | To use additional output plugins. | `Nil`
 `fluentd.logs.kubelet.outputConf` | Output configuration for kubelet. | `@include logs.output.conf`
 `fluentd.logs.kubelet.overrideOutputConf` | Override output section for kubelet logs. Leave empty for the default output section. | `Nil`
 `fluentd.logs.kubelet.sourceName` | Set the _sourceName metadata field in Sumo Logic. | `k8s_kubelet`
@@ -122,6 +125,8 @@ Parameter | Description | Default
 `fluentd.logs.kubelet.excludePriorityRegex` | A regular expression for priority. Matching priority will be excluded from Sumo. The logs will still be sent to FluentD. | `Nil`
 `fluentd.logs.kubelet.excludeUnitRegex` | A regular expression for unit. Matching unit will be excluded from Sumo. The logs will still be sent to FluentD. | `Nil`
 `fluentd.logs.systemd.enabled` | Collect systemd logs. | `true`
+`fluentd.logs.systemd.extraFilterPluginConf` | To use additional filter plugins. | `Nil`
+`fluentd.logs.systemd.extraOutputPluginConf` | To use additional output plugins. | `Nil`
 `fluentd.logs.systemd.outputConf` | Output configuration for systemd. | `@include logs.output.conf`
 `fluentd.logs.systemd.overrideOutputConf` | Override output section for systemd logs. Leave empty for the default output section. | `Nil`
 `fluentd.logs.systemd.sourceCategory` | Set the _sourceCategory metadata field in Sumo Logic. | `system`
@@ -131,6 +136,8 @@ Parameter | Description | Default
 `fluentd.logs.systemd.excludeHostRegex` | A regular expression for hosts. Matching hosts will be excluded from Sumo. The logs will still be sent to FluentD. | `Nil`
 `fluentd.logs.systemd.excludePriorityRegex` | A regular expression for priority. Matching priority will be excluded from Sumo. The logs will still be sent to FluentD. | `Nil`
 `fluentd.logs.systemd.excludeUnitRegex` | A regular expression for unit. Matching unit will be excluded from Sumo. The logs will still be sent to FluentD. | `Nil`
+`fluentd.logs.default.extraFilterPluginConf` | To use additional filter plugins. | `Nil`
+`fluentd.logs.default.extraOutputPluginConf` | To use additional output plugins. | `Nil`
 `fluentd.logs.default.outputConf` | Default log configuration (catch-all). | `@include logs.output.conf`
 `fluentd.logs.default.overrideOutputConf` | Override output section for untagged logs. Leave empty for the default output section. | `Nil`
 `fluentd.metrics.enabled` | Flag to control deploying the Fluentd metrics statefulsets. | `true`

diff --git a/deploy/helm/sumologic/conf/logs/logs.source.containers.conf b/deploy/helm/sumologic/conf/logs/logs.source.containers.conf
@@ -69,6 +69,7 @@
     exclude_namespace_regex {{ .Values.fluentd.logs.containers.excludeNamespaceRegex | quote }}
     {{- end }}
   </filter>
+{{- .Values.fluentd.logs.containers.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.containers.overrideOutputConf }}
   {{ .Values.fluentd.logs.containers.overrideOutputConf | nindent 4 }}
   {{- else }}

diff --git a/deploy/helm/sumologic/conf/logs/logs.source.default.conf b/deploy/helm/sumologic/conf/logs/logs.source.default.conf
@@ -11,6 +11,7 @@
     pattern /disable filter chain optimization/
   </exclude>
 </filter>
+{{- .Values.fluentd.logs.default.extraFilterPluginConf | nindent 4 }}
 <filter **>
   @type kubernetes_sumologic
   source_name {{ .Values.fluentd.logs.default.sourceName | quote }}
@@ -22,6 +23,7 @@
   exclude_priority_regex {{ .Values.fluentd.logs.default.excludePriorityRegex | quote }}
   exclude_unit_regex {{ .Values.fluentd.logs.default.excludeUnitRegex | quote }}
 </filter>
+{{- .Values.fluentd.logs.default.extraOutputPluginConf | nindent 4 }}
 <match **>
   @type copy
   <store>

diff --git a/deploy/helm/sumologic/conf/logs/logs.source.systemd.conf b/deploy/helm/sumologic/conf/logs/logs.source.systemd.conf
@@ -7,6 +7,7 @@
   @label @KUBELET
 </match>
 <label @KUBELET>
+{{- .Values.fluentd.logs.kubelet.extraFilterPluginConf | nindent 4 }}
   <filter host.kubelet.**>
     @type kubernetes_sumologic
     source_category {{ .Values.fluentd.logs.kubelet.sourceCategory | quote }}
@@ -18,6 +19,7 @@
     exclude_priority_regex {{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}
     exclude_unit_regex {{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}
   </filter>
+{{- .Values.fluentd.logs.kubelet.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.kubelet.overrideOutputConf }}
   {{ .Values.fluentd.logs.kubelet.overrideOutputConf | nindent 4}}
   {{- else}}
@@ -58,6 +60,7 @@
   @label @SYSTEMD
 </match>
 <label @SYSTEMD>
+{{- .Values.fluentd.logs.systemd.extraFilterPluginConf | nindent 4 }}
   <filter host.**>
     @type kubernetes_sumologic
     source_name {{ .Values.fluentd.logs.systemd.sourceName | quote }}
@@ -75,6 +78,7 @@
       _sumo_metadata ${record["_sumo_metadata"][:source] = tag_parts[1]; record["_sumo_metadata"]}
     </record>
   </filter>
+{{- .Values.fluentd.logs.systemd.extraOutputPluginConf | nindent 4 }}
   {{ if .Values.fluentd.logs.systemd.overrideOutputConf }}
   {{ .Values.fluentd.logs.systemd.overrideOutputConf | nindent 4 }}
   {{- else}}

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -443,13 +443,22 @@ fluentd:
       ## To use additional filter plugins
       extraFilterPluginConf: |-
 
+      ## To use additional output plugins
+      extraOutputPluginConf: |-
+
       ## To enable stiching multiline logs in fluentd when fluent-bit Multiline feature is On
       multiline:
         enabled: true
 
     ## Kubelet log configuration
     kubelet:
       enabled: true
+      ## To use additional filter plugins
+      extraFilterPluginConf: |-
+
+      ## To use additional output plugins
+      extraOutputPluginConf: |-
+
       outputConf: |-
         @include logs.output.conf
 
@@ -481,6 +490,12 @@ fluentd:
     ## Systemd log configuration
     systemd:
       enabled: true
+      ## To use additional filter plugins
+      extraFilterPluginConf: |-
+
+      ## To use additional output plugins
+      extraOutputPluginConf: |-
+
       outputConf: |-
         @include logs.output.conf
 
@@ -511,6 +526,12 @@ fluentd:
 
     ## Default log configuration (catch-all)
     default:
+      ## To use additional filter plugins
+      extraFilterPluginConf: |-
+
+      ## To use additional output plugins
+      extraOutputPluginConf: |-
+
       outputConf: |-
         @include logs.output.conf