Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(logs): add sumologic.logs.container.otelcol.extraProcessors #2790

Merged
merged 1 commit into from
Jan 13, 2023
Merged

feat(logs): add sumologic.logs.container.otelcol.extraProcessors #2790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- feat(logs): add `sumologic.logs.container.otelcol.extraProcessors` [#2790]

### Changed

- feat(metrics): add `sumologic.metrics.otelcol.extraProcessors` [#2724] [#2780]
Expand Down Expand Up @@ -45,6 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
[#2785]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2785
[#2791]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2791
[#2773]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2773
[#2790]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2790
[v1.15.3-sumo-0]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/releases/tag/v1.15.3-sumo-0
[Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0-beta.0...main

Expand Down
1 change: 1 addition & 0 deletions deploy/helm/sumologic/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
| `sumologic.logs.container.excludeHostRegex` | A regular expression for Kubernetes node names. Logs from pods running on matching nodes will not be sent to Sumo. | `""` |
| `sumologic.logs.container.excludeNamespaceRegex` | A regular expression for Kubernetes namespace names. Logs from pods running in matching namespaces will not be sent to Sumo. | `""` |
| `sumologic.logs.container.excludePodRegex` | A regular expression for pod names. Logs from matching pods will not be sent to Sumo. | `""` |
| `sumologic.logs.container.otelcol.extraProcessors` | Extra processors for container logs. See [/docs/collecting-container-logs.md](/docs/collecting-container-logs.md) for details. | `[]` |
| `sumologic.logs.container.perContainerAnnotationsEnabled` | Enable container-level pod annotations. | `false` |
| `sumologic.logs.container.perContainerAnnotationPrefixes` | Defines the list of prefixes of container-level pod annotations. | `[]` |
| `sumologic.logs.systemd.sourceName` | Set the _sourceName metadata field in Sumo Logic. | `"%{_sourceName}"` |
Expand Down
15 changes: 15 additions & 0 deletions deploy/helm/sumologic/conf/logs/otelcol/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,11 @@ processors:
parse_from: body
parse_to: body
if: body matches "^{.*"
{{- if .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{- range $processor := .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{ toYaml $processor | indent 2}}
{{- end }}
{{- end }}
{{ end }}
{{ if .Values.sumologic.logs.systemd.enabled }}
## Systemd related processors
Expand Down Expand Up @@ -362,6 +367,11 @@ service:
- k8s_tagger
- resource/add_cluster
- source/containers
{{- if .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{- range $processor := .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{ printf "- %s" ( $processor | keys | first ) | indent 8 }}
{{- end }}
{{- end }}
- resource/remove_k8s_pod_pod_name
- resource/drop_annotations
- attributes/remove_fluent_tag
Expand Down Expand Up @@ -430,6 +440,11 @@ service:
- resource/add_cluster
- source/containers
- logstransform/containers_parse_json
{{- if .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{- range $processor := .Values.sumologic.logs.container.otelcol.extraProcessors }}
{{ printf "- %s" ( $processor | keys | first ) | indent 8 }}
{{- end }}
{{- end }}
- resource/remove_k8s_pod_pod_name
- resource/drop_annotations
- resource/containers_copy_node_to_host
Expand Down
4 changes: 4 additions & 0 deletions deploy/helm/sumologic/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ sumologic:
container:
enabled: true

otelcol:
## Extra processors for container logs. See [/docs/collecting-container-logs.md](/docs/collecting-container-logs.md) for details.
extraProcessors: []

## Set the _sourceHost metadata field in Sumo Logic.
sourceHost: "%{k8s.pod.hostname}"
## Set the _sourceName metadata field in Sumo Logic.
Expand Down
18 changes: 0 additions & 18 deletions docs/best-practices.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
- [Overriding chart resource names with `fullnameOverride`](#overriding-chart-resource-names-with-fullnameoverride)
- [OpenTelemetry Collector Autoscaling](#opentelemetry-collector-autoscaling)
- [OpenTelemetry Collector Persistent Buffer](#opentelemetry-collector-persistent-buffer)
- [Excluding Logs From Specific Components](#excluding-logs-from-specific-components)
- [Modifying logs](#modifying-logs)
- [Excluding Metrics](#excluding-metrics)
- [Prometheus](#prometheus)
- [Fluentd](#fluentd)
Expand Down Expand Up @@ -165,22 +163,6 @@ For events, use the following properties:
- `sumologic.events.persistence.persistentVolume.storageClass: null`
- `sumologic.events.persistence.size: 10Gi`

## Excluding Logs From Specific Components

You can exclude logs from specific components from being sent to Sumo Logic
using the following properties:

- `sumologic.logs.container.excludeContainerRegex` excludes logs from specific containers
- `sumologic.logs.container.excludeHostRegex` excludes logs from specific Kubernetes nodes
- `sumologic.logs.container.excludeNamespaceRegex` excludes logs from specific Kubernetes namespaces
- `sumologic.logs.container.excludePodRegex` excludes logs from specific pods

:construction: *TODO*, explain how to filter based on log message here

## Modifying logs

:construction: *TODO*, see [the Fluentd section](fluent/best-practices.md#modifying-logs-in-fluentd)

## Excluding Metrics

### Prometheus
Expand Down
107 changes: 99 additions & 8 deletions docs/collecting-container-logs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
- [Modifying log records](#modifying-log-records)
- [Persistence](#persistence)
- [Advanced Configuration](#advanced-configuration)
- [Advanced Filtering](#advanced-filtering)
- [Disabling container logs](#disabling-container-logs)
<!-- /TOC -->

Expand Down Expand Up @@ -109,11 +108,102 @@ sumologic:
excludePodRegex: ""
```

For more advanced filtering logic, see [here](#advanced-filtering).
For more advanced scenarios, use [OpenTelemetry processors][opentelemetry_processors].
Add them to `sumologic.logs.container.otelcol.extraProcessors`.

Here are some examples:

```yaml
sumologic:
logs:
container:
otelcol:
extraProcessors:
- filter/include-logs-based-on-resource-attribute:
logs:
include:
match_type: strict
resource_attributes:
- Key: host.name
Value: just_this_one_hostname
- filter/include-logs-based-on-resource-attribute-regex:
logs:
include:
match_type: regexp
resource_attributes:
- Key: host.name
Value: prefix.*
- filter/exclude-healthcheck-logs:
logs:
exclude:
match_type: regexp
bodies:
- /healthcheck
```

For more examples and detailed documentation, see [Filter processor docs][filter_processor_docs].

### Modifying log records

:construction: This needs `extraProcessors` for logs.
To modify log records, use [OpenTelemetry processors][opentelemetry_processors].
Add them to `sumologic.logs.container.otelcol.extraProcessors`.

Here are some examples.

To modify log body, use the [Transform processor][transform_processor_docs]:

```yaml
sumologic:
logs:
container:
otelcol:
extraProcessors:
- transform/mask-card-numbers:
log_statements:
- context: log
statements:
- replace_pattern(body, "card=\\d+", "card=***")
```

To modify record attributes, use the [Attributes processor][attributes_processor_docs]:

```yaml
sumologic:
logs:
container:
otelcol:
extraProcessors:
- attributes/delete-record-attribute:
actions:
- action: delete
key: unwanted.attribute
# To rename old.attribute to new.attribute, first create new.attribute and then delete old.attribute.
- attributes/rename-old-to-new:
- action: insert
key: new.attribute
from_attribute: old.attribute
- action: delete
key: old.attribute
```

To modify resource attributes, use the [Resource processor][resource_processor_docs]:

```yaml
sumologic:
logs:
container:
otelcol:
extraProcessors:
- resource/add-resource-attribute:
attributes:
- action: insert
key: environment
value: staging
- resource/remove:
attributes:
- action: delete
key: redundant-attribute
```

### Persistence

Expand Down Expand Up @@ -161,10 +251,6 @@ that may change between minor releases of this Chart.

See [Sumologic OpenTelemetry Collector configuration][configuration] for more information.

### Advanced Filtering

:construction: This needs `extraProcessors` for logs.

### Disabling container logs

Container logs are collected by default. This can be disabled by setting:
Expand All @@ -176,7 +262,12 @@ sumologic:
enabled: false
```

[configuration]: https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md
[configuration]: https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/configuration.md
[values]: /deploy/helm/sumologic/values.yaml
[source_name]: https://help.sumologic.com/docs/send-data/reference-information/metadata-naming-conventions/#Source_Name
[opentelemetry_k8s]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/k8s.md
[filter_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/filterprocessor/README.md
[opentelemetry_processors]: https://opentelemetry.io/docs/collector/configuration/#processors
[attributes_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/attributesprocessor/README.md
[resource_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/resourceprocessor/README.md
[transform_processor_docs]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.69.0/processor/transformprocessor/README.md
84 changes: 84 additions & 0 deletions tests/helm/logs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,90 @@ sumologic:
}
}

func TestMetadataOtelExtraProcessors(t *testing.T) {
t.Parallel()
templatePath := "templates/logs/otelcol/configmap.yaml"
valuesYaml := `
sumologic:
logs:
container:
otelcol:
enabled: true
extraProcessors:
- filter/include-host:
logs:
include:
match_type: strict
resource_attributes:
- key: host.name
value: just_this_one_hostname
`
otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)

var otelConfig struct {
Exporters map[string]interface{}
Processors map[string]interface{}
Service struct {
Pipelines map[string]struct {
Receivers []string
Processors []string
Exporters []string
}
}
}
err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
require.NoError(t, err)

require.Contains(t, otelConfig.Processors, "filter/include-host")

containersPipeline := otelConfig.Service.Pipelines["logs/otlp/containers"]
require.Contains(t, containersPipeline.Processors, "filter/include-host")
}

func TestMetadataFluentBitExtraProcessors(t *testing.T) {
t.Parallel()
templatePath := "templates/logs/otelcol/configmap.yaml"
valuesYaml := `
sumologic:
logs:
collector:
otelcol:
enabled: false
container:
otelcol:
extraProcessors:
- filter/include-host:
logs:
include:
match_type: strict
resource_attributes:
- key: host.name
value: just_this_one_hostname
fluent-bit:
enabled: true
`
otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)

var otelConfig struct {
Exporters map[string]interface{}
Processors map[string]interface{}
Service struct {
Pipelines map[string]struct {
Receivers []string
Processors []string
Exporters []string
}
}
}
err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
require.NoError(t, err)

require.Contains(t, otelConfig.Processors, "filter/include-host")

containersPipeline := otelConfig.Service.Pipelines["logs/fluent/containers"]
require.Contains(t, containersPipeline.Processors, "filter/include-host")
}

func TestCollectorOtelConfigMerge(t *testing.T) {
t.Parallel()
templatePath := "templates/logs/collector/otelcol/configmap.yaml"
Expand Down