-
Notifications
You must be signed in to change notification settings - Fork 183
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: refactor OT log collector configuration
- Loading branch information
Mikołaj Świątek
committed
Aug 24, 2022
1 parent
6bf0bbe
commit 2c585a4
Showing
16 changed files
with
713 additions
and
338 deletions.
There are no files selected for viewing
244 changes: 243 additions & 1 deletion
244
deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,243 @@ | ||
{{ tpl (toYaml .Values.otellogs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} | ||
extensions: | ||
health_check: {} | ||
file_storage: | ||
directory: /var/lib/storage/otc | ||
timeout: 10s | ||
compaction: | ||
on_start: true | ||
on_rebound: true | ||
# Can't be /tmp yet, see https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/13449 | ||
directory: /var/lib/storage/otc | ||
pprof: {} | ||
service: | ||
telemetry: | ||
logs: | ||
level: {{ .Values.otellogs.logLevel | quote }} | ||
extensions: | ||
- health_check | ||
- file_storage | ||
- pprof | ||
pipelines: | ||
logs/containers: | ||
receivers: | ||
- filelog/containers | ||
processors: | ||
- batch | ||
exporters: | ||
- otlphttp | ||
{{- if .Values.sumologic.logs.systemd.enabled }} | ||
logs/systemd: | ||
receivers: | ||
- journald | ||
processors: | ||
- logstransform/systemd | ||
- batch | ||
exporters: | ||
- otlphttp | ||
{{- end }} | ||
receivers: | ||
filelog/containers: | ||
include: | ||
- /var/log/pods/*/*/*.log | ||
start_at: beginning | ||
## sets fingerprint_size to 17kb in order to match the longest possible docker line (which by default is 16kb) | ||
## we want to include timestamp, which is at the end of the line | ||
fingerprint_size: 17408 | ||
include_file_path: true | ||
include_file_name: false | ||
operators: | ||
## Detect the container runtime log format | ||
## Can be: docker-shim, CRI-O and containerd | ||
- id: get-format | ||
type: router | ||
routes: | ||
- output: parser-docker | ||
expr: 'body matches "^\\{"' | ||
- output: parser-crio | ||
expr: 'body matches "^[^ Z]+ "' | ||
- output: parser-containerd | ||
expr: 'body matches "^[^ Z]+Z"' | ||
## Parse CRI-O format | ||
- id: parser-crio | ||
type: regex_parser | ||
regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$' | ||
output: merge-cri-lines | ||
parse_to: body | ||
timestamp: | ||
parse_from: body.time | ||
layout_type: gotime | ||
layout: '2006-01-02T15:04:05.000000000-07:00' | ||
## Parse CRI-Containerd format | ||
- id: parser-containerd | ||
type: regex_parser | ||
regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$' | ||
output: merge-cri-lines | ||
parse_to: body | ||
timestamp: | ||
parse_from: body.time | ||
layout: '%Y-%m-%dT%H:%M:%S.%LZ' | ||
## Parse docker-shim format | ||
## parser-docker interprets the input string as JSON and moves the `time` field from the JSON to Timestamp field in the OTLP log | ||
## record. | ||
## Input Body (string): '{"log":"2001-02-03 04:05:06 first line\n","stream":"stdout","time":"2021-11-25T09:59:13.23887954Z"}' | ||
## Output Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" } | ||
## Input Timestamp: _empty_ | ||
## Output Timestamp: 2021-11-25 09:59:13.23887954 +0000 UTC | ||
- id: parser-docker | ||
type: json_parser | ||
parse_to: body | ||
output: merge-docker-lines | ||
timestamp: | ||
parse_from: body.time | ||
layout: '%Y-%m-%dT%H:%M:%S.%LZ' | ||
|
||
## merge-docker-lines stitches back together log lines split by Docker logging driver. | ||
## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "stream": "stdout" } | ||
## Input Body (JSON): { "log": "ne that was split by the logging driver\n", "stream": "stdout" } | ||
## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n","stream":"stdout"} | ||
- id: merge-docker-lines | ||
type: recombine | ||
source_identifier: attributes["log.file.path"] | ||
output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }} | ||
combine_field: body.log | ||
combine_with: "" | ||
is_last_entry: body.log matches "\n$" | ||
|
||
## merge-cri-lines stitches back together log lines split by CRI logging drivers. | ||
## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" } | ||
## Input Body (JSON): { "log": "ne that was split by the logging driver", "logtag": "F" } | ||
## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" } | ||
- id: merge-cri-lines | ||
type: recombine | ||
source_identifier: attributes["log.file.path"] | ||
output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }} | ||
combine_field: body.log | ||
combine_with: "" | ||
is_last_entry: body.logtag == "F" | ||
overwrite_with: newest | ||
|
||
## merge-multiline-logs merges incoming log records into multiline logs. | ||
## Input Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" } | ||
## Input Body (JSON): { "log": " second line\n", "stream": "stdout" } | ||
## Input Body (JSON): { "log": " third line\n", "stream": "stdout" } | ||
## Output Body (JSON): { "log": "2001-02-03 04:05:06 first line\n second line\n third line\n", "stream": "stdout" } | ||
{{- if .Values.sumologic.logs.multiline.enabled }} | ||
- id: merge-multiline-logs | ||
type: recombine | ||
output: extract-metadata-from-filepath | ||
source_identifier: attributes["log.file.path"] | ||
combine_field: body.log | ||
combine_with: "" | ||
is_first_entry: body.log matches {{ .Values.sumologic.logs.multiline.start_regex | quote }} | ||
{{- end }} | ||
|
||
## extract-metadata-from-filepath extracts data from the `log.file.path` Attribute into the Attributes | ||
## Input Attributes: | ||
## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
## Output Attributes: | ||
## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
## - container_name: "loggercontainer", | ||
## - namespace: "default", | ||
## - pod_name: "logger-multiline-4nvg4", | ||
## - run_id: "0", | ||
## - uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
## } | ||
- id: extract-metadata-from-filepath | ||
type: regex_parser | ||
regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$' | ||
parse_from: attributes["log.file.path"] | ||
|
||
## The following actions are being performed: | ||
## - renaming attributes | ||
## - moving stream from body to attribtues | ||
## - using body.log as body | ||
## - create fluent.tag attribute in order to route in metadata pods | ||
## Input Body (JSON): { | ||
## "log": "2001-02-03 04:05:06 loggerlog 1 first line\n", | ||
## "stream": "stdout", | ||
## } | ||
## Output Body (String): "2001-02-03 04:05:06 loggerlog 1 first line\n" | ||
## Input Attributes: | ||
## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
## - container_name: "loggercontainer", | ||
## - namespace: "default", | ||
## - pod_name: "logger-multiline-4nvg4", | ||
## - run_id: "0", | ||
## - uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
## Output Attributes: | ||
## - k8s.container.name: "loggercontainer" | ||
## - k8s.namespace.name: "default" | ||
## - k8s.pod.name: "logger-multiline-4nvg4" | ||
## - k8s.pod.uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
## - run_id: "0" | ||
## - stream: "stdout" | ||
## - fluent.tag: "containers.loggercontainer" | ||
- id: move-attributes | ||
type: move | ||
from: body.stream | ||
to: attributes["stream"] | ||
- type: move | ||
from: attributes.container_name | ||
to: attributes["k8s.container.name"] | ||
- type: move | ||
from: attributes.namespace | ||
to: attributes["k8s.namespace.name"] | ||
- type: move | ||
from: attributes.pod_name | ||
to: attributes["k8s.pod.name"] | ||
- type: move | ||
from: attributes.run_id | ||
to: attributes["run_id"] | ||
- type: move | ||
from: attributes.uid | ||
to: attributes["k8s.pod.uid"] | ||
- type: add | ||
field: attributes["fluent.tag"] | ||
value: EXPR("containers." + attributes["k8s.container.name"]) | ||
- type: remove | ||
field: attributes["log.file.path"] | ||
- type: move | ||
from: body.log | ||
to: body | ||
{{- if .Values.sumologic.logs.systemd.enabled }} | ||
journald: | ||
directory: /var/log/journal | ||
## This is not a full equivalent of fluent-bit filtering as fluent-bit filters by `_SYSTEMD_UNIT` | ||
## Here is filtering by `UNIT` | ||
units: | ||
{{ toYaml .Values.sumologic.logs.systemd.units | nindent 6 }} | ||
{{- end }} | ||
exporters: | ||
otlphttp: | ||
endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local:4318 | ||
processors: | ||
batch: | ||
send_batch_size: 1000 | ||
timeout: 1s | ||
## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes | ||
## so they can be used by metadata processors same way like for fluentd | ||
## build fluent.tag attribute as `host.{_SYSTEMD_UNIT}` | ||
{{- if .Values.sumologic.logs.systemd.enabled }} | ||
logstransform/systemd: | ||
operators: | ||
- type: copy | ||
from: body._SYSTEMD_UNIT | ||
to: attributes._SYSTEMD_UNIT | ||
- type: copy | ||
from: body.SYSLOG_FACILITY | ||
to: attributes.SYSLOG_FACILITY | ||
- type: copy | ||
from: body._HOSTNAME | ||
to: attributes._HOSTNAME | ||
- type: copy | ||
from: body.PRIORITY | ||
to: attributes.PRIORITY | ||
- type: add | ||
field: attributes["fluent.tag"] | ||
value: EXPR("host." + attributes["_SYSTEMD_UNIT"]) | ||
## Removes __CURSOR and __MONOTONIC_TIMESTAMP keys from body | ||
- type: remove | ||
field: body.__CURSOR | ||
- type: remove | ||
field: body.__MONOTONIC_TIMESTAMP | ||
{{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
deploy/helm/sumologic/templates/logs/collector/otelcol/service.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
deploy/helm/sumologic/templates/logs/collector/otelcol/serviceaccount.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.