fix(logs): correctly handle newlines

Fixes #2802 #2803
SumoLogic · Jan 19, 2023 · d24928f · d24928f
1 parent 37c8c00
commit d24928f
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,6 +33,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - fix(logs): make `excludeHostRegex` consistent between Otelcol and Fluentd [#2771]
   - The `sumologic.logs.container.excludeHostRegex` should filter on the Kubernetes node name,
     to be consistent with Fluentd and chart v2.
+- fix(logs): correctly handle newlines [#2805]
+  - Fixes [#2802], [#2803]
 
 [#2724]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2724
 [#2745]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2745
@@ -50,6 +52,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2791]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2791
 [#2773]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2773
 [#2790]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2790
+[#2802]: https://github.com/SumoLogic/sumologic-kubernetes-collection/issues/2802
+[#2803]: https://github.com/SumoLogic/sumologic-kubernetes-collection/issues/2803
+[#2805]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2805
 [v1.15.3-sumo-0]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/releases/tag/v1.15.3-sumo-0
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0-beta.0...main
 

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
@@ -101,10 +101,10 @@ receivers:
       - id: merge-docker-lines
         type: recombine
         source_identifier: attributes["log.file.path"]
-        output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }}
         combine_field: body.log
         combine_with: ""
         is_last_entry: body.log matches "\n$"
+        output: strip-trailing-newline
 
       ## merge-cri-lines stitches back together log lines split by CRI logging drivers.
       ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" }
@@ -119,6 +119,16 @@ receivers:
         is_last_entry: body.logtag == "F"
         overwrite_with: newest
 
+      ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime.
+      ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
+      ## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver", "stream": "stdout" }
+      - id: strip-trailing-newline
+        type: regex_parser
+        regex: "^(?P<log>.*)\n$"
+        parse_from: body.log
+        parse_to: body
+        output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }}
+
       ## merge-multiline-logs merges incoming log records into multiline logs.
       ## Input Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" }
       ## Input Body (JSON): { "log": "  second line\n", "stream": "stdout" }
@@ -130,7 +140,7 @@ receivers:
         output: extract-metadata-from-filepath
         source_identifier: attributes["log.file.path"]
         combine_field: body.log
-        combine_with: ""
+        combine_with: "\n"
         is_first_entry: body.log matches {{ .Values.sumologic.logs.multiline.first_line_regex | quote }}
 {{- end }}
 

diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml
@@ -94,7 +94,7 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          output: merge-multiline-logs
+          output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
@@ -105,8 +105,17 @@ data:
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]
           type: recombine
+        - id: strip-trailing-newline
+          output: merge-multiline-logs
+          parse_from: body.log
+          parse_to: body
+          regex: |-
+            ^(?P<log>.*)
+            $
+          type: regex_parser
         - combine_field: body.log
-          combine_with: ""
+          combine_with: |2+
+
           id: merge-multiline-logs
           is_first_entry: body.log matches "^\\[?\\d{4}-\\d{1,2}-\\d{1,2}.\\d{2}:\\d{2}:\\d{2}"
           output: extract-metadata-from-filepath