From 4892a5526e5a0dba61ce8570900841084d280269 Mon Sep 17 00:00:00 2001 From: Katarzyna Kujawa Date: Wed, 16 Nov 2022 14:21:19 +0100 Subject: [PATCH] chore!: remove replacing values in configuration marked by 'replace' suffix make values in configuration for otelcol dependended on otelcol parameters in values.yaml removes "otelcol.k8s.cluster.name" which was added in #1721 and it is no longer in use this pull request reverts changes introduced in #1721 related to #1729 --- CHANGELOG.md | 2 + deploy/docs/v3_migration_doc.md | 40 +++++++++++++++++++ .../conf/traces/traces.otelagent.conf.yaml | 7 +--- .../conf/traces/traces.otelcol.conf.yaml | 27 +------------ deploy/helm/sumologic/templates/_helpers.tpl | 13 ------ deploy/helm/sumologic/values.yaml | 28 ++++++------- 6 files changed, 58 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdba8e8a7e..dab4ccfc45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - fix(fluentd): Removing PodSecurityPolicy for fluentd [#2605] - feat!: refactor event collection configuration [#2444] - fix(logs): configure fluentbit to send data to metadata-logs [#2610] +- chore!: remove replacing values in configuration marked by 'replace' suffix [#2615] ### Changed @@ -71,6 +72,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2444]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2444 [#2610]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2610 [#2619]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2619 +[#2615]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2615 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.17.0...main [telegraf_operator_comapare_1.3.5_and_1.3.10]: https://github.com/influxdata/helm-charts/compare/telegraf-operator-1.3.5...telegraf-operator-1.3.10 [cert-manager-1.4]: https://github.com/cert-manager/cert-manager/releases/tag/v1.4.0 diff --git a/deploy/docs/v3_migration_doc.md b/deploy/docs/v3_migration_doc.md index b1e9cd2ddc..124b75c6dc 100644 --- a/deploy/docs/v3_migration_doc.md +++ b/deploy/docs/v3_migration_doc.md @@ -5,6 +5,7 @@ - [Requirements](#requirements) - [Manual steps](#manual-steps) - [Upgrade kube-prometheus-stack](#upgrade-kube-prometheus-stack) + - [Replace special configuration values marked by 'replace' suffix](#replace-special-configuration-values-marked-by-replace-suffix) Based on the feedback from our users, we will be introducing several changes to the Sumo Logic Kubernetes Collection solution. @@ -18,6 +19,24 @@ In this document we detail the changes as well as the exact steps for migration. We are updating Kube-prometheus-stack to newest available version. Major feature related to that change is upgrading kube-state-metrics to v2 +- Removing mechanism to replace values in configuration for traces marked by 'replace' suffix, + following special values in configuration are no longer automatically replaced and they need to be changed: + + - `exporters.otlptraces.endpoint.replace` + - `exporters.otlpmetrics.endpoint.replace` + - `processors.source.collector.replace` + - `processors.source.name.replace` + - `processors.source.category.replace` + - `processors.source.category_prefix.replace` + - `processors.source.category_replace_dash.replace` + - `processors.source.exclude_namespace_regex.replace` + - `processors.source.exclude_pod_regex.replace` + - `processors.source.exclude_container_regex.replace` + - `processors.source.exclude_host_regex.replace` + - `processors.resource.cluster.replace` + - `exporters.sumologic.source_name.replace` + - `exporters.sumologic.source_category.replace` + ## How to upgrade ### Requirements @@ -73,3 +92,24 @@ Upgrade of kube-prometheus-stack is a breaking change and requires manual steps: ```text Error: UPGRADE FAILED: cannot patch "collection-kube-state-metrics" with kind Deployment: Deployment.apps "collection-kube-state-metrics" is invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/instance":"collection", "app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable ``` + +### Replace special configuration values marked by 'replace' suffix + +Mechanism to replace special configuration values for traces marked by 'replace' suffix was removed and following special values in configuration are no longer automatically replaced and they need to be changed: + +- `exporters.otlptraces.endpoint.replace` +- `exporters.otlpmetrics.endpoint.replace` +- `processors.source.collector.replace` +- `processors.source.name.replace` +- `processors.source.category.replace` +- `processors.source.category_prefix.replace` +- `processors.source.category_replace_dash.replace` +- `processors.source.exclude_namespace_regex.replace` +- `processors.source.exclude_pod_regex.replace` +- `processors.source.exclude_container_regex.replace` +- `processors.source.exclude_host_regex.replace` +- `processors.resource.cluster.replace` +- `exporters.sumologic.source_name.replace` +- `exporters.sumologic.source_category.replace` + +Above special configuration values can be replaced either to direct values or be set as reference to other parameters form `values.yaml`. diff --git a/deploy/helm/sumologic/conf/traces/traces.otelagent.conf.yaml b/deploy/helm/sumologic/conf/traces/traces.otelagent.conf.yaml index d7fa87a6bd..3cc86d1ec1 100644 --- a/deploy/helm/sumologic/conf/traces/traces.otelagent.conf.yaml +++ b/deploy/helm/sumologic/conf/traces/traces.otelagent.conf.yaml @@ -1,6 +1 @@ -{{- $yamlFile := toYaml $.Values.otelagent.config }} -{{- $otelTracesService := include "traces.otelagent.exporter.endpoint" . }} -{{- $yamlFile = replace "exporters.otlptraces.endpoint.replace" $otelTracesService $yamlFile }} -{{- $otelMetricsService := printf "%s.%s" ( include "sumologic.metadata.name.otelcol.service-metrics" . ) .Release.Namespace }} -{{- $yamlFile = replace "exporters.otlpmetrics.endpoint.replace" $otelMetricsService $yamlFile }} -{{ tpl ($yamlFile | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} +{{ tpl (toYaml .Values.otelagent.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} diff --git a/deploy/helm/sumologic/conf/traces/traces.otelcol.conf.yaml b/deploy/helm/sumologic/conf/traces/traces.otelcol.conf.yaml index c899cb193d..c36af0b9a0 100644 --- a/deploy/helm/sumologic/conf/traces/traces.otelcol.conf.yaml +++ b/deploy/helm/sumologic/conf/traces/traces.otelcol.conf.yaml @@ -1,26 +1 @@ -{{- $yamlFile := toYaml $.Values.otelcol.config }} -{{- $_collector := .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }} -{{- $sourceName := .Values.otelcol.sourceMetadata.sourceName | quote }} -{{- $sourceCategory := .Values.otelcol.sourceMetadata.sourceCategory | quote }} -{{- $sourceCategoryPrefix := .Values.otelcol.sourceMetadata.sourceCategoryPrefix | quote }} -{{- $sourceCategoryReplaceDash := .Values.otelcol.sourceMetadata.sourceCategoryReplaceDash | quote }} -{{- $excludeNamespaceRegex := .Values.otelcol.sourceMetadata.excludeNamespaceRegex | quote }} -{{- $excludePodRegex := .Values.otelcol.sourceMetadata.excludePodRegex | quote }} -{{- $excludeContainerRegex := .Values.otelcol.sourceMetadata.excludeContainerRegex | quote }} -{{- $excludeHostRegex := .Values.otelcol.sourceMetadata.excludeHostRegex | quote }} -{{- $clusterName := include "sumologic.clusterNameReplaceSpaceWithDash" . }} - -{{- $yamlFile := replace "processors.source.collector.replace" $_collector $yamlFile }} -{{- $yamlFile := replace "processors.source.name.replace" $sourceName $yamlFile }} -{{- $yamlFile := replace "processors.source.category.replace" $sourceCategory $yamlFile }} -{{- $yamlFile := replace "processors.source.category_prefix.replace" $sourceCategoryPrefix $yamlFile }} -{{- $yamlFile := replace "processors.source.category_replace_dash.replace" $sourceCategoryReplaceDash $yamlFile }} -{{- $yamlFile := replace "processors.source.exclude_namespace_regex.replace" $excludeNamespaceRegex $yamlFile }} -{{- $yamlFile := replace "processors.source.exclude_pod_regex.replace" $excludePodRegex $yamlFile }} -{{- $yamlFile := replace "processors.source.exclude_container_regex.replace" $excludeContainerRegex $yamlFile }} -{{- $yamlFile := replace "processors.source.exclude_host_regex.replace" $excludeHostRegex $yamlFile }} -{{- $yamlFile := replace "processors.resource.cluster.replace" $clusterName $yamlFile }} -{{- $yamlFile := replace "exporters.sumologic.source_name.replace" $sourceName $yamlFile }} -{{- $yamlFile := replace "exporters.sumologic.source_category.replace" $sourceCategory $yamlFile }} - -{{- tpl ($yamlFile | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} +{{- tpl (toYaml .Values.otelcol.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 001a5898d2..67ce07636d 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -1240,19 +1240,6 @@ Example: {{- print $annotations_match }} {{- end -}} - -{{/* -Return k8s.cluster.name for opentelemetry collector - -Example: - -{{ include "otelcol.k8s.cluster.name" . }} -*/}} -{{- define "otelcol.k8s.cluster.name" -}} -{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }} -{{- end -}} - - {{/* Returns list of namespaces to exclude diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index ba50b4af5a..0f4ebe7c37 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3452,9 +3452,9 @@ otelagent: pprof: {} exporters: otlphttp/traces: - endpoint: 'http://exporters.otlptraces.endpoint.replace:4318' + endpoint: 'http://{{ include "traces.otelagent.exporter.endpoint" . }}:4318' otlphttp/metrics: - endpoint: 'http://exporters.otlpmetrics.endpoint.replace:4318' + endpoint: 'http://{{ printf "%s.%s" ( include "sumologic.metadata.name.otelcol.service-metrics" . ) .Release.Namespace }}:4318' service: extensions: [health_check, memory_ballast, pprof] pipelines: @@ -3569,26 +3569,26 @@ otelcol: ## Source processor adds Sumo Logic related metadata source: annotation_prefix: "k8s.pod.annotation." - collector: "processors.source.collector.replace" + collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' exclude: - k8s.container.name: "processors.source.exclude_container_regex.replace" - k8s.host.name: "processors.source.exclude_host_regex.replace" - k8s.namespace.name: "processors.source.exclude_namespace_regex.replace" - k8s.pod.name: "processors.source.exclude_pod_regex.replace" + k8s.container.name: '{{ .Values.otelcol.sourceMetadata.excludeContainerRegex | quote }}' + k8s.host.name: '{{ .Values.otelcol.sourceMetadata.excludeHostRegex | quote }}' + k8s.namespace.name: '{{ .Values.otelcol.sourceMetadata.excludeNamespaceRegex | quote }}' + k8s.pod.name: '{{ .Values.otelcol.sourceMetadata.excludePodRegex| quote }}' pod_key: "k8s.pod.name" pod_name_key: "k8s.pod.pod_name" pod_template_hash_key: "k8s.pod.label.pod-template-hash" - source_category: "processors.source.category.replace" - source_category_prefix: "processors.source.category_prefix.replace" - source_category_replace_dash: "processors.source.category_replace_dash.replace" + source_category: '{{ .Values.otelcol.sourceMetadata.sourceCategory | quote }}' + source_category_prefix: '{{ .Values.otelcol.sourceMetadata.sourceCategoryPrefix | quote }}' + source_category_replace_dash: '{{ .Values.otelcol.sourceMetadata.sourceCategoryReplaceDash | quote }}' source_host: "%{k8s.pod.hostname}" - source_name: "processors.source.name.replace" + source_name: '{{ .Values.otelcol.sourceMetadata.sourceName | quote }}' ## Resource processor sets the associted cluster attribute resource: attributes: - key: k8s.cluster.name - value: "processors.resource.cluster.replace" + value: '{{ include "sumologic.clusterNameReplaceSpaceWithDash" . }}' action: upsert ## The memory_limiter processor is used to prevent out of memory situations on the collector. @@ -3653,9 +3653,9 @@ otelcol: ## carbon2 and graphite are going to be supported soon. metric_format: prometheus ## Desired source category. Useful if you want to override the source category configured for the source. - source_category: "exporters.sumologic.source_category.replace" + source_category: '{{ .Values.otelcol.sourceMetadata.sourceCategory| quote }}' ## Desired source name. Useful if you want to override the source name configured for the source. - source_name: "exporters.sumologic.source_name.replace" + source_name: '{{ .Values.otelcol.sourceMetadata.sourceName | quote }}' ## Desired host name. Useful if you want to override the source host configured for the source. source_host: "%{k8s.pod.hostname}" ## Timeout for every attempt to send data to Sumo Logic backend. Maximum connection timeout is 55s.