diff --git a/deploy/docs/Best_Practices.md b/deploy/docs/Best_Practices.md index f6833427a6..f5125df4bc 100644 --- a/deploy/docs/Best_Practices.md +++ b/deploy/docs/Best_Practices.md @@ -16,6 +16,7 @@ - [Missing labels](#missing-labels) - [Configure Ignore_Older Config for Fluentbit](#configure-ignore_older-config-for-fluentbit) - [Disable logs, metrics, or falco](#disable-logs-metrics-or-falco) +- [Load Balancing Prometheus traffic between Fluentds](#load-balancing-prometheus-traffic-between-fluentds) ## Multiline Log Support @@ -475,3 +476,25 @@ respectively in the `values.yaml` file and run the `helm upgrade` command. | `sumologic.logs.enabled` | false | disable logs collection | | `sumologic.metrics.enabled` | false | disable metrics collection | | `falco.enabled` | false | disable falco | + +## Load Balancing Prometheus traffic between Fluentds + +Equal utilization of the Fluentd pods is important for collection process. +If Fluentd pod is under high pressure, incoming connections can be handled with some delay. +To avoid backpressure, `remote_timeout` configuration options for Prometheus' `remote_write` can be used. +By default this is `30s`, which means that Prometheus is going to wait such amount of time +for connection to specific fluentd before trying to reach another. +This significantly decreases performance and can lead to the Prometheus memory issues, +so we decided to override it with `5s`. + +```yaml +kube-prometheus-stack: + prometheus: + prometheusSpec: + additionalScrapeConfigs: + - #... + remoteTimeout: 5s +``` + +**NOTE** We observed that changing this value increases metrics loss during prometheus resharding, +but the traffic is much better balanced between Fluentds and Prometheus is more stable in terms of memory. diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index b9d64aa505..9b50b820ca 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -1413,6 +1413,7 @@ kube-prometheus-stack: ## kube_hpa_status_current_replicas ## kube_hpa_status_desired_replicas - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_hpa_spec_max_replicas|kube_hpa_spec_min_replicas|kube_hpa_status_current_replicas|kube_hpa_status_desired_replicas) @@ -1432,6 +1433,7 @@ kube-prometheus-stack: ## kube_pod_container_status_waiting_reason ## kube_pod_status_phase - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.state + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kube-state-metrics;(?:kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase) @@ -1443,6 +1445,7 @@ kube-prometheus-stack: ## cloudprovider_aws_api_request_duration_seconds_count ## cloudprovider_aws_api_request_duration_seconds_sum - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.controller-manager + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kubelet;cloudprovider_.*_api_request_duration_seconds.* @@ -1458,6 +1461,7 @@ kube-prometheus-stack: ## scheduler_scheduling_algorithm_duration_seconds_bucket ## scheduler_scheduling_algorithm_duration_seconds_count - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.scheduler + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kube-scheduler;scheduler_(?:e2e_scheduling|binding|scheduling_algorithm)_duration_seconds.* @@ -1485,6 +1489,7 @@ kube-prometheus-stack: ## etcd_helper_cache_miss_count ## etcd_helper_cache_miss_total - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.apiserver + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: apiserver;(?:apiserver_request_(?:count|total)|apiserver_request_(?:duration_seconds|latencies)_(?:count|sum)|apiserver_request_latencies_summary(?:|_count|_sum)|etcd_request_cache_(?:add|get)_(?:duration_seconds|latencies_summary)_(?:count|sum)|etcd_helper_cache_(?:hit|miss)_(?:count|total)) @@ -1507,6 +1512,7 @@ kube-prometheus-stack: ## kubelet_runtime_operations_latency_microseconds_count ## kubelet_runtime_operations_latency_microseconds_sum - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.kubelet + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kubelet;(?:kubelet_docker_operations_errors(?:|_total)|kubelet_(?:docker|runtime)_operations_duration_seconds_(?:count|sum)|kubelet_running_(?:container|pod)(?:_count|s)|kubelet_(:?docker|runtime)_operations_latency_microseconds(?:|_count|_sum)) @@ -1518,6 +1524,7 @@ kube-prometheus-stack: ## container_memory_working_set_bytes ## container_cpu_cfs_throttled_seconds_total - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container + remoteTimeout: 5s writeRelabelConfigs: - action: labelmap regex: container_name @@ -1532,6 +1539,7 @@ kube-prometheus-stack: ## container_network_receive_bytes_total ## container_network_transmit_bytes_total - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.container + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total) @@ -1544,6 +1552,7 @@ kube-prometheus-stack: ## node_load5 ## node_load15 - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.node + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: node-exporter;(?:node_load1|node_load5|node_load15|node_cpu_seconds_total) @@ -1584,6 +1593,7 @@ kube-prometheus-stack: ## node:node_num_cpu:sum ## node_namespace_pod:kube_pod_info: - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.operator.rule + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: 'cluster_quantile:apiserver_request_duration_seconds:histogram_quantile|instance:node_filesystem_usage:sum|instance:node_network_receive_bytes:rate:sum|cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile|cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile|cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile|node_namespace_pod:kube_pod_info:|:kube_pod_info_node_count:|node:node_num_cpu:sum|:node_cpu_utilisation:avg1m|node:node_cpu_utilisation:avg1m|node:cluster_cpu_utilisation:ratio|:node_cpu_saturation_load1:|node:node_cpu_saturation_load1:|:node_memory_utilisation:|node:node_memory_bytes_total:sum|node:node_memory_utilisation:ratio|node:cluster_memory_utilisation:ratio|:node_memory_swap_io_bytes:sum_rate|node:node_memory_utilisation:|node:node_memory_utilisation_2:|node:node_memory_swap_io_bytes:sum_rate|:node_disk_utilisation:avg_irate|node:node_disk_utilisation:avg_irate|:node_disk_saturation:avg_irate|node:node_disk_saturation:avg_irate|node:node_filesystem_usage:|node:node_filesystem_avail:|:node_net_utilisation:sum_irate|node:node_net_utilisation:sum_irate|:node_net_saturation:sum_irate|node:node_net_saturation:sum_irate|node:node_inodes_total:|node:node_inodes_free:' @@ -1646,6 +1656,7 @@ kube-prometheus-stack: ## prometheus_remote_storage_succeeded_samples_total ## up - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: (?:up|prometheus_remote_storage_.*|fluentd_.*|fluentbit.*|otelcol.*) @@ -1664,6 +1675,7 @@ kube-prometheus-stack: ## process_open_fds ## process_resident_memory_bytes - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.control-plane.coredns + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: coredns;(?:coredns_cache_(size|(hits|misses)_total)|coredns_dns_request_duration_seconds_(count|sum)|coredns_(dns_request|dns_response_rcode|forward_request)_count_total|process_(cpu_seconds_total|open_fds|resident_memory_bytes)) @@ -1688,6 +1700,7 @@ kube-prometheus-stack: ## process_open_fds ## process_resident_memory_bytes - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.control-plane.kube-etcd + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: kube-etcd;(?:etcd_debugging_(mvcc_db_total_size_in_bytes|store_(expires_total|watchers))|etcd_disk_(backend_commit|wal_fsync)_duration_seconds_bucket|etcd_grpc_proxy_cache_(hits|misses)_total|etcd_network_client_grpc_(received|sent)_bytes_total|etcd_server_(has_leader|leader_changes_seen_total)|etcd_server_proposals_(pending|(applied|committed|failed)_total)|process_(cpu_seconds_total|open_fds|resident_memory_bytes)) @@ -1710,6 +1723,7 @@ kube-prometheus-stack: ## nginx_ingress_nginx_connections_writing ## nginx_ingress_nginx_http_requests_total - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.applications.nginx-ingress + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: (?:nginx_ingress_controller_ingress_resources_total|nginx_ingress_controller_nginx_(last_reload_(milliseconds|status)|reload(s|_errors)_total)|nginx_ingress_controller_virtualserver(|route)_resources_total|nginx_ingress_nginx_connections_(accepted|active|handled|reading|waiting|writing)|nginx_ingress_nginx_http_requests_total) @@ -1724,6 +1738,7 @@ kube-prometheus-stack: ## nginx_waiting ## nginx_writing - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.applications.nginx + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: (?:nginx_(accepts|active|handled|reading|requests|waiting|writing)) @@ -1760,6 +1775,7 @@ kube-prometheus-stack: ## redis_used_memory_rss ## redis_used_memory_startup - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.applications.redis + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: (?:redis_((blocked_|)clients|cluster_enabled|cmdstat_calls|connected_slaves|(evicted|expired|tracking_total)_keys|instantaneous_ops_per_sec|keyspace_(hitrate|hits|misses)|(master|slave)_repl_offset|maxmemory|mem_fragmentation_(bytes|ratio)|rdb_changes_since_last_save|rejected_connections|total_commands_processed|total_net_(input|output)_bytes|uptime|used_(cpu_(sys|user)|memory(_overhead|_rss|_startup|)))) @@ -1827,6 +1843,7 @@ kube-prometheus-stack: ## java_lang_Threading_ThreadCpuTime* ## java_lang_Threading_TotalStartedThreadCount - url: http://$(CHART).$(NAMESPACE).svc.cluster.local:9888/prometheus.metrics.applications.jmx + remoteTimeout: 5s writeRelabelConfigs: - action: keep regex: (?:java_lang_(ClassLoading_(TotalL|Unl|L)oadedClassCount|Compilation_TotalCompilationTime|GarbageCollector_(Collection(Count|Time)|LastGcInfo_(GcThreadCount|duration|(memoryU|u)sage(After|Before)Gc_.*_used))|MemoryPool_(CollectionUsage(ThresholdSupported|_committed|_max|_used)|(Peak|)Usage_(committed|max|used)|UsageThresholdSupported)|Memory_((Non|)HeapMemoryUsage_(committed|max|used)|ObjectPendingFinalizationCount)|OperatingSystem_(AvailableProcessors|(CommittedVirtual|(Free|Total)(Physical|))MemorySize|(Free|Total)SwapSpaceSize|(Max|Open)FileDescriptorCount|ProcessCpu(Load|Time)|System(CpuLoad|LoadAverage))|Runtime_(BootClassPathSupported|Pid|Uptime|StartTime)|Threading_(CurrentThread(AllocatedBytes|(Cpu|User)Time)|(Daemon|Peak|TotalStarted|)ThreadCount|(ObjectMonitor|Synchronizer)UsageSupported|Thread(AllocatedMemory.*|ContentionMonitoring.*|CpuTime.*))))