From 571e05627d42fc32dc28319fcee79643a7f59046 Mon Sep 17 00:00:00 2001 From: Sam Song Date: Wed, 9 Oct 2019 02:28:44 -0700 Subject: [PATCH 1/2] cleanup some of filter plugin code --- .../plugin/filter_kubernetes_sumologic.rb | 93 +++++++------------ 1 file changed, 34 insertions(+), 59 deletions(-) diff --git a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb index 68f512807c..3d3ab6e9d0 100644 --- a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -57,51 +57,44 @@ def to_hash(pod_template_hash) end def filter(tag, time, record) + log_fields = {} + # Set the sumo metadata fields sumo_metadata = record["_sumo_metadata"] || {} record["_sumo_metadata"] = sumo_metadata - log_fields = {} sumo_metadata[:log_format] = @log_format sumo_metadata[:host] = @source_host if @source_host sumo_metadata[:source] = @source_name if @source_name - unless @source_category.nil? sumo_metadata[:category] = @source_category.dup unless @source_category_prefix.nil? sumo_metadata[:category].prepend(@source_category_prefix) end end + sumo_metadata[:category].gsub!("-", @source_category_replace_dash) + # Check systemd exclude filters if record.key?("_SYSTEMD_UNIT") and not record.fetch("_SYSTEMD_UNIT").nil? unless @exclude_unit_regex.empty? - if Regexp.compile(@exclude_unit_regex).match(record["_SYSTEMD_UNIT"]) - return nil - end + return nil if Regexp.compile(@exclude_unit_regex).match(record["_SYSTEMD_UNIT"]) end - unless @exclude_facility_regex.empty? - if Regexp.compile(@exclude_facility_regex).match(record["SYSLOG_FACILITY"]) - return nil - end + return nil if Regexp.compile(@exclude_facility_regex).match(record["SYSLOG_FACILITY"]) end - unless @exclude_priority_regex.empty? - if Regexp.compile(@exclude_priority_regex).match(record["PRIORITY"]) - return nil - end + return nil if Regexp.compile(@exclude_priority_regex).match(record["PRIORITY"]) end - unless @exclude_host_regex.empty? - if Regexp.compile(@exclude_host_regex).match(record["_HOSTNAME"]) - return nil - end + return nil if Regexp.compile(@exclude_host_regex).match(record["_HOSTNAME"]) end end - # Allow fields to be overridden by annotations if record.key?("kubernetes") and not record.fetch("kubernetes").nil? # Clone kubernetes hash so we don't override the cache + # Note (sam 10/9/19): this is a shallow copy; nested hashes can still be overriden kubernetes = record["kubernetes"].clone + + # Populate k8s_metadata to use later in sumo_metadata k8s_metadata = { :namespace => kubernetes["namespace_name"], :pod => kubernetes["pod_name"], @@ -109,8 +102,6 @@ def filter(tag, time, record) :container => kubernetes["container_name"], :source_host => kubernetes["host"], } - - if kubernetes.has_key? "labels" kubernetes["labels"].each { |k, v| k8s_metadata["label:#{k}".to_sym] = v } end @@ -119,34 +110,22 @@ def filter(tag, time, record) end k8s_metadata.default = "undefined" + # Fetch annotations for config annotations = kubernetes.fetch("annotations", {}) - if annotations["sumologic.com/include"] == "true" - include = true - else - include = false - end - unless @exclude_namespace_regex.empty? - if Regexp.compile(@exclude_namespace_regex).match(k8s_metadata[:namespace]) and not include - return nil + unless annotations["sumologic.com/include"] == "true" + # Check kubernetes exclude filters + unless @exclude_namespace_regex.empty? + return nil if Regexp.compile(@exclude_namespace_regex).match(k8s_metadata[:namespace]) end - end - - unless @exclude_pod_regex.empty? - if Regexp.compile(@exclude_pod_regex).match(k8s_metadata[:pod]) and not include - return nil + unless @exclude_pod_regex.empty? + return nil if Regexp.compile(@exclude_pod_regex).match(k8s_metadata[:pod]) end - end - - unless @exclude_container_regex.empty? - if Regexp.compile(@exclude_container_regex).match(k8s_metadata[:container]) and not include - return nil + unless @exclude_container_regex.empty? + return nil if Regexp.compile(@exclude_container_regex).match(k8s_metadata[:container]) end - end - - unless @exclude_host_regex.empty? - if Regexp.compile(@exclude_host_regex).match(k8s_metadata[:source_host]) and not include - return nil + unless @exclude_host_regex.empty? + return nil if Regexp.compile(@exclude_host_regex).match(k8s_metadata[:source_host]) end end @@ -158,23 +137,18 @@ def filter(tag, time, record) sumo_metadata[:log_format] = annotations["sumologic.com/format"] if annotations["sumologic.com/format"] - if annotations["sumologic.com/sourceHost"].nil? - sumo_metadata[:host] = sumo_metadata[:host] % k8s_metadata - else - sumo_metadata[:host] = annotations["sumologic.com/sourceHost"] % k8s_metadata + unless annotations["sumologic.com/sourceHost"].nil? + sumo_metadata[:host] = annotations["sumologic.com/sourceHost"] end - - if annotations["sumologic.com/sourceName"].nil? - sumo_metadata[:source] = sumo_metadata[:source] % k8s_metadata - else - sumo_metadata[:source] = annotations["sumologic.com/sourceName"] % k8s_metadata + unless annotations["sumologic.com/sourceName"].nil? + sumo_metadata[:source] = annotations["sumologic.com/sourceName"] end - - if annotations["sumologic.com/sourceCategory"].nil? - sumo_metadata[:category] = sumo_metadata[:category] % k8s_metadata - else - sumo_metadata[:category] = (annotations["sumologic.com/sourceCategory"] % k8s_metadata).prepend(@source_category_prefix) + unless annotations["sumologic.com/sourceCategory"].nil? + sumo_metadata[:category] = annotations["sumologic.com/sourceCategory"].dup.prepend(@source_category_prefix) end + sumo_metadata[:host] = sumo_metadata[:host] % k8s_metadata + sumo_metadata[:source] = sumo_metadata[:source] % k8s_metadata + sumo_metadata[:category] = sumo_metadata[:category] % k8s_metadata sumo_metadata[:category].gsub!("-", @source_category_replace_dash) # Strip kubernetes metadata from json if disabled @@ -198,10 +172,12 @@ def filter(tag, time, record) record.delete("time") end # Strip sumologic.com annotations + # Note (sam 10/9/19): we're stripping from the copy, so this has no affect on output kubernetes.delete("annotations") if annotations if @log_format == "fields" and record.key?("docker") and not record.fetch("docker").nil? record["docker"].each {|k, v| log_fields[k] = v} + record.delete("docker") end if @log_format == "fields" and record.key?("kubernetes") and not record.fetch("kubernetes").nil? @@ -218,13 +194,12 @@ def filter(tag, time, record) log_fields[key] = kubernetes[key] unless kubernetes[key].nil? end log_fields["node"] = kubernetes["host"] unless kubernetes["host"].nil? + record.delete("kubernetes") end end if @log_format == "fields" and not log_fields.nil? sumo_metadata[:fields] = log_fields.select{|k,v| !(v.nil? || v.empty?)}.map{|k,v| "#{k}=#{v}"}.join(',') - record.delete("docker") - record.delete("kubernetes") end record end From 64ef90b74dfdb693acaa2e9d84dc32ffc1b8b1b4 Mon Sep 17 00:00:00 2001 From: Sam Song Date: Fri, 22 Nov 2019 14:42:44 -0800 Subject: [PATCH 2/2] fix typo --- .../lib/fluent/plugin/filter_kubernetes_sumologic.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb index 3d3ab6e9d0..06669ce5f0 100644 --- a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -172,7 +172,7 @@ def filter(tag, time, record) record.delete("time") end # Strip sumologic.com annotations - # Note (sam 10/9/19): we're stripping from the copy, so this has no affect on output + # Note (sam 10/9/19): we're stripping from the copy, so this has no effect on output kubernetes.delete("annotations") if annotations if @log_format == "fields" and record.key?("docker") and not record.fetch("docker").nil?