From 95e92b25246b7991b761d61f7bd16e630789f117 Mon Sep 17 00:00:00 2001 From: Sam Song Date: Fri, 30 Aug 2019 11:01:39 -0700 Subject: [PATCH 1/3] migrate plugin code from old repo --- deploy/docker/Dockerfile | 4 +- fluent-plugin-kubernetes-sumologic/Gemfile | 9 + fluent-plugin-kubernetes-sumologic/README.md | 5 + fluent-plugin-kubernetes-sumologic/Rakefile | 13 + ...fluent-plugin-kubernetes-sumologic.gemspec | 31 + .../plugin/filter_kubernetes_sumologic.rb | 231 +++ .../test/helper.rb | 16 + .../test_filter_kubernetes_sumologic.rb | 1665 +++++++++++++++++ 8 files changed, 1972 insertions(+), 2 deletions(-) create mode 100644 fluent-plugin-kubernetes-sumologic/Gemfile create mode 100644 fluent-plugin-kubernetes-sumologic/README.md create mode 100644 fluent-plugin-kubernetes-sumologic/Rakefile create mode 100644 fluent-plugin-kubernetes-sumologic/fluent-plugin-kubernetes-sumologic.gemspec create mode 100644 fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb create mode 100644 fluent-plugin-kubernetes-sumologic/test/helper.rb create mode 100644 fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index db13b722dd..b4dcbc85c5 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -30,11 +30,11 @@ RUN gem install fluent-plugin-systemd -v 1.0.2 \ && gem install fluent-plugin-sumologic_output -v 1.5.0 \ && gem install fluent-plugin-concat -v 2.4.0 \ && gem install fluent-plugin-rewrite-tag-filter -v 2.2.0 \ - && gem install fluent-plugin-prometheus -v 1.5.0 \ - && gem install fluent-plugin-kubernetes_sumologic -v 2.4.2 + && gem install fluent-plugin-prometheus -v 1.5.0 # FluentD plugins from this repository RUN gem install --local fluent-plugin-prometheus-format \ + && gem install --local fluent-plugin-kubernetes-sumologic \ && gem install --local fluent-plugin-enhance-k8s-metadata \ && gem install --local fluent-plugin-datapoint \ && gem install --local fluent-plugin-protobuf \ diff --git a/fluent-plugin-kubernetes-sumologic/Gemfile b/fluent-plugin-kubernetes-sumologic/Gemfile new file mode 100644 index 0000000000..9342b36475 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/Gemfile @@ -0,0 +1,9 @@ +source 'https://rubygems.org' + +group :test do + gem 'codecov' + gem 'simplecov' + gem 'webmock' +end + +gemspec \ No newline at end of file diff --git a/fluent-plugin-kubernetes-sumologic/README.md b/fluent-plugin-kubernetes-sumologic/README.md new file mode 100644 index 0000000000..9d3b3799f1 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/README.md @@ -0,0 +1,5 @@ +# fluent-plugin-kubernetes-sumologic + +[Fluentd](https://fluentd.org/) filter plugin to enrich logs with Sumo Logic specific metadata. + +This README is a WIP. For detailed documentation please go [here](https://github.com/SumoLogic/fluentd-kubernetes-sumologic). \ No newline at end of file diff --git a/fluent-plugin-kubernetes-sumologic/Rakefile b/fluent-plugin-kubernetes-sumologic/Rakefile new file mode 100644 index 0000000000..9756c6eeb6 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/Rakefile @@ -0,0 +1,13 @@ +require "bundler" +Bundler::GemHelper.install_tasks + +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs.push("lib", "test") + t.test_files = FileList["test/**/test_*.rb"] + t.verbose = true + t.warning = true +end + +task default: [:test] diff --git a/fluent-plugin-kubernetes-sumologic/fluent-plugin-kubernetes-sumologic.gemspec b/fluent-plugin-kubernetes-sumologic/fluent-plugin-kubernetes-sumologic.gemspec new file mode 100644 index 0000000000..5bfdf5b969 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/fluent-plugin-kubernetes-sumologic.gemspec @@ -0,0 +1,31 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) + +Gem::Specification.new do |spec| + spec.name = "fluent-plugin-kubernetes-sumologic" + spec.version = "0.0.0" + spec.authors = ["Sumo Logic"] + spec.email = ["collection@sumologic.com"] + spec.description = %q{FluentD plugin to enrich logs with Sumo Logic specific metadata.} + spec.summary = %q{FluentD plugin to enrich logs with Sumo Logic specific metadata.} + spec.homepage = "https://github.com/SumoLogic/sumologic-kubernetes-collection" + spec.license = "Apache-2.0" + + test_files, files = `git ls-files -z`.split("\x0").partition do |f| + f.match(%r{^(test|spec|features)/}) + end + spec.files = files + spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) } + spec.test_files = test_files + spec.require_paths = ["lib"] + + spec.required_ruby_version = '>= 2.0.0' + + spec.add_development_dependency "bundler", "~> 2" + spec.add_development_dependency "rake" + spec.add_development_dependency 'test-unit', '~> 3.1.0' + spec.add_development_dependency "codecov", ">= 0.1.10" + spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"] + spec.add_runtime_dependency 'httpclient', '~> 2.8.0' +end diff --git a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb new file mode 100644 index 0000000000..35afcc5115 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -0,0 +1,231 @@ +require "fluent/filter" + +module Fluent::Plugin + class SumoContainerOutput < Fluent::Plugin::Filter + # Register type + Fluent::Plugin.register_filter("kubernetes_sumologic", self) + + config_param :kubernetes_meta, :bool, :default => true + config_param :kubernetes_meta_reduce, :bool, :default => false + config_param :source_category, :string, :default => "%{namespace}/%{pod_name}" + config_param :source_category_replace_dash, :string, :default => "/" + config_param :source_category_prefix, :string, :default => "kubernetes/" + config_param :source_name, :string, :default => "%{namespace}.%{pod}.%{container}" + config_param :log_format, :string, :default => "json" + config_param :source_host, :string, :default => "" + config_param :exclude_container_regex, :string, :default => "" + config_param :exclude_facility_regex, :string, :default => "" + config_param :exclude_host_regex, :string, :default => "" + config_param :exclude_namespace_regex, :string, :default => "" + config_param :exclude_pod_regex, :string, :default => "" + config_param :exclude_priority_regex, :string, :default => "" + config_param :exclude_unit_regex, :string, :default => "" + config_param :add_stream, :bool, :default => true + config_param :add_time, :bool, :default => true + + def configure(conf) + super + end + + def is_number?(string) + true if Float(string) rescue false + end + + def sanitize_pod_name(k8s_metadata) + # Strip out dynamic bits from pod name. + # NOTE: Kubernetes deployments append a template hash. + # At the moment this can be in 3 different forms: + # 1) pre-1.8: numeric in pod_template_hash and pod_parts[-2] + # 2) 1.8-1.11: numeric in pod_template_hash, hash in pod_parts[-2] + # 3) post-1.11: hash in pod_template_hash and pod_parts[-2] + + pod_parts = k8s_metadata[:pod].split("-") + pod_template_hash = k8s_metadata[:"label:pod-template-hash"] + if (pod_template_hash == pod_parts[-2] || + to_hash(pod_template_hash) == pod_parts[-2]) + k8s_metadata[:pod_name] = pod_parts[0..-3].join("-") + else + k8s_metadata[:pod_name] = pod_parts[0..-2].join("-") + end + end + + def to_hash(pod_template_hash) + # Convert the pod_template_hash to an alphanumeric string using the same logic Kubernetes + # uses at https://github.com/kubernetes/apimachinery/blob/18a5ff3097b4b189511742e39151a153ee16988b/pkg/util/rand/rand.go#L119 + alphanums = "bcdfghjklmnpqrstvwxz2456789" + pod_template_hash.each_byte.map { |i| alphanums[i.to_i % alphanums.length] }.join("") + end + + def filter(tag, time, record) + # Set the sumo metadata fields + sumo_metadata = record["_sumo_metadata"] || {} + record["_sumo_metadata"] = sumo_metadata + log_fields = {} + sumo_metadata[:log_format] = @log_format + sumo_metadata[:host] = @source_host if @source_host + sumo_metadata[:source] = @source_name if @source_name + + unless @source_category.nil? + sumo_metadata[:category] = @source_category.dup + unless @source_category_prefix.nil? + sumo_metadata[:category].prepend(@source_category_prefix) + end + end + + if record.key?("_SYSTEMD_UNIT") and not record.fetch("_SYSTEMD_UNIT").nil? + unless @exclude_unit_regex.empty? + if Regexp.compile(@exclude_unit_regex).match(record["_SYSTEMD_UNIT"]) + return nil + end + end + + unless @exclude_facility_regex.empty? + if Regexp.compile(@exclude_facility_regex).match(record["SYSLOG_FACILITY"]) + return nil + end + end + + unless @exclude_priority_regex.empty? + if Regexp.compile(@exclude_priority_regex).match(record["PRIORITY"]) + return nil + end + end + + unless @exclude_host_regex.empty? + if Regexp.compile(@exclude_host_regex).match(record["_HOSTNAME"]) + return nil + end + end + end + + # Allow fields to be overridden by annotations + if record.key?("kubernetes") and not record.fetch("kubernetes").nil? + # Clone kubernetes hash so we don't override the cache + kubernetes = record["kubernetes"].clone + k8s_metadata = { + :namespace => kubernetes["namespace_name"], + :pod => kubernetes["pod_name"], + :pod_id => kubernetes['pod_id'], + :container => kubernetes["container_name"], + :source_host => kubernetes["host"], + } + + + if kubernetes.has_key? "labels" + kubernetes["labels"].each { |k, v| k8s_metadata["label:#{k}".to_sym] = v } + end + if kubernetes.has_key? "namespace_labels" + kubernetes["namespace_labels"].each { |k, v| k8s_metadata["namespace_label:#{k}".to_sym] = v } + end + k8s_metadata.default = "undefined" + + annotations = kubernetes.fetch("annotations", {}) + if annotations["sumologic.com/include"] == "true" + include = true + else + include = false + end + + unless @exclude_namespace_regex.empty? + if Regexp.compile(@exclude_namespace_regex).match(k8s_metadata[:namespace]) and not include + return nil + end + end + + unless @exclude_pod_regex.empty? + if Regexp.compile(@exclude_pod_regex).match(k8s_metadata[:pod]) and not include + return nil + end + end + + unless @exclude_container_regex.empty? + if Regexp.compile(@exclude_container_regex).match(k8s_metadata[:container]) and not include + return nil + end + end + + unless @exclude_host_regex.empty? + if Regexp.compile(@exclude_host_regex).match(k8s_metadata[:source_host]) and not include + return nil + end + end + + sanitize_pod_name(k8s_metadata) + + if annotations["sumologic.com/exclude"] == "true" + return nil + end + + sumo_metadata[:log_format] = annotations["sumologic.com/format"] if annotations["sumologic.com/format"] + + if annotations["sumologic.com/sourceHost"].nil? + sumo_metadata[:host] = sumo_metadata[:host] % k8s_metadata + else + sumo_metadata[:host] = annotations["sumologic.com/sourceHost"] % k8s_metadata + end + + if annotations["sumologic.com/sourceName"].nil? + sumo_metadata[:source] = sumo_metadata[:source] % k8s_metadata + else + sumo_metadata[:source] = annotations["sumologic.com/sourceName"] % k8s_metadata + end + + if annotations["sumologic.com/sourceCategory"].nil? + sumo_metadata[:category] = sumo_metadata[:category] % k8s_metadata + else + sumo_metadata[:category] = (annotations["sumologic.com/sourceCategory"] % k8s_metadata).prepend(@source_category_prefix) + end + sumo_metadata[:category].gsub!("-", @source_category_replace_dash) + + # Strip kubernetes metadata from json if disabled + if annotations["sumologic.com/kubernetes_meta"] == "false" || !@kubernetes_meta + record.delete("docker") + record.delete("kubernetes") + end + if annotations["sumologic.com/kubernetes_meta_reduce"] == "true" || annotations["sumologic.com/kubernetes_meta_reduce"].nil? && @kubernetes_meta_reduce == true + record.delete("docker") + record["kubernetes"].delete("pod_id") + record["kubernetes"].delete("namespace_id") + record["kubernetes"].delete("labels") + record["kubernetes"].delete("namespace_labels") + record["kubernetes"].delete("master_url") + record["kubernetes"].delete("annotations") + end + if @add_stream == false + record.delete("stream") + end + if @add_time == false + record.delete("time") + end + # Strip sumologic.com annotations + kubernetes.delete("annotations") if annotations + + if @log_format == "fields" and record.key?("docker") and not record.fetch("docker").nil? + record["docker"].each {|k, v| log_fields[k] = v} + end + + if @log_format == "fields" and record.key?("kubernetes") and not record.fetch("kubernetes").nil? + if kubernetes.has_key? "labels" + kubernetes["labels"].each { |k, v| log_fields["pod_labels_#{k}".to_sym] = v } + end + if kubernetes.has_key? "namespace_labels" + kubernetes["namespace_labels"].each { |k, v| log_fields["namespace_labels_#{k}".to_sym] = v } + end + log_fields["container"] = kubernetes["container_name"] unless kubernetes["container_name"].nil? + log_fields["namespace"] = kubernetes["namespace_name"] unless kubernetes["namespace_name"].nil? + log_fields["pod"] = kubernetes["pod_name"] unless kubernetes["pod_name"].nil? + ["pod_id", "host", "master_url", "namespace_id", "service", "deployment", "daemonset", "replicaset", "statefulset"].each do |key| + log_fields[key] = kubernetes[key] unless kubernetes[key].nil? + end + end + end + + if @log_format == "fields" and not log_fields.nil? + sumo_metadata[:fields] = log_fields.map{|k,v| "#{k}=#{v}"}.join(',') + record.delete("docker") + record.delete("kubernetes") + end + record + end + end +end \ No newline at end of file diff --git a/fluent-plugin-kubernetes-sumologic/test/helper.rb b/fluent-plugin-kubernetes-sumologic/test/helper.rb new file mode 100644 index 0000000000..5a12ee9c08 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/test/helper.rb @@ -0,0 +1,16 @@ +require "simplecov" +SimpleCov.start + +if ENV["CI"] == "true" + require "codecov" + SimpleCov.formatter = SimpleCov::Formatter::Codecov +end + +$LOAD_PATH.unshift(File.expand_path("../../", __FILE__)) +require "test-unit" +require "fluent/test" +require "fluent/test/driver/filter" +require "fluent/test/helpers" + +Test::Unit::TestCase.include(Fluent::Test::Helpers) +Test::Unit::TestCase.extend(Fluent::Test::Helpers) diff --git a/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb new file mode 100644 index 0000000000..735b79cd56 --- /dev/null +++ b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb @@ -0,0 +1,1665 @@ +require "fluent/test" +require "fluent/test/helpers" +require "fluent/test/driver/filter" +require "fluent/plugin/filter_kubernetes_sumologic" +require "test-unit" +require "webmock/test_unit" + +class SumoContainerOutputTest < Test::Unit::TestCase + include Fluent::Test::Helpers + + setup do + Fluent::Test.setup + @time = Fluent::Engine.now + end + + def create_driver(conf = CONFIG) + Fluent::Test::Driver::Filter.new(Fluent::Plugin::SumoContainerOutput).configure(conf) + end + + test "test_empty_config" do + conf = %{} + assert_nothing_raised do + create_driver(conf) + end + end + + test "test_default_config" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_default_config_no_labels" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/54575ccdb9", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_fields_format" do + conf = %{ + log_format fields + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "fields", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_fields_namespace_labels" do + conf = %{ + log_format fields + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "namespace_labels" => { + "app" => "sumologic" + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "fields", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,namespace_labels_app=sumologic,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_fields_format_no_timestamp" do + conf = %{ + log_format fields + add_stream false + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "fields", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_no_k8s_labels" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/54575ccdb9", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcecategory_prefix" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_add_stream" do + conf = %{ + add_stream false + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_add_time" do + conf = %{ + add_time false + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "time" => time, + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcecategory_replace_dash" do + conf = %{ + source_category_replace_dash - + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log-format-labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_kubernetes_meta" do + conf = %{ + kubernetes_meta false + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_kubernetes_meta_reduce_via_annotation" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs" + }, + "annotations" => { + "sumologic.com/kubernetes_meta_reduce" => "true", + }, + "host" => "docker-for-desktop", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "kubernetes" => { + "container_name" => "log-format-labs", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "host" => "docker-for-desktop", + "namespace_name" => "default", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_kubernetes_meta_reduce_via_conf" do + conf = %{ + kubernetes_meta_reduce true + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs" + }, + "host" => "docker-for-desktop", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "kubernetes" => { + "container_name" => "log-format-labs", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "host" => "docker-for-desktop", + "namespace_name" => "default", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_kubernetes_meta_reduce_via_annotation_and_conf" do + conf = %{ + kubernetes_meta_reduce false + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs" + }, + "annotations" => { + "sumologic.com/kubernetes_meta_reduce" => "true", + }, + "host" => "docker-for-desktop", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "kubernetes" => { + "container_name" => "log-format-labs", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "host" => "docker-for-desktop", + "namespace_name" => "default", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_log_format_json_merge" do + conf = %{ + log_format json_merge + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json_merge", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_log_format_text" do + conf = %{ + log_format text + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "text", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_exclude_pod_regex" do + conf = %{ + exclude_pod_regex foo + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_pod_regex_whitelist" do + conf = %{ + exclude_pod_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_container_regex" do + conf = %{ + exclude_container_regex foo + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_container_regex_whitelist" do + conf = %{ + exclude_container_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_namespace_regex" do + conf = %{ + exclude_namespace_regex foo + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "foo", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "bar", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_namespace_regex_whitelist" do + conf = %{ + exclude_namespace_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_host_regex" do + conf = %{ + exclude_host_regex foo + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "foo", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "bar", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_host_regex_whitelist" do + conf = %{ + exclude_host_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "false"}}, "message" => "foo"}) + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "5679EFGH", "pod_name" => "bar-6554321-a87f", "container_name" => "bar", "labels" => {"app" => "bar"}, "host" => "localhost", "annotations" => {"sumologic.com/include" => "true"}}, "message" => "foo"}) + end + assert_equal(1, d.filtered_records.size) + end + + test "test_exclude_annotation" do + conf = %{ + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost", "annotations" => {"sumologic.com/exclude" => "true"}}, "message" => "foo"}) + end + assert_equal(0, d.filtered_records.size) + end + + test "test_sourcehost_annotation" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceHost" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceHost" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "foo", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcename_annotation" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceName" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceName" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "foo", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcecategory_annotation" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceCategory" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "annotations" => { + "sumologic.com/sourceCategory" => "foo", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/foo", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcecategory_using_labels" do + conf = %{ + source_category %{namespace}/%{pod_name}/%{label:run} + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_sourcehost_using_pod_id" do + conf = %{ + source_host %{pod_id} + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "170af806-c801-11e8-9009-025000000001", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_undefined_labels" do + conf = %{ + source_category %{namespace}/%{pod_name}/%{label:foo} + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/undefined", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_exclude_systemd_unit_regex" do + conf = %{ + exclude_unit_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"_SYSTEMD_UNIT" => "test", "kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost"}, "message" => "foo"}) + end + assert_equal(0, d.filtered_records.size) + end + + test "test_exclude_systemd_facility_regex" do + conf = %{ + exclude_facility_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"_SYSTEMD_UNIT" => "test", "SYSLOG_FACILITY" => "test", "kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost"}, "message" => "foo"}) + end + assert_equal(0, d.filtered_records.size) + end + + test "test_exclude_systemd_priority_regex" do + conf = %{ + exclude_priority_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"_SYSTEMD_UNIT" => "test", "PRIORITY" => "test", "kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost"}, "message" => "foo"}) + end + assert_equal(0, d.filtered_records.size) + end + + test "test_exclude_systemd_hostname_regex" do + conf = %{ + exclude_host_regex .* + } + d = create_driver(conf) + time = @time + d.run do + d.feed("filter.test", time, {"_SYSTEMD_UNIT" => "test", "_HOSTNAME" => "test", "kubernetes" => {"namespace_name" => "test", "pod_id" => "1234ABCD", "pod_name" => "foo-1234556-f87a", "container_name" => "foo", "labels" => {"app" => "foo"}, "host" => "localhost"}, "message" => "foo"}) + end + assert_equal(0, d.filtered_records.size) + end + + test "test_pre_1.8_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-1013177865-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-1013177865-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-1013177865-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_1.8-1.11_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_post_1.11_dynamic_bit_removal" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + + test "test_mismatch_dynamic_bit_is_left" do + conf = %{} + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-53575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-53575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "54575ccdb9", + "run" => "log-format-labs", + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs/53575ccdb9", + :host => "", + :log_format => "json", + :source => "default.log-format-labs-53575ccdb9-9d677.log-format-labs", + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end +end \ No newline at end of file From 847c0d24dfc71cefe42ed1bccc5a39f4b852b006 Mon Sep 17 00:00:00 2001 From: Sam Song Date: Fri, 30 Aug 2019 13:06:41 -0700 Subject: [PATCH 2/3] add new 'node' field to metadata --- .../lib/fluent/plugin/filter_kubernetes_sumologic.rb | 1 + .../test/plugin/test_filter_kubernetes_sumologic.rb | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb index 35afcc5115..e340e0a302 100644 --- a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -217,6 +217,7 @@ def filter(tag, time, record) ["pod_id", "host", "master_url", "namespace_id", "service", "deployment", "daemonset", "replicaset", "statefulset"].each do |key| log_fields[key] = kubernetes[key] unless kubernetes[key].nil? end + log_fields["node"] = kubernetes["host"] unless kubernetes["host"].nil? end end diff --git a/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb index 735b79cd56..36e28fc0da 100644 --- a/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb @@ -173,7 +173,7 @@ def create_driver(conf = CONFIG) :host => "", :log_format => "fields", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", - :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001,node=docker-for-desktop" }, } assert_equal(1, d.filtered_records.size) @@ -222,7 +222,7 @@ def create_driver(conf = CONFIG) :host => "", :log_format => "fields", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", - :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,namespace_labels_app=sumologic,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,namespace_labels_app=sumologic,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001,node=docker-for-desktop" }, } assert_equal(1, d.filtered_records.size) @@ -268,7 +268,7 @@ def create_driver(conf = CONFIG) :host => "", :log_format => "fields", :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", - :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001" + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001,node=docker-for-desktop" }, } assert_equal(1, d.filtered_records.size) From 6a43affa59b77227b8ed5e61f3768788fc21a12c Mon Sep 17 00:00:00 2001 From: Sam Song Date: Fri, 30 Aug 2019 13:25:35 -0700 Subject: [PATCH 3/3] move null field dropping to kubernetes_sumologic plugin --- .../plugin/filter_enhance_k8s_metadata.rb | 2 - .../plugin/filter_kubernetes_sumologic.rb | 2 +- .../test_filter_kubernetes_sumologic.rb | 51 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb b/fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb index 95a1ef3d09..2cb5ff0fc7 100644 --- a/fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb +++ b/fluent-plugin-enhance-k8s-metadata/lib/fluent/plugin/filter_enhance_k8s_metadata.rb @@ -93,8 +93,6 @@ def decorate_record(record) end end end - - record['kubernetes']['labels'].select!{|k,v| !(v.nil? || v.empty?)} if @data_type == 'logs' end end diff --git a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb index e340e0a302..68f512807c 100644 --- a/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb @@ -222,7 +222,7 @@ def filter(tag, time, record) end if @log_format == "fields" and not log_fields.nil? - sumo_metadata[:fields] = log_fields.map{|k,v| "#{k}=#{v}"}.join(',') + sumo_metadata[:fields] = log_fields.select{|k,v| !(v.nil? || v.empty?)}.map{|k,v| "#{k}=#{v}"}.join(',') record.delete("docker") record.delete("kubernetes") end diff --git a/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb index 36e28fc0da..f1c88505b8 100644 --- a/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb +++ b/fluent-plugin-kubernetes-sumologic/test/plugin/test_filter_kubernetes_sumologic.rb @@ -229,6 +229,57 @@ def create_driver(conf = CONFIG) assert_equal(d.filtered_records[0], expected) end + test "test_fields_null_field_values" do + conf = %{ + log_format fields + } + d = create_driver(conf) + time = @time + input = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "docker" => { + "container_id" => "5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0", + }, + "kubernetes" => { + "container_name" => "log-format-labs", + "namespace_name" => "default", + "pod_name" => "log-format-labs-54575ccdb9-9d677", + "pod_id" => "170af806-c801-11e8-9009-025000000001", + "labels" => { + "pod-template-hash" => "1013177865", + "empty-value" => "", + "run" => "log-format-labs", + "null-value" => nil, + }, + "namespace_labels" => { + "app" => "sumologic" + }, + "host" => "docker-for-desktop", + "master_url" => "https =>//10.96.0.1 =>443/api", + "namespace_id" => "e8572415-9596-11e8-b28b-025000000001", + }, + } + d.run do + d.feed("filter.test", time, input) + end + expected = { + "timestamp" => 1538677347823, + "log" => "some message", + "stream" => "stdout", + "_sumo_metadata" => { + :category => "kubernetes/default/log/format/labs", + :host => "", + :log_format => "fields", + :source => "default.log-format-labs-54575ccdb9-9d677.log-format-labs", + :fields => "container_id=5c280b6ad5abec32e9af729295c20f60fbeadf3ba16fda2d121f87228e6822e0,pod_labels_pod-template-hash=1013177865,pod_labels_run=log-format-labs,namespace_labels_app=sumologic,container=log-format-labs,namespace=default,pod=log-format-labs-54575ccdb9-9d677,pod_id=170af806-c801-11e8-9009-025000000001,host=docker-for-desktop,master_url=https =>//10.96.0.1 =>443/api,namespace_id=e8572415-9596-11e8-b28b-025000000001,node=docker-for-desktop" + }, + } + assert_equal(1, d.filtered_records.size) + assert_equal(d.filtered_records[0], expected) + end + test "test_fields_format_no_timestamp" do conf = %{ log_format fields