Collect logs with FluentBit and Fluentd

deploy/docker/Dockerfile

@@ -15,6 +15,16 @@ RUN apk add --no-cache --update --virtual .build-deps sudo build-base ruby-dev \
        && gem install lru_redux \
        && gem install snappy
 
+RUN gem install fluent-plugin-s3 -v 1.1.4 \
+       && gem install fluent-plugin-systemd -v 0.3.1 \
+       && gem install fluent-plugin-record-modifier \
+       && gem install fluent-plugin-kubernetes_metadata_filter -v 1.0.2 \
+       && gem install fluent-plugin-sumologic_output -v 1.4.0 \
+       && gem install fluent-plugin-concat -v 2.3.0 \
+       && gem install fluent-plugin-rewrite-tag-filter -v 2.1.0 \
+       && gem install fluent-plugin-prometheus -v 1.1.0 \
+       && gem install fluent-plugin-kubernetes_sumologic
+
 RUN gem install fluent-plugin-sumologic_output \
        && gem install fluent-plugin-carbon-v2 \
        && gem install fluent-plugin-prometheus-format \
@@ -28,6 +38,28 @@ RUN gem sources --clear-all \
        && rm -rf /home/fluent/.gem/ruby/2.5.0/cache/*.gem \
        && rm -f ./*.gem
 
+# Default settings for log collection
+ENV LOG_FORMAT "json"
+ENV FLUSH_INTERVAL "5s"
+ENV NUM_THREADS "1"
+ENV SOURCE_CATEGORY "%{namespace}/%{pod_name}"
+ENV SOURCE_CATEGORY_PREFIX "kubernetes/"
+ENV SOURCE_CATEGORY_REPLACE_DASH "/"
+ENV SOURCE_NAME "%{namespace}.%{pod}.%{container}"
+ENV KUBERNETES_META "true"
+ENV KUBERNETES_META_REDUCE "false"
+ENV MULTILINE_START_REGEXP "/^\w{3} \d{1,2}, \d{4}/"
+ENV CONCAT_SEPARATOR ""
+ENV ADD_TIMESTAMP "true"
+ENV TIMESTAMP_KEY "timestamp"
+ENV ADD_STREAM "true"
+ENV ADD_TIME "true"
+ENV K8S_METADATA_FILTER_WATCH "true"
+ENV K8S_METADATA_FILTER_VERIFY_SSL "true"
+ENV K8S_METADATA_FILTER_BEARER_CACHE_SIZE "1000"
+ENV K8S_METADATA_FILTER_BEARER_CACHE_TTL "3600"
+ENV VERIFY_SSL "true"
+
 RUN mkdir -p /fluentd/conf.d
 
 COPY ./fluent.conf /fluentd/conf.d/fluent.conf

deploy/docker/fluent.conf

@@ -23,6 +23,12 @@
 <filter FOR_TEST_ONLY>
   @type enhance_k8s_metadata
 </filter>
+<filter FOR_TEST_ONLY>
+  @type kubernetes_metadata
+</filter>
+<filter FOR_TEST_ONLY>
+  @type kubernetes_sumologic
+</filter>
 <match **>
   @type stdout
 </match>

deploy/fluent-bit/overrides.yaml

@@ -0,0 +1,84 @@
+backend:
+  type: forward
+  forward:
+    host: fluentd
+    port: 24321
+    tls: "off"
+    tls_verify: "on"
+    tls_debug: 1
+    shared_key:
+
+trackOffsets: true
+
+tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule
+
+input:
+  tail:
+    memBufLimit: 5MB
+    parser: docker
+    path: /var/log/containers/*.log
+  systemd:
+    enabled: true
+    filters:
+      systemdUnit:
+        - addon-config.service
+        - addon-run.service
+        - cfn-etcd-environment.service
+        - cfn-signal.service
+        - clean-ca-certificates.service
+        - containerd.service
+        - coreos-metadata.service
+        - coreos-setup-environment.service
+        - coreos-tmpfiles.service
+        - dbus.service
+        - docker.service
+        - efs.service
+        - etcd-member.service
+        - etcd.service
+        - etcd2.service
+        - etcd3.service
+        - etcdadm-check.service
+        - etcdadm-reconfigure.service
+        - etcdadm-save.service
+        - etcdadm-update-status.service
+        - flanneld.service
+        - format-etcd2-volume.service
+        - kube-node-taint-and-uncordon.service
+        - kubelet.service
+        - ldconfig.service
+        - locksmithd.service
+        - logrotate.service
+        - lvm2-monitor.service
+        - mdmon.service
+        - nfs-idmapd.service
+        - nfs-mountd.service
+        - nfs-server.service
+        - nfs-utils.service
+        - node-problem-detector.service
+        - ntp.service
+        - oem-cloudinit.service
+        - rkt-gc.service
+        - rkt-metadata.service
+        - rpc-idmapd.service
+        - rpc-mountd.service
+        - rpc-statd.service
+        - rpcbind.service
+        - set-aws-environment.service
+        - system-cloudinit.service
+        - systemd-timesyncd.service
+        - update-ca-certificates.service
+        - user-cloudinit.service
+        - var-lib-etcd2.service
+    maxEntries: 1000
+    readFromTail: true
+    tag: host.*
+
+filter:
+  kubeTag: containers
+
+rawConfig: |-
+  @INCLUDE fluent-bit-service.conf
+  @INCLUDE fluent-bit-input.conf
+  @INCLUDE fluent-bit-output.conf

deploy/kubernetes/fluentd-sumologic.yaml

@@ -136,12 +136,127 @@ data:
     </match>
   logs.conf: |-
     <source>
-      @type dummy
-      tag "dummy.logs"
-      dummy {"hello":"world"}
+      @type forward
+      port 24321
+      bind 0.0.0.0
     </source>
-    <match "dummy.logs">
-      @type null
+
+    @include logs.source.containers.conf
+    @include logs.source.systemd.conf
+
+  logs.source.containers.conf: |-
+    <filter containers.**>
+      @type concat
+      key log
+      multiline_start_regexp "#{ENV['MULTILINE_START_REGEXP']}"
+      separator "#{ENV['CONCAT_SEPARATOR']}"
+      timeout_label @NORMAL
+    </filter>
+
+    <match containers.**>
+      @type relabel
+      @label @NORMAL
+    </match>
+
+    <label @NORMAL>
+      <filter containers.**>
+        @type kubernetes_metadata
+        @log_level warn
+        annotation_match ["sumologic\.com.*"]
+        de_dot false
+        watch "#{ENV['K8S_METADATA_FILTER_WATCH']}"
+        ca_file "#{ENV['K8S_METADATA_FILTER_CA_FILE']}"
+        verify_ssl "#{ENV['K8S_METADATA_FILTER_VERIFY_SSL']}"
+        client_cert "#{ENV['K8S_METADATA_FILTER_CLIENT_CERT']}"
+        client_key "#{ENV['K8S_METADATA_FILTER_CLIENT_KEY']}"
+        bearer_token_file "#{ENV['K8S_METADATA_FILTER_BEARER_TOKEN_FILE']}"
+        cache_size "#{ENV['K8S_METADATA_FILTER_BEARER_CACHE_SIZE']}"
+        cache_ttl "#{ENV['K8S_METADATA_FILTER_BEARER_CACHE_TTL']}"
+        tag_to_kubernetes_name_regexp 'containers\.(?<pod_name>[^_]+)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+        merge_json_log false
+      </filter>
+
+      <filter containers.**>
+        @type kubernetes_sumologic
+        source_name "#{ENV['SOURCE_NAME']}"
+        source_host "#{ENV['SOURCE_HOST']}"
+        log_format "#{ENV['LOG_FORMAT']}"
+        kubernetes_meta "#{ENV['KUBERNETES_META']}"
+        kubernetes_meta_reduce "#{ENV['KUBERNETES_META_REDUCE']}"
+        add_stream "#{ENV['ADD_STREAM']}"
+        add_time "#{ENV['ADD_TIME']}"
+        source_category "#{ENV['SOURCE_CATEGORY']}"
+        source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
+        source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
+        exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
+        exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
+        exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
+        exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
+      </filter>
+
+      @include logs.output.conf
+    </label>
+
+  logs.source.systemd.conf: |-
+    <match host.kubelet.**>
+      @type relabel
+      @label @KUBELET
+    </match>
+
+    <label @KUBELET>
+      <filter host.kubelet.**>
+        @type kubernetes_sumologic
+        source_category kubelet
+        source_name k8s_kubelet
+        source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
+        exclude_facility_regex "#{ENV['EXCLUDE_FACILITY_REGEX']}"
+        exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
+        exclude_priority_regex "#{ENV['EXCLUDE_PRIORITY_REGEX']}"
+        exclude_unit_regex "#{ENV['EXCLUDE_UNIT_REGEX']}"
+      </filter>
+
+      @include logs.output.conf
+    </label>
+
+    <match host.**>
+      @type relabel
+      @label @SYSTEMD
+    </match>
+
+    <label @SYSTEMD>
+      <filter host.**>
+        @type kubernetes_sumologic
+        source_category system
+        source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
+        exclude_facility_regex "#{ENV['EXCLUDE_FACILITY_REGEX']}"
+        exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
+        exclude_priority_regex "#{ENV['EXCLUDE_PRIORITY_REGEX']}"
+        exclude_unit_regex "#{ENV['EXCLUDE_UNIT_REGEX']}"
+      </filter>
+
+      <filter host.**>
+        @type record_modifier
+        <record>
+          _sumo_metadata ${record["_sumo_metadata"][:source] = tag_parts[1]; record["_sumo_metadata"]}
+        </record>
+      </filter>
+
+      @include logs.output.conf
+    </label>
+
+  logs.output.conf: |-
+    <match **>
+      @type sumologic
+      log_key log
+      endpoint "#{ENV['SUMO_ENDPOINT_LOGS']}"
+      verify_ssl "#{ENV['VERIFY_SSL']}"
+      log_format "#{ENV['LOG_FORMAT']}"
+      flush_interval "#{ENV['FLUSH_INTERVAL']}"
+      num_threads "#{ENV['NUM_THREADS']}"
+      open_timeout 60
+      add_timestamp "#{ENV['ADD_TIMESTAMP']}"
+      timestamp_key "#{ENV['TIMESTAMP_KEY']}"
+      proxy_uri "#{ENV['PROXY_URI']}"
     </match>
 ---
 apiVersion: apps/v1
@@ -185,6 +300,9 @@ spec:
         - name: prom-write
           containerPort: 9888
           protocol: TCP
+        - name: fluent-bit
+          containerPort: 24321
+          protocol: TCP
         livenessProbe:
           exec:
             command:
@@ -242,6 +360,11 @@ spec:
             secretKeyRef:
               name: metric-endpoints
               key: endpoint-metrics-node-exporter
+        - name: SUMO_ENDPOINT_LOGS
+          valueFrom:
+            secretKeyRef:
+              name: sumologic
+              key: endpoint-logs
 ---
 apiVersion: v1
 kind: Service
@@ -258,4 +381,8 @@ spec:
     port: 9888
     targetPort: 9888
     protocol: TCP
+  - name: fluent-bit
+    port: 24321
+    targetPort: 24321
+    protocol: TCP
 ---