From c268487961c18e6ad54dc64bfa6a38279fe770b6 Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Fri, 2 Aug 2019 11:22:01 -0700 Subject: [PATCH 1/7] Helm Chart initial commit --- deploy/helm/sumologic/.helmignore | 22 +++ deploy/helm/sumologic/Chart.yaml | 5 + deploy/helm/sumologic/conf/buffer.output.conf | 8 + deploy/helm/sumologic/conf/common.conf | 8 + deploy/helm/sumologic/conf/events/events.conf | 29 ++++ deploy/helm/sumologic/conf/logs/logs.conf | 7 + .../helm/sumologic/conf/logs/logs.output.conf | 9 ++ .../conf/logs/logs.source.containers.conf | 57 +++++++ .../conf/logs/logs.source.systemd.conf | 47 ++++++ .../helm/sumologic/conf/metrics/metrics.conf | 72 +++++++++ .../conf/metrics/metrics.output.conf | 4 + deploy/helm/sumologic/templates/_helpers.tpl | 56 +++++++ .../helm/sumologic/templates/clusterrole.yaml | 26 ++++ .../templates/clusterrolebinding.yaml | 15 ++ .../helm/sumologic/templates/configmap.yaml | 15 ++ .../helm/sumologic/templates/deployment.yaml | 146 ++++++++++++++++++ .../sumologic/templates/events-configmap.yaml | 11 ++ .../templates/events-deployment.yaml | 70 +++++++++ .../sumologic/templates/events-service.yaml | 15 ++ deploy/helm/sumologic/templates/service.yaml | 23 +++ .../sumologic/templates/serviceaccount.yaml | 7 + deploy/helm/sumologic/values.yaml | 80 ++++++++++ 22 files changed, 732 insertions(+) create mode 100644 deploy/helm/sumologic/.helmignore create mode 100644 deploy/helm/sumologic/Chart.yaml create mode 100644 deploy/helm/sumologic/conf/buffer.output.conf create mode 100644 deploy/helm/sumologic/conf/common.conf create mode 100644 deploy/helm/sumologic/conf/events/events.conf create mode 100644 deploy/helm/sumologic/conf/logs/logs.conf create mode 100644 deploy/helm/sumologic/conf/logs/logs.output.conf create mode 100644 deploy/helm/sumologic/conf/logs/logs.source.containers.conf create mode 100644 deploy/helm/sumologic/conf/logs/logs.source.systemd.conf create mode 100644 deploy/helm/sumologic/conf/metrics/metrics.conf create mode 100644 deploy/helm/sumologic/conf/metrics/metrics.output.conf create mode 100644 deploy/helm/sumologic/templates/_helpers.tpl create mode 100644 deploy/helm/sumologic/templates/clusterrole.yaml create mode 100644 deploy/helm/sumologic/templates/clusterrolebinding.yaml create mode 100644 deploy/helm/sumologic/templates/configmap.yaml create mode 100644 deploy/helm/sumologic/templates/deployment.yaml create mode 100644 deploy/helm/sumologic/templates/events-configmap.yaml create mode 100644 deploy/helm/sumologic/templates/events-deployment.yaml create mode 100644 deploy/helm/sumologic/templates/events-service.yaml create mode 100644 deploy/helm/sumologic/templates/service.yaml create mode 100644 deploy/helm/sumologic/templates/serviceaccount.yaml create mode 100644 deploy/helm/sumologic/values.yaml diff --git a/deploy/helm/sumologic/.helmignore b/deploy/helm/sumologic/.helmignore new file mode 100644 index 0000000000..50af031725 --- /dev/null +++ b/deploy/helm/sumologic/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/helm/sumologic/Chart.yaml b/deploy/helm/sumologic/Chart.yaml new file mode 100644 index 0000000000..2b583ee20b --- /dev/null +++ b/deploy/helm/sumologic/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: sumologic +version: 0.1.0 diff --git a/deploy/helm/sumologic/conf/buffer.output.conf b/deploy/helm/sumologic/conf/buffer.output.conf new file mode 100644 index 0000000000..42e3166098 --- /dev/null +++ b/deploy/helm/sumologic/conf/buffer.output.conf @@ -0,0 +1,8 @@ + + @type memory + compress gzip + flush_interval "#{ENV['FLUSH_INTERVAL']}" + flush_thread_count "#{ENV['NUM_THREADS']}" + chunk_limit_size "#{ENV['CHUNK_LIMIT_SIZE']}" + total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}" + \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/common.conf b/deploy/helm/sumologic/conf/common.conf new file mode 100644 index 0000000000..95c7a60be7 --- /dev/null +++ b/deploy/helm/sumologic/conf/common.conf @@ -0,0 +1,8 @@ + + @type prometheus + metrics_path /metrics + port 24231 + + + @type prometheus_output_monitor + \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf new file mode 100644 index 0000000000..dc07fab999 --- /dev/null +++ b/deploy/helm/sumologic/conf/events/events.conf @@ -0,0 +1,29 @@ + + @type prometheus + metrics_path /metrics + port 24231 + + + @type prometheus_output_monitor + + + @type events + deploy_namespace $NAMESPACE + + + @type sumologic + @id sumologic.endpoint.events + endpoint "#{ENV['SUMO_ENDPOINT_EVENTS']}" + data_type logs + disable_cookies true + verify_ssl "#{ENV['VERIFY_SSL']}" + proxy_uri "#{ENV['PROXY_URI']}" + + @type memory + compress gzip + flush_interval "#{ENV['FLUSH_INTERVAL']}" + flush_thread_count "#{ENV['NUM_THREADS']}" + chunk_limit_size "#{ENV['CHUNK_LIMIT_SIZE']}" + total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}" + + \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/logs.conf b/deploy/helm/sumologic/conf/logs/logs.conf new file mode 100644 index 0000000000..d4fbc8b3e1 --- /dev/null +++ b/deploy/helm/sumologic/conf/logs/logs.conf @@ -0,0 +1,7 @@ + + @type forward + port 24321 + bind 0.0.0.0 + +@include logs.source.containers.conf +@include logs.source.systemd.conf \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/logs.output.conf b/deploy/helm/sumologic/conf/logs/logs.output.conf new file mode 100644 index 0000000000..fb34fcd92e --- /dev/null +++ b/deploy/helm/sumologic/conf/logs/logs.output.conf @@ -0,0 +1,9 @@ +data_type logs +log_key log +endpoint "#{ENV['SUMO_ENDPOINT_LOGS']}" +verify_ssl "#{ENV['VERIFY_SSL']}" +log_format "#{ENV['LOG_FORMAT']}" +add_timestamp "#{ENV['ADD_TIMESTAMP']}" +timestamp_key "#{ENV['TIMESTAMP_KEY']}" +proxy_uri "#{ENV['PROXY_URI']}" +@include buffer.output.conf diff --git a/deploy/helm/sumologic/conf/logs/logs.source.containers.conf b/deploy/helm/sumologic/conf/logs/logs.source.containers.conf new file mode 100644 index 0000000000..992c49b3d5 --- /dev/null +++ b/deploy/helm/sumologic/conf/logs/logs.source.containers.conf @@ -0,0 +1,57 @@ + + @type concat + key log + multiline_start_regexp "#{ENV['MULTILINE_START_REGEXP']}" + separator "#{ENV['CONCAT_SEPARATOR']}" + timeout_label @NORMAL + + + @type relabel + @label @NORMAL + + \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/logs.source.systemd.conf b/deploy/helm/sumologic/conf/logs/logs.source.systemd.conf new file mode 100644 index 0000000000..58e3360711 --- /dev/null +++ b/deploy/helm/sumologic/conf/logs/logs.source.systemd.conf @@ -0,0 +1,47 @@ + + @type relabel + @label @KUBELET + + + + @type relabel + @label @SYSTEMD + + \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/metrics/metrics.conf b/deploy/helm/sumologic/conf/metrics/metrics.conf new file mode 100644 index 0000000000..6cf7a8cb1e --- /dev/null +++ b/deploy/helm/sumologic/conf/metrics/metrics.conf @@ -0,0 +1,72 @@ + + @type http + port 9888 + + @type protobuf + + + + @type datapoint + tag prometheus.datapoint.operator.rule + + + @type datapoint + tag prometheus.datapoint + + + @type rewrite_tag_filter + + key job + pattern ^(.*)$ + tag ${tag}.$1 + + + + @type enhance_k8s_metadata + + + @type prometheus_format + relabel container_name:container,pod_name:pod + + + @type sumologic + @id sumologic.endpoint.metrics.apiserver + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_APISERVER']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics.kubelet + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBELET']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics.kube.controller.manager + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_CONTROLLER_MANAGER']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics.kube.scheduler + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_SCHEDULER']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics.kube.state + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_KUBE_STATE']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics.node.exporter + endpoint "#{ENV['SUMO_ENDPOINT_METRICS_NODE_EXPORTER']}" + @include metrics.output.conf + + + @type sumologic + @id sumologic.endpoint.metrics + endpoint "#{ENV['SUMO_ENDPOINT_METRICS']}" + @include metrics.output.conf + diff --git a/deploy/helm/sumologic/conf/metrics/metrics.output.conf b/deploy/helm/sumologic/conf/metrics/metrics.output.conf new file mode 100644 index 0000000000..bf96188e3d --- /dev/null +++ b/deploy/helm/sumologic/conf/metrics/metrics.output.conf @@ -0,0 +1,4 @@ +data_type metrics +metric_data_format prometheus +disable_cookies true +@include buffer.output.conf \ No newline at end of file diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl new file mode 100644 index 0000000000..8011118a4a --- /dev/null +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "sumologic.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If dryRun=true, we use fixed value "fluentd". +*/}} +{{- define "sumologic.fullname" -}} +{{- if .Values.dryRun }} +{{- printf "fluentd" }} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end -}} +{{- end -}} + +{{/* +Create default fully qualified labels. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If dryRun=true, we use the Chart name "sumologic" and do not include labels specific to Helm. +*/}} +{{- define "sumologic.labels.app" -}} +{{- if .Values.dryRun }} +{{- template "sumologic.name" . }} +{{- else -}} +{{- template "sumologic.fullname" . }} +{{- end -}} +{{- end -}} + +{{- define "sumologic.labels.common" -}} +{{- if .Values.dryRun -}} +{{- else -}} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified fluentd user conf name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "sumologic.fluentdUserConfig.fullname" -}} +{{- printf "%s-config" (include "sumologic.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{- define "sumologic.fluentdUserConfigEvents.fullname" -}} +{{- printf "%s-events-config" (include "sumologic.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end -}} + diff --git a/deploy/helm/sumologic/templates/clusterrole.yaml b/deploy/helm/sumologic/templates/clusterrole.yaml new file mode 100644 index 0000000000..d20f6324d5 --- /dev/null +++ b/deploy/helm/sumologic/templates/clusterrole.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "sumologic.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} +rules: +- apiGroups: ["", "apps", "extensions", "events.k8s.io"] + resources: + - configmaps + - daemonsets + - deployments + - endpoints + - events + - namespaces + - nodes + - pods + - replicasets + - services + - statefulsets + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["create", "patch"] \ No newline at end of file diff --git a/deploy/helm/sumologic/templates/clusterrolebinding.yaml b/deploy/helm/sumologic/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..bb6ef352da --- /dev/null +++ b/deploy/helm/sumologic/templates/clusterrolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "sumologic.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} +subjects: +- kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ template "sumologic.fullname" . }} +roleRef: + kind: ClusterRole + name: {{ template "sumologic.fullname" . }} + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/helm/sumologic/templates/configmap.yaml b/deploy/helm/sumologic/templates/configmap.yaml new file mode 100644 index 0000000000..1cf7812fdd --- /dev/null +++ b/deploy/helm/sumologic/templates/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "sumologic.fluentdUserConfig.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} +data: + fluent.conf: |- + @include common.conf + @include metrics.conf + @include logs.conf + {{- (tpl (.Files.Glob "conf/*.conf").AsConfig .) | nindent 2 }} + {{- (tpl (.Files.Glob "conf/metrics/*.conf").AsConfig .) | nindent 2 }} + {{- (tpl (.Files.Glob "conf/logs/*.conf").AsConfig .) | nindent 2 }} diff --git a/deploy/helm/sumologic/templates/deployment.yaml b/deploy/helm/sumologic/templates/deployment.yaml new file mode 100644 index 0000000000..9b1081c541 --- /dev/null +++ b/deploy/helm/sumologic/templates/deployment.yaml @@ -0,0 +1,146 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "sumologic.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} +spec: + selector: + matchLabels: + app: {{ template "sumologic.labels.app" . }} + replicas: {{ .Values.deployment.replicaCount }} + template: + metadata: + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} + spec: + serviceAccountName: {{ template "sumologic.fullname" . }} + volumes: + - name: pos-files + hostPath: + path: /var/run/fluentd-pos + type: "" + - name: config-volume + configMap: + name: {{ template "sumologic.fluentdUserConfig.fullname" . }} + containers: + - name: fluentd + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.deployment.resources | nindent 10 }} + ports: + - name: prom-write + containerPort: 9888 + protocol: TCP + - name: fluent-bit + containerPort: 24321 + protocol: TCP + livenessProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "[[ $( pgrep ruby | wc -l) == 2 ]]" + initialDelaySeconds: 300 + periodSeconds: 20 + readinessProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "[[ $( pgrep ruby | wc -l) == 2 ]]" + initialDelaySeconds: 45 + periodSeconds: 5 + volumeMounts: + - name: config-volume + mountPath: /fluentd/etc/ + - name: pos-files + mountPath: /mnt/pos/ + env: + - name: SUMO_ENDPOINT_METRICS + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics + - name: SUMO_ENDPOINT_METRICS_APISERVER + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-apiserver + - name: SUMO_ENDPOINT_METRICS_KUBELET + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-kubelet + - name: SUMO_ENDPOINT_METRICS_KUBE_CONTROLLER_MANAGER + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-kube-controller-manager + - name: SUMO_ENDPOINT_METRICS_KUBE_SCHEDULER + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-kube-scheduler + - name: SUMO_ENDPOINT_METRICS_KUBE_STATE + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-kube-state + - name: SUMO_ENDPOINT_METRICS_NODE_EXPORTER + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-metrics-node-exporter + - name: SUMO_ENDPOINT_LOGS + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-logs + - name: LOG_FORMAT + value: {{ .Values.sumologic.logFormat | quote }} + - name: FLUSH_INTERVAL + value: {{ .Values.sumologic.flushInterval | quote }} + - name: NUM_THREADS + value: {{ .Values.sumologic.numThreads | quote }} + - name: CHUNK_LIMIT_SIZE + value: {{ .Values.sumologic.chunkLimitSize | quote }} + - name: TOTAL_LIMIT_SIZE + value: {{ .Values.sumologic.totalLimitSize | quote }} + - name: SOURCE_CATEGORY + value: {{ .Values.sumologic.sourceCategory | quote }} + - name: SOURCE_CATEGORY_PREFIX + value: {{ .Values.sumologic.sourceCategoryPrefix | quote }} + - name: SOURCE_CATEGORY_REPLACE_DASH + value: {{ .Values.sumologic.sourceCategoryReplaceDash | quote }} + - name: SOURCE_NAME + value: {{ .Values.sumologic.sourceName | quote }} + - name: KUBERNETES_META + value: {{ .Values.sumologic.kubernetesMeta | quote }} + - name: KUBERNETES_META_REDUCE + value: {{ .Values.sumologic.kubernetesMetaReduce | quote }} + - name: MULTILINE_START_REGEXP + value: {{ .Values.sumologic.multilineStartRegexp | quote }} + - name: CONCAT_SEPARATOR + value: {{ .Values.sumologic.concatSeparator | quote }} + - name: ADD_TIMESTAMP + value: {{ .Values.sumologic.addTimestamp | quote }} + - name: TIMESTAMP_KEY + value: {{ .Values.sumologic.timestampKey | quote }} + - name: ADD_STREAM + value: {{ .Values.sumologic.addStream | quote }} + - name: ADD_TIME + value: {{ .Values.sumologic.addTime | quote }} + - name: K8S_METADATA_FILTER_WATCH + value: {{ .Values.sumologic.k8sMetadataFilter.watch | quote }} + - name: K8S_METADATA_FILTER_VERIFY_SSL + value: {{ .Values.sumologic.k8sMetadataFilter.verifySsl | quote }} + - name: K8S_METADATA_FILTER_BEARER_CACHE_SIZE + value: {{ .Values.sumologic.k8sMetadataFilter.bearerCacheSize | quote }} + - name: K8S_METADATA_FILTER_BEARER_CACHE_TTL + value: {{ .Values.sumologic.k8sMetadataFilter.bearerCacheTtl | quote }} + - name: VERIFY_SSL + value: {{ .Values.sumologic.verifySsl | quote }} \ No newline at end of file diff --git a/deploy/helm/sumologic/templates/events-configmap.yaml b/deploy/helm/sumologic/templates/events-configmap.yaml new file mode 100644 index 0000000000..05ce269179 --- /dev/null +++ b/deploy/helm/sumologic/templates/events-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "sumologic.fluentdUserConfigEvents.fullname" . }} + labels: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + {{ template "sumologic.labels.common" . }} +data: + fluent.conf: |- + @include events.conf + {{- (tpl (.Files.Glob "conf/events/*.conf").AsConfig .) | nindent 2 }} \ No newline at end of file diff --git a/deploy/helm/sumologic/templates/events-deployment.yaml b/deploy/helm/sumologic/templates/events-deployment.yaml new file mode 100644 index 0000000000..0599fc8b1f --- /dev/null +++ b/deploy/helm/sumologic/templates/events-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-events" (include "sumologic.fullname" .) }} + labels: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + {{ template "sumologic.labels.common" . }} +spec: + selector: + matchLabels: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + template: + metadata: + labels: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + {{ template "sumologic.labels.common" . }} + spec: + serviceAccountName: {{ template "sumologic.fullname" . }} + volumes: + - name: pos-files + hostPath: + path: /var/run/fluentd-pos + type: "" + - name: config-volume + configMap: + name: {{ template "sumologic.fluentdUserConfigEvents.fullname" . }} + containers: + - name: fluentd-events + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.eventsDeployment.resources | nindent 10 }} + volumeMounts: + - name: config-volume + mountPath: /fluentd/etc/ + - name: pos-files + mountPath: /mnt/pos/ + livenessProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "[[ $( pgrep ruby | wc -l) == 2 ]]" + initialDelaySeconds: 300 + periodSeconds: 20 + readinessProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "[[ $( pgrep ruby | wc -l) == 2 ]]" + initialDelaySeconds: 30 + periodSeconds: 5 + env: + - name: SUMO_ENDPOINT_EVENTS + valueFrom: + secretKeyRef: + name: sumologic + key: endpoint-events + - name: VERIFY_SSL + value: {{ .Values.sumologic.verifySsl| quote }} + - name: FLUSH_INTERVAL + value: {{ .Values.sumologic.flushInterval | quote }} + - name: NUM_THREADS + value: {{ .Values.sumologic.numThreads | quote }} + - name: CHUNK_LIMIT_SIZE + value: {{ .Values.sumologic.chunkLimitSize | quote }} + - name: TOTAL_LIMIT_SIZE + value: {{ .Values.sumologic.totalLimitSize | quote }} + diff --git a/deploy/helm/sumologic/templates/events-service.yaml b/deploy/helm/sumologic/templates/events-service.yaml new file mode 100644 index 0000000000..2a7a039354 --- /dev/null +++ b/deploy/helm/sumologic/templates/events-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-events" (include "sumologic.fullname" .) }} + labels: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + {{ template "sumologic.labels.common" . }} +spec: + selector: + app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} + ports: + - name: metrics + port: 24231 + targetPort: 24231 + protocol: TCP diff --git a/deploy/helm/sumologic/templates/service.yaml b/deploy/helm/sumologic/templates/service.yaml new file mode 100644 index 0000000000..7f8e688083 --- /dev/null +++ b/deploy/helm/sumologic/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "sumologic.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} +spec: + selector: + k8s-app: fluentd-sumologic + ports: + - name: prom-write + port: 9888 + targetPort: 9888 + protocol: TCP + - name: fluent-bit + port: 24321 + targetPort: 24321 + protocol: TCP + - name: metrics + port: 24231 + targetPort: 24231 + protocol: TCP diff --git a/deploy/helm/sumologic/templates/serviceaccount.yaml b/deploy/helm/sumologic/templates/serviceaccount.yaml new file mode 100644 index 0000000000..ce3a0d7e5c --- /dev/null +++ b/deploy/helm/sumologic/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "sumologic.fullname" . }} + labels: + app: {{ template "sumologic.labels.app" . }} + {{ template "sumologic.labels.common" . }} \ No newline at end of file diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml new file mode 100644 index 0000000000..6263444a75 --- /dev/null +++ b/deploy/helm/sumologic/values.yaml @@ -0,0 +1,80 @@ +image: + repository: sumologic/kubernetes-fluentd + tag: 0.0.0 + pullPolicy: Always + +nameOverride: "" +fullnameOverride: "" + +deployment: + replicaCount: 3 + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + memory: 1Gi + cpu: 1 + requests: + memory: 768Mi + cpu: 0.5 + +eventsDeployment: + resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: + memory: 256Mi + cpu: "100m" + requests: + memory: 256Mi + cpu: "100m" + +sumologic: + logFormat: fields + + flushInterval: 5 + + numThreads: 1 + + chunkLimitSize: "100k" + + totalLimitSize: "128m" + + sourceCategory: "%{namespace}/%{pod_name}" + + sourceCategoryPrefix: "kubernetes/" + + sourceCategoryReplaceDash: "/" + + sourceName: "%{namespace}.%{pod}.%{container}" + + kubernetesMeta: "true" + + kubernetesMetaReduce: "false" + + multilineStartRegexp: '/^\w{3} \d{1,2}, \d{4}/' + + concatSeparator: "" + + addTimestamp: "true" + + timestampKey: "timestamp" + + addStream: "true" + + addTime: "true" + + verifySsl: "true" + + k8sMetadataFilter: + watch: "true" + verifySsl: "true" + bearerCacheSize: "1000" + bearerCacheTtl: "3600" + + + From 2286bb95c52f44d650b4508fe266c68b9f4d6c78 Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Fri, 2 Aug 2019 11:22:52 -0700 Subject: [PATCH 2/7] Change description --- deploy/helm/sumologic/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/Chart.yaml b/deploy/helm/sumologic/Chart.yaml index 2b583ee20b..6f6a2a72ed 100644 --- a/deploy/helm/sumologic/Chart.yaml +++ b/deploy/helm/sumologic/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 appVersion: "1.0" -description: A Helm chart for Kubernetes +description: A Helm chart for collecting Kubernetes logs, metrics and events into Sumo Logic. name: sumologic version: 0.1.0 From dac8fea741c08056c279cc88b6b798d2158042ee Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Mon, 5 Aug 2019 11:29:36 -0700 Subject: [PATCH 3/7] Remove comments in values.yaml --- deploy/helm/sumologic/values.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 6263444a75..01101c2478 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -9,10 +9,6 @@ fullnameOverride: "" deployment: replicaCount: 3 resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. limits: memory: 1Gi cpu: 1 @@ -22,10 +18,6 @@ deployment: eventsDeployment: resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. limits: memory: 256Mi cpu: "100m" From 969eb7b24b8befe10e82eec38551069658595604 Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Mon, 5 Aug 2019 13:08:49 -0700 Subject: [PATCH 4/7] Fix namespace in events.conf --- deploy/helm/sumologic/conf/events/events.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf index dc07fab999..8f469d8746 100644 --- a/deploy/helm/sumologic/conf/events/events.conf +++ b/deploy/helm/sumologic/conf/events/events.conf @@ -8,7 +8,7 @@ @type events - deploy_namespace $NAMESPACE + deploy_namespace {{ .Release.Namespace }} @type sumologic From 974984f984702d05f5af550b00c282c31eef4d1b Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Tue, 6 Aug 2019 10:52:41 -0700 Subject: [PATCH 5/7] Remove fluentdUserConfig and fluentdUserConfigEvents --- deploy/helm/sumologic/templates/_helpers.tpl | 13 ------------- deploy/helm/sumologic/templates/configmap.yaml | 2 +- deploy/helm/sumologic/templates/deployment.yaml | 2 +- .../helm/sumologic/templates/events-configmap.yaml | 2 +- .../helm/sumologic/templates/events-deployment.yaml | 2 +- 5 files changed, 4 insertions(+), 17 deletions(-) diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 8011118a4a..07a7430d1b 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -41,16 +41,3 @@ If dryRun=true, we use the Chart name "sumologic" and do not include labels spec heritage: "{{ .Release.Service }}" {{- end -}} {{- end -}} - -{{/* -Create a default fully qualified fluentd user conf name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "sumologic.fluentdUserConfig.fullname" -}} -{{- printf "%s-config" (include "sumologic.fullname" .) | trunc 63 | trimSuffix "-" }} -{{- end -}} - -{{- define "sumologic.fluentdUserConfigEvents.fullname" -}} -{{- printf "%s-events-config" (include "sumologic.fullname" .) | trunc 63 | trimSuffix "-" }} -{{- end -}} - diff --git a/deploy/helm/sumologic/templates/configmap.yaml b/deploy/helm/sumologic/templates/configmap.yaml index 1cf7812fdd..d738f352da 100644 --- a/deploy/helm/sumologic/templates/configmap.yaml +++ b/deploy/helm/sumologic/templates/configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "sumologic.fluentdUserConfig.fullname" . }} + name: {{ template "sumologic.fullname" . }} labels: app: {{ template "sumologic.labels.app" . }} {{ template "sumologic.labels.common" . }} diff --git a/deploy/helm/sumologic/templates/deployment.yaml b/deploy/helm/sumologic/templates/deployment.yaml index 9b1081c541..a30dfa5e02 100644 --- a/deploy/helm/sumologic/templates/deployment.yaml +++ b/deploy/helm/sumologic/templates/deployment.yaml @@ -24,7 +24,7 @@ spec: type: "" - name: config-volume configMap: - name: {{ template "sumologic.fluentdUserConfig.fullname" . }} + name: {{ template "sumologic.fullname" . }} containers: - name: fluentd image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/deploy/helm/sumologic/templates/events-configmap.yaml b/deploy/helm/sumologic/templates/events-configmap.yaml index 05ce269179..8c7e3ad7a4 100644 --- a/deploy/helm/sumologic/templates/events-configmap.yaml +++ b/deploy/helm/sumologic/templates/events-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "sumologic.fluentdUserConfigEvents.fullname" . }} + name: {{ printf "%s-events" (include "sumologic.fullname" .) }} labels: app: {{ printf "%s-events" (include "sumologic.labels.app" .) }} {{ template "sumologic.labels.common" . }} diff --git a/deploy/helm/sumologic/templates/events-deployment.yaml b/deploy/helm/sumologic/templates/events-deployment.yaml index 0599fc8b1f..41f34a913e 100644 --- a/deploy/helm/sumologic/templates/events-deployment.yaml +++ b/deploy/helm/sumologic/templates/events-deployment.yaml @@ -23,7 +23,7 @@ spec: type: "" - name: config-volume configMap: - name: {{ template "sumologic.fluentdUserConfigEvents.fullname" . }} + name: {{ printf "%s-events" (include "sumologic.fullname" .) }} containers: - name: fluentd-events image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" From 4b957d1a7e2b6a0cbef8245c5bbeecd26a08d473 Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Tue, 6 Aug 2019 15:13:30 -0700 Subject: [PATCH 6/7] Expose fluentdLogLevel and watchResourceEventsOverrides. Add some docs to values.yaml --- deploy/helm/sumologic/conf/events/events.conf | 26 ++++++++-- deploy/helm/sumologic/conf/logs/logs.conf | 7 ++- .../helm/sumologic/conf/metrics/metrics.conf | 5 ++ deploy/helm/sumologic/values.yaml | 51 ++++++++++++++++--- 4 files changed, 77 insertions(+), 12 deletions(-) diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf index 8f469d8746..f40293626c 100644 --- a/deploy/helm/sumologic/conf/events/events.conf +++ b/deploy/helm/sumologic/conf/events/events.conf @@ -1,3 +1,18 @@ +{{- if .Values.watchResourceEventsOverrides }} +{{- range $name, $version := .Values.watchResourceEventsOverrides }} + + @type events + deploy_namespace {{ $.Release.Namespace }} + resource_name {{ $name }} + api_version {{ $version }} + +{{- end }} +{{- else }} + + @type events + deploy_namespace {{ .Release.Namespace }} + +{{- end }} @type prometheus metrics_path /metrics @@ -6,10 +21,6 @@ @type prometheus_output_monitor - - @type events - deploy_namespace {{ .Release.Namespace }} - @type sumologic @id sumologic.endpoint.events @@ -26,4 +37,9 @@ chunk_limit_size "#{ENV['CHUNK_LIMIT_SIZE']}" total_limit_size "#{ENV['TOTAL_LIMIT_SIZE']}" - \ No newline at end of file + +{{- if .Values.fluentdLogLevel }} + + log_level {{ .Values.fluentdLogLevel }} + +{{- end }} \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/logs/logs.conf b/deploy/helm/sumologic/conf/logs/logs.conf index d4fbc8b3e1..ac884f3bd6 100644 --- a/deploy/helm/sumologic/conf/logs/logs.conf +++ b/deploy/helm/sumologic/conf/logs/logs.conf @@ -4,4 +4,9 @@ bind 0.0.0.0 @include logs.source.containers.conf -@include logs.source.systemd.conf \ No newline at end of file +@include logs.source.systemd.conf +{{- if .Values.fluentdLogLevel }} + + log_level {{ .Values.fluentdLogLevel }} + +{{- end }} \ No newline at end of file diff --git a/deploy/helm/sumologic/conf/metrics/metrics.conf b/deploy/helm/sumologic/conf/metrics/metrics.conf index 6cf7a8cb1e..c0e5805e73 100644 --- a/deploy/helm/sumologic/conf/metrics/metrics.conf +++ b/deploy/helm/sumologic/conf/metrics/metrics.conf @@ -70,3 +70,8 @@ endpoint "#{ENV['SUMO_ENDPOINT_METRICS']}" @include metrics.output.conf +{{- if .Values.fluentdLogLevel }} + + log_level {{ .Values.fluentdLogLevel }} + +{{- end }} \ No newline at end of file diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 01101c2478..0e7177fa7a 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -26,47 +26,86 @@ eventsDeployment: cpu: "100m" sumologic: + + ## Format to post logs into Sumo. json, json_merge, or text logFormat: fields + ## How frequently to push logs to SumoLogic + ## ref: https://github.com/SumoLogic/fluentd-kubernetes-sumologic#options flushInterval: 5 + ## Increase number of http threads to Sumo. May be required in heavy logging clusters numThreads: 1 chunkLimitSize: "100k" totalLimitSize: "128m" + ## Set the _sourceName metadata field in SumoLogic. + sourceName: "%{namespace}.%{pod}.%{container}" + + ## Set the _sourceCategory metadata field in SumoLogic. sourceCategory: "%{namespace}/%{pod_name}" + ## Set the prefix, for _sourceCategory metadata. sourceCategoryPrefix: "kubernetes/" + ## Used to replace - with another character. sourceCategoryReplaceDash: "/" - sourceName: "%{namespace}.%{pod}.%{container}" - + ## Include or exclude Kubernetes metadata such as namespace and pod_name if + ## using json log format. kubernetesMeta: "true" + ## Reduces redundant Kubernetes metadata. + ## ref: https://github.com/SumoLogic/fluentd-kubernetes-sumologic#reducing-kubernetes-metadata kubernetesMetaReduce: "false" + ## The regular expression for the "concat" plugin to use when merging multi-line messages multilineStartRegexp: '/^\w{3} \d{1,2}, \d{4}/' - + + ## The character to use to delimit lines within the final concatenated message. + ## Most multi-line messages contain a newline at the end of each line concatSeparator: "" + ## Option to control adding timestamp to logs. addTimestamp: "true" + ## Field name when add_timestamp is on. timestampKey: "timestamp" + ## Option to control adding stream to logs. addStream: "true" + ## Option to control adding time to logs. addTime: "true" + ## Verify SumoLogic HTTPS certificates verifySsl: "true" + ## Sets the fluentd log level. The default log level, if not specified, is info. + ## ref: https://docs.fluentd.org/deployment/logging + # fluentdLogLevel: "debug" + + ## Override Kubernetes resource types you want to get events for from different Kubernetes + ## API versions. The key represents the name of the resource type and the value represents + ## the API version. + # watchResourceEventsOverrides: + # pods: "v1" + # events: "events.k8s.io/v1beta1" + k8sMetadataFilter: + ## Option to control the enabling of metadata filter plugin watch. + ## ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration watch: "true" + + ## Verify ssl certificate of sumologic endpoint. verifySsl: "true" - bearerCacheSize: "1000" - bearerCacheTtl: "3600" - + ## Option to control the enabling of metadata filter plugin cache_size. + ## ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration + bearerCacheSize: "1000" + ## Option to control the enabling of metadata filter plugin cache_ttl. + ## ref: https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter#configuration + bearerCacheTtl: "3600" \ No newline at end of file From 318b10266df3c531b09443d2df5cf36e4ff6eb04 Mon Sep 17 00:00:00 2001 From: Maisie Wang Date: Wed, 7 Aug 2019 11:05:18 -0700 Subject: [PATCH 7/7] Change appVersion to 0.0.0 --- deploy/helm/sumologic/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/Chart.yaml b/deploy/helm/sumologic/Chart.yaml index 6f6a2a72ed..fd4cc9dd73 100644 --- a/deploy/helm/sumologic/Chart.yaml +++ b/deploy/helm/sumologic/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.0" +appVersion: "0.0.0" description: A Helm chart for collecting Kubernetes logs, metrics and events into Sumo Logic. name: sumologic version: 0.1.0