From f808152353475b8f435a2b8a1a0460565b9de362 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Fri, 31 Jan 2020 16:37:28 +0100 Subject: [PATCH] *: Refactor to not mutate global objects Signed-off-by: Frederic Branczyk --- README.md | 112 ++++-- all.jsonnet | 142 +++++--- example.jsonnet | 102 ++++-- .../manifests/thanos-bucket-deployment.yaml | 18 +- .../all/manifests/thanos-bucket-service.yaml | 11 +- .../manifests/thanos-compactor-service.yaml | 11 +- .../thanos-compactor-serviceMonitor.yaml | 7 +- .../thanos-compactor-statefulSet.yaml | 33 +- .../manifests/thanos-querier-deployment.yaml | 37 +- .../all/manifests/thanos-querier-service.yaml | 13 +- .../thanos-querier-serviceMonitor.yaml | 9 +- .../all/manifests/thanos-receive-service.yaml | 11 +- .../thanos-receive-serviceMonitor.yaml | 7 +- .../manifests/thanos-receive-statefulSet.yaml | 32 +- .../all/manifests/thanos-ruler-service.yaml | 11 +- .../thanos-ruler-serviceMonitor.yaml | 7 +- .../manifests/thanos-ruler-statefulSet.yaml | 35 +- .../all/manifests/thanos-store-service.yaml | 11 +- .../thanos-store-serviceMonitor.yaml | 7 +- .../manifests/thanos-store-statefulSet.yaml | 27 +- examples/development-minio/deployment.yaml | 11 +- examples/development-minio/pvc.yaml | 2 +- examples/development-minio/secret.yaml | 20 +- examples/development-minio/service.yaml | 1 - examples/thanos-receive.jsonnet | 21 ++ .../kube-thanos/kube-thanos-bucket.libsonnet | 133 +++---- .../kube-thanos-compactor.libsonnet | 245 +++++++++---- .../kube-thanos/kube-thanos-querier.libsonnet | 174 +++++---- .../kube-thanos-receive-pvc.libsonnet | 45 --- .../kube-thanos/kube-thanos-receive.libsonnet | 338 ++++++++++++------ .../kube-thanos/kube-thanos-ruler.libsonnet | 247 +++++++------ .../kube-thanos-servicemonitors.libsonnet | 94 ----- .../kube-thanos/kube-thanos-sidecar.libsonnet | 32 -- .../kube-thanos-store-pvc.libsonnet | 45 --- .../kube-thanos/kube-thanos-store.libsonnet | 218 ++++++----- jsonnet/kube-thanos/thanos.libsonnet | 8 + jsonnetfile.lock.json | 8 +- manifests/thanos-querier-deployment.yaml | 33 +- manifests/thanos-querier-service.yaml | 13 +- manifests/thanos-querier-serviceMonitor.yaml | 14 + manifests/thanos-store-service.yaml | 11 +- manifests/thanos-store-serviceMonitor.yaml | 14 + manifests/thanos-store-statefulSet.yaml | 35 +- 43 files changed, 1423 insertions(+), 982 deletions(-) create mode 100644 examples/thanos-receive.jsonnet delete mode 100644 jsonnet/kube-thanos/kube-thanos-receive-pvc.libsonnet delete mode 100644 jsonnet/kube-thanos/kube-thanos-servicemonitors.libsonnet delete mode 100644 jsonnet/kube-thanos/kube-thanos-sidecar.libsonnet delete mode 100644 jsonnet/kube-thanos/kube-thanos-store-pvc.libsonnet create mode 100644 jsonnet/kube-thanos/thanos.libsonnet create mode 100644 manifests/thanos-querier-serviceMonitor.yaml create mode 100644 manifests/thanos-store-serviceMonitor.yaml diff --git a/README.md b/README.md index bed120bf..02c83602 100644 --- a/README.md +++ b/README.md @@ -71,38 +71,90 @@ Here's [example.jsonnet](example.jsonnet): local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; local sts = k.apps.v1.statefulSet; local deployment = k.apps.v1.deployment; - -local kt = - (import 'kube-thanos/kube-thanos-querier.libsonnet') + - (import 'kube-thanos/kube-thanos-store.libsonnet') + - // (import 'kube-thanos/kube-thanos-pvc.libsonnet') + // Uncomment this line to enable PVCs - // (import 'kube-thanos/kube-thanos-receive.libsonnet') + - // (import 'kube-thanos/kube-thanos-sidecar.libsonnet') + - // (import 'kube-thanos/kube-thanos-servicemonitors.libsonnet') + - { - thanos+:: { - // This is just an example image, set what you need - image:: 'quay.io/thanos/thanos:v0.9.0', - objectStorageConfig+:: { - name: 'thanos-objectstorage', - key: 'thanos.yaml', - }, - - querier+: { - replicas:: 3, - }, - store+: { - replicas:: 1, +local t = (import 'kube-thanos/thanos.libsonnet'); + +local commonConfig = { + config+:: { + namespace: 'thanos', + version: '0.10.1', + image: 'quay.io/thanos/thanos:v0.10.1', + objectStorageConfig: { + name: 'thanos-objectstorage', + key: 'thanos.yaml', + }, + volumeClaimTemplate: { + spec: { + accessModes: ['ReadWriteOnce'], + resources: { + requests: { + storage: '10Gi', + }, + }, }, }, - }; - -{ ['thanos-querier-' + name]: kt.thanos.querier[name] for name in std.objectFields(kt.thanos.querier) } + -{ ['thanos-store-' + name]: kt.thanos.store[name] for name in std.objectFields(kt.thanos.store) } -// { ['thanos-receive-' + name]: kt.thanos.receive[name] for name in std.objectFields(kt.thanos.receive) } - - - + }, +}; + +//local b = t.bucket + commonConfig + { +// config+:: { +// name: 'thanos-bucket', +// replicas: 1, +// }, +//}; +// +//local c = t.compactor + t.compactor.withVolumeClaimTemplate + t.compactor.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-compactor', +// replicas: 1, +// }, +//}; +// +//local re = t.receive + t.receive.withVolumeClaimTemplate + t.receive.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-receive', +// replicas: 1, +// replicationFactor: 1, +// }, +//}; +// +//local ru = t.ruler + t.ruler.withVolumeClaimTemplate + t.ruler.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-ruler', +// replicas: 1, +// }, +//}; + +local s = t.store + t.store.withVolumeClaimTemplate + t.store.withServiceMonitor + commonConfig + { + config+:: { + name: 'thanos-store', + replicas: 1, + }, +}; + +local q = t.querier + t.querier.withServiceMonitor + commonConfig + { + config+:: { + name: 'thanos-query', + replicas: 1, + stores: [ + 'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [service.metadata.name, service.metadata.namespace] + for service in [s.service] + ], + replicaLabels: ['prometheus_replica', 'ruler_replica'], + }, +}; + +//local finalRu = ru { +// config+:: { +// queriers: ['dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [q.service.metadata.name, q.service.metadata.namespace]], +// }, +//}; + +//{ ['thanos-bucket-' + name]: b[name] for name in std.objectFields(b) } + +//{ ['thanos-compactor-' + name]: c[name] for name in std.objectFields(c) } + +//{ ['thanos-receive-' + name]: re[name] for name in std.objectFields(re) } + +//{ ['thanos-ruler-' + name]: finalRu[name] for name in std.objectFields(finalRu) } + +{ ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } + +{ ['thanos-querier-' + name]: q[name] for name in std.objectFields(q) } ``` And here's the [build.sh](build.sh) script (which uses `vendor/` to render all manifests in a json structure of `{filename: manifest-content}`): diff --git a/all.jsonnet b/all.jsonnet index a07a0a28..74271d8f 100644 --- a/all.jsonnet +++ b/all.jsonnet @@ -1,48 +1,102 @@ -local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; -local sts = k.apps.v1.statefulSet; -local deployment = k.apps.v1.deployment; - -local kt = - (import 'kube-thanos/kube-thanos-compactor.libsonnet') + - (import 'kube-thanos/kube-thanos-querier.libsonnet') + - (import 'kube-thanos/kube-thanos-store.libsonnet') + - (import 'kube-thanos/kube-thanos-store-pvc.libsonnet') + - (import 'kube-thanos/kube-thanos-receive.libsonnet') + - (import 'kube-thanos/kube-thanos-receive-pvc.libsonnet') + - (import 'kube-thanos/kube-thanos-sidecar.libsonnet') + - (import 'kube-thanos/kube-thanos-servicemonitors.libsonnet') + - (import 'kube-thanos/kube-thanos-bucket.libsonnet') + - (import 'kube-thanos/kube-thanos-ruler.libsonnet') + - { - thanos+:: { - // This is just an example image, set what you need - image:: 'quay.io/thanos/thanos:v0.9.0', - objectStorageConfig+:: { - name: 'thanos-objectstorage', - key: 'thanos.yaml', - }, +local t = (import 'kube-thanos/thanos.libsonnet'); - querier+: { - replicas:: 3, - }, - store+: { - replicas:: 1, - pvc+:: { - size: '50Gi', - }, +{ + local obs = self, + config+:: { + namespace: error 'must provide namespace', + thanosVersion: error 'must provide thanosVersion', + thanosImage: error 'must provide thanosImage', + objectStorageConfig: error 'must provide objectStorageConfig', + hashrings: error 'must provide hashring configuration', + }, + + compactor: + t.compactor + + t.compactor.withVolumeClaimTemplate + + t.compactor.withRetention + + t.compactor.withDownsamplingDisabled + + t.compactor.withServiceMonitor + + obs.config + { + config+:: { + name: 'thanos-compactor', + replicas: 1, + retentionResolutionRaw: '14d', + retentionResolution5m: '1s', + retentionResolution1h: '1s', }, - receive+: { - replicas:: 3, - pvc+:: { - size: '50Gi', + }, + + 'thanos-receive-controller': (import 'thanos-receive-controller/thanos-receive-controller.libsonnet') + { + config+:: { + name: 'thanos-receive-controller', + namespace: obs.config.namespace, + version: 'latest', + image: 'quay.io/observatorium/thanos-receive-controller:latest', + replicas: 1, + hashrings: hashrings, + }, + }, + + receivers: { + [hashring]: + t.receive + + t.receive.withVolumeClaimTemplate + + t.receive.withRetention + + t.receive.withHashringConfigMap + + t.receive.withServiceMonitor + + obs.config + { + config+:: { + name: 'thanos-receive-' + hashring, + replicas: 3, + replicationFactor: 3, + retention: '6h', + hashringConfigMapName: '%s-generated' % trc.configmap.metadata.name, + volumeClaimTemplate: { + spec: { + accessModes: ['ReadWriteOnce'], + resources: { + requests: { + storage: '50Gi', + }, + }, + }, + }, }, - }, + } + for hashring in hashrings + }, + + ruler: t.ruler + t.ruler.withVolumeClaimTemplate + t.ruler.withServiceMonitor + obs.config + { + config+:: { + name: 'thanos-ruler', + replicas: 2, + }, + }, + + store: t.store + t.store.withVolumeClaimTemplate + t.store.withServiceMonitor + obs.config + { + config+:: { + name: 'thanos-store', + replicas: 1, + }, + }, + + querier: t.querier + t.querier.withServiceMonitor + obs.config + { + config+:: { + name: 'thanos-query', + replicas: 1, + stores: [ + 'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [service.metadata.name, service.metadata.namespace] + for service in [obs.ruler.service, obs.store.service] + [receiver.service for receiver in obs.receivers] + ], + replicaLabels: ['prometheus_replica', 'ruler_replica'], + }, + }, +} + { + local obs = self, + + ruler+: { + config+:: { + queriers: ['dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [obs.querier.service.metadata.name, obs.querier.service.metadata.namespace]], }, - }; - -{ ['thanos-compactor-' + name]: kt.thanos.compactor[name] for name in std.objectFields(kt.thanos.compactor) } + -{ ['thanos-querier-' + name]: kt.thanos.querier[name] for name in std.objectFields(kt.thanos.querier) } + -{ ['thanos-receive-' + name]: kt.thanos.receive[name] for name in std.objectFields(kt.thanos.receive) } + -{ ['thanos-bucket-' + name]: kt.thanos.bucket[name] for name in std.objectFields(kt.thanos.bucket) } + -{ ['thanos-store-' + name]: kt.thanos.store[name] for name in std.objectFields(kt.thanos.store) } + -{ ['thanos-ruler-' + name]: kt.thanos.ruler[name] for name in std.objectFields(kt.thanos.ruler) } + }, +} diff --git a/example.jsonnet b/example.jsonnet index 0c000faa..ac6493d3 100644 --- a/example.jsonnet +++ b/example.jsonnet @@ -1,35 +1,87 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; local sts = k.apps.v1.statefulSet; local deployment = k.apps.v1.deployment; +local t = (import 'kube-thanos/thanos.libsonnet'); -local kt = - (import 'kube-thanos/kube-thanos-querier.libsonnet') + - (import 'kube-thanos/kube-thanos-store.libsonnet') + - // (import 'kube-thanos/kube-thanos-pvc.libsonnet') + // Uncomment this line to enable PVCs - // (import 'kube-thanos/kube-thanos-receive.libsonnet') + - // (import 'kube-thanos/kube-thanos-sidecar.libsonnet') + - // (import 'kube-thanos/kube-thanos-servicemonitors.libsonnet') + - { - thanos+:: { - // This is just an example image, set what you need - image:: 'quay.io/thanos/thanos:v0.9.0', - objectStorageConfig+:: { - name: 'thanos-objectstorage', - key: 'thanos.yaml', - }, - - querier+: { - replicas:: 3, - }, - store+: { - replicas:: 1, +local commonConfig = { + config+:: { + namespace: 'thanos', + version: '0.10.1', + image: 'quay.io/thanos/thanos:v0.10.1', + objectStorageConfig: { + name: 'thanos-objectstorage', + key: 'thanos.yaml', + }, + volumeClaimTemplate: { + spec: { + accessModes: ['ReadWriteOnce'], + resources: { + requests: { + storage: '10Gi', + }, + }, }, }, - }; + }, +}; + +//local b = t.bucket + commonConfig + { +// config+:: { +// name: 'thanos-bucket', +// replicas: 1, +// }, +//}; +// +//local c = t.compactor + t.compactor.withVolumeClaimTemplate + t.compactor.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-compactor', +// replicas: 1, +// }, +//}; +// +//local re = t.receive + t.receive.withVolumeClaimTemplate + t.receive.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-receive', +// replicas: 1, +// replicationFactor: 1, +// }, +//}; +// +//local ru = t.ruler + t.ruler.withVolumeClaimTemplate + t.ruler.withServiceMonitor + commonConfig + { +// config+:: { +// name: 'thanos-ruler', +// replicas: 1, +// }, +//}; -{ ['thanos-querier-' + name]: kt.thanos.querier[name] for name in std.objectFields(kt.thanos.querier) } + -{ ['thanos-store-' + name]: kt.thanos.store[name] for name in std.objectFields(kt.thanos.store) } -// { ['thanos-receive-' + name]: kt.thanos.receive[name] for name in std.objectFields(kt.thanos.receive) } +local s = t.store + t.store.withVolumeClaimTemplate + t.store.withServiceMonitor + commonConfig + { + config+:: { + name: 'thanos-store', + replicas: 1, + }, +}; +local q = t.querier + t.querier.withServiceMonitor + commonConfig + { + config+:: { + name: 'thanos-query', + replicas: 1, + stores: [ + 'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [service.metadata.name, service.metadata.namespace] + for service in [s.service] + ], + replicaLabels: ['prometheus_replica', 'ruler_replica'], + }, +}; +//local finalRu = ru { +// config+:: { +// queriers: ['dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [q.service.metadata.name, q.service.metadata.namespace]], +// }, +//}; +//{ ['thanos-bucket-' + name]: b[name] for name in std.objectFields(b) } + +//{ ['thanos-compactor-' + name]: c[name] for name in std.objectFields(c) } + +//{ ['thanos-receive-' + name]: re[name] for name in std.objectFields(re) } + +//{ ['thanos-ruler-' + name]: finalRu[name] for name in std.objectFields(finalRu) } + +{ ['thanos-store-' + name]: s[name] for name in std.objectFields(s) } + +{ ['thanos-querier-' + name]: q[name] for name in std.objectFields(q) } diff --git a/examples/all/manifests/thanos-bucket-deployment.yaml b/examples/all/manifests/thanos-bucket-deployment.yaml index 3552b06f..16b6063d 100644 --- a/examples/all/manifests/thanos-bucket-deployment.yaml +++ b/examples/all/manifests/thanos-bucket-deployment.yaml @@ -2,18 +2,26 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/name: thanos-bucket + app.kubernetes.io/component: thanos-bucket + app.kubernetes.io/instance: thanos-bucket + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-bucket - namespace: monitoring + namespace: thanos spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-bucket + app.kubernetes.io/component: thanos-bucket + app.kubernetes.io/instance: thanos-bucket + app.kubernetes.io/name: thanos template: metadata: labels: - app.kubernetes.io/name: thanos-bucket + app.kubernetes.io/component: thanos-bucket + app.kubernetes.io/instance: thanos-bucket + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: containers: - args: @@ -26,7 +34,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 4 httpGet: diff --git a/examples/all/manifests/thanos-bucket-service.yaml b/examples/all/manifests/thanos-bucket-service.yaml index 116d3515..bd7efd9a 100644 --- a/examples/all/manifests/thanos-bucket-service.yaml +++ b/examples/all/manifests/thanos-bucket-service.yaml @@ -2,13 +2,18 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-bucket + app.kubernetes.io/component: thanos-bucket + app.kubernetes.io/instance: thanos-bucket + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-bucket - namespace: monitoring + namespace: thanos spec: ports: - name: http port: 8080 targetPort: http selector: - app.kubernetes.io/name: thanos-bucket + app.kubernetes.io/component: thanos-bucket + app.kubernetes.io/instance: thanos-bucket + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-compactor-service.yaml b/examples/all/manifests/thanos-compactor-service.yaml index 6a7ac852..c2f2425c 100644 --- a/examples/all/manifests/thanos-compactor-service.yaml +++ b/examples/all/manifests/thanos-compactor-service.yaml @@ -2,13 +2,18 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-compactor - namespace: monitoring + namespace: thanos spec: ports: - name: http port: 10902 targetPort: http selector: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-compactor-serviceMonitor.yaml b/examples/all/manifests/thanos-compactor-serviceMonitor.yaml index 99364b0c..b20c7487 100644 --- a/examples/all/manifests/thanos-compactor-serviceMonitor.yaml +++ b/examples/all/manifests/thanos-compactor-serviceMonitor.yaml @@ -2,10 +2,13 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: thanos-compactor - namespace: monitoring + namespace: thanos spec: endpoints: - port: http selector: matchLabels: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/examples/all/manifests/thanos-compactor-statefulSet.yaml b/examples/all/manifests/thanos-compactor-statefulSet.yaml index b0756097..ced7cb83 100644 --- a/examples/all/manifests/thanos-compactor-statefulSet.yaml +++ b/examples/all/manifests/thanos-compactor-statefulSet.yaml @@ -2,19 +2,27 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-compactor - namespace: monitoring + namespace: thanos spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos serviceName: thanos-compactor template: metadata: labels: - app.kubernetes.io/name: thanos-compactor + app.kubernetes.io/component: thanos-compactor + app.kubernetes.io/instance: thanos-compactor + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: containers: - args: @@ -31,7 +39,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 4 httpGet: @@ -52,9 +60,16 @@ spec: periodSeconds: 5 volumeMounts: - mountPath: /var/thanos/compactor - name: thanos-compactor-data + name: data readOnly: false terminationGracePeriodSeconds: 120 - volumes: - - emptyDir: {} - name: thanos-compactor-data + volumes: null + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/examples/all/manifests/thanos-querier-deployment.yaml b/examples/all/manifests/thanos-querier-deployment.yaml index 5c6cd13e..14da142a 100644 --- a/examples/all/manifests/thanos-querier-deployment.yaml +++ b/examples/all/manifests/thanos-querier-deployment.yaml @@ -2,18 +2,26 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/name: thanos-querier - name: thanos-querier - namespace: monitoring + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 + name: thanos-query + namespace: thanos spec: - replicas: 3 + replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos template: metadata: labels: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: affinity: podAntiAffinity: @@ -24,23 +32,22 @@ spec: - key: app.kubernetes.io/name operator: In values: - - thanos-querier + - thanos namespaces: - - monitoring + - thanos topologyKey: kubernetes.io/hostname weight: 100 containers: - args: - query - - --query.replica-label=prometheus_replica - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:9090 - - --store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc.cluster.local - - --store=dnssrv+_grpc._tcp.thanos-receive.monitoring.svc.cluster.local - - --store=dnssrv+_grpc._tcp.prometheus-k8s.monitoring.svc.cluster.local + - --query.replica-label=prometheus_replica - --query.replica-label=ruler_replica - - --store=dnssrv+_grpc._tcp.thanos-ruler.monitoring.svc.cluster.local - image: quay.io/thanos/thanos:v0.9.0 + - --store=dnssrv+_grpc._tcp.thanos-receive.thanos.svc.cluster.local + - --store=dnssrv+_grpc._tcp.thanos-ruler.thanos.svc.cluster.local + - --store=dnssrv+_grpc._tcp.thanos-store.thanos.svc.cluster.local + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 4 httpGet: @@ -48,7 +55,7 @@ spec: port: 9090 scheme: HTTP periodSeconds: 30 - name: thanos-querier + name: thanos-query ports: - containerPort: 10901 name: grpc diff --git a/examples/all/manifests/thanos-querier-service.yaml b/examples/all/manifests/thanos-querier-service.yaml index 5ca6108b..700c6a64 100644 --- a/examples/all/manifests/thanos-querier-service.yaml +++ b/examples/all/manifests/thanos-querier-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-querier - name: thanos-querier - namespace: monitoring + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 + name: thanos-query + namespace: thanos spec: ports: - name: grpc @@ -14,4 +17,6 @@ spec: port: 9090 targetPort: http selector: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-querier-serviceMonitor.yaml b/examples/all/manifests/thanos-querier-serviceMonitor.yaml index 0ef1d7f7..8d032b01 100644 --- a/examples/all/manifests/thanos-querier-serviceMonitor.yaml +++ b/examples/all/manifests/thanos-querier-serviceMonitor.yaml @@ -1,11 +1,14 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: thanos-querier - namespace: monitoring + name: thanos-query + namespace: thanos spec: endpoints: - port: http selector: matchLabels: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/examples/all/manifests/thanos-receive-service.yaml b/examples/all/manifests/thanos-receive-service.yaml index fd41d828..1371a3f3 100644 --- a/examples/all/manifests/thanos-receive-service.yaml +++ b/examples/all/manifests/thanos-receive-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-receive - namespace: monitoring + namespace: thanos spec: clusterIP: None ports: @@ -18,4 +21,6 @@ spec: port: 19291 targetPort: 19291 selector: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-receive-serviceMonitor.yaml b/examples/all/manifests/thanos-receive-serviceMonitor.yaml index b6cb905e..03ca1f18 100644 --- a/examples/all/manifests/thanos-receive-serviceMonitor.yaml +++ b/examples/all/manifests/thanos-receive-serviceMonitor.yaml @@ -2,10 +2,13 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: thanos-receive - namespace: monitoring + namespace: thanos spec: endpoints: - port: http selector: matchLabels: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/examples/all/manifests/thanos-receive-statefulSet.yaml b/examples/all/manifests/thanos-receive-statefulSet.yaml index 3c624b66..2a1994d7 100644 --- a/examples/all/manifests/thanos-receive-statefulSet.yaml +++ b/examples/all/manifests/thanos-receive-statefulSet.yaml @@ -2,19 +2,27 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-receive - namespace: monitoring + namespace: thanos spec: - replicas: 3 + replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos serviceName: thanos-receive template: metadata: labels: - app.kubernetes.io/name: thanos-receive + app.kubernetes.io/component: thanos-receive + app.kubernetes.io/instance: thanos-receive + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: affinity: podAntiAffinity: @@ -22,12 +30,12 @@ spec: - podAffinityTerm: labelSelector: matchExpressions: - - key: app.kubernetes.io/name + - key: app.kubernetes.io/instance operator: In values: - thanos-receive namespaces: - - monitoring + - thanos topologyKey: kubernetes.io/hostname weight: 100 containers: @@ -36,6 +44,7 @@ spec: - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:10902 - --remote-write.address=0.0.0.0:19291 + - --receive.replication-factor=1 - --objstore.config=$(OBJSTORE_CONFIG) - --tsdb.path=/var/thanos/receive - --label=replica="$(NAME)" @@ -50,7 +59,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 8 httpGet: @@ -75,17 +84,16 @@ spec: periodSeconds: 5 volumeMounts: - mountPath: /var/thanos/receive - name: thanos-receive-data + name: data readOnly: false terminationGracePeriodSeconds: 120 volumes: null volumeClaimTemplates: - metadata: - name: thanos-receive-data + name: data spec: accessModes: - ReadWriteOnce resources: requests: - storage: 50Gi - storageClassName: standard + storage: 10Gi diff --git a/examples/all/manifests/thanos-ruler-service.yaml b/examples/all/manifests/thanos-ruler-service.yaml index 74c82bcd..09fb2dba 100644 --- a/examples/all/manifests/thanos-ruler-service.yaml +++ b/examples/all/manifests/thanos-ruler-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-ruler - namespace: monitoring + namespace: thanos spec: clusterIP: None ports: @@ -15,4 +18,6 @@ spec: port: 10902 targetPort: http selector: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-ruler-serviceMonitor.yaml b/examples/all/manifests/thanos-ruler-serviceMonitor.yaml index 6411eaae..1cd31175 100644 --- a/examples/all/manifests/thanos-ruler-serviceMonitor.yaml +++ b/examples/all/manifests/thanos-ruler-serviceMonitor.yaml @@ -2,10 +2,13 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: thanos-ruler - namespace: monitoring + namespace: thanos spec: endpoints: - port: http selector: matchLabels: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/examples/all/manifests/thanos-ruler-statefulSet.yaml b/examples/all/manifests/thanos-ruler-statefulSet.yaml index f0d700c4..91d1eb48 100644 --- a/examples/all/manifests/thanos-ruler-statefulSet.yaml +++ b/examples/all/manifests/thanos-ruler-statefulSet.yaml @@ -2,19 +2,27 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-ruler - namespace: monitoring + namespace: thanos spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos serviceName: thanos-ruler template: metadata: labels: - app.kubernetes.io/name: thanos-ruler + app.kubernetes.io/component: thanos-ruler + app.kubernetes.io/instance: thanos-ruler + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: containers: - args: @@ -25,7 +33,7 @@ spec: - --data-dir=/var/thanos/ruler - --label=ruler_replica="$(NAME)" - --alert.label-drop="ruler_replica" - - --query=dnssrv+_http._tcp.thanos-querier.monitoring.svc.cluster.local + - --query=dnssrv+_http._tcp.thanos-query.thanos.svc.cluster.local env: - name: NAME valueFrom: @@ -36,7 +44,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 24 httpGet: @@ -60,8 +68,15 @@ spec: periodSeconds: 5 volumeMounts: - mountPath: /var/thanos/ruler - name: thanos-ruler-data + name: data readOnly: false - volumes: - - emptyDir: {} - name: thanos-ruler-data + volumes: null + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/examples/all/manifests/thanos-store-service.yaml b/examples/all/manifests/thanos-store-service.yaml index 10a762ca..3e27d984 100644 --- a/examples/all/manifests/thanos-store-service.yaml +++ b/examples/all/manifests/thanos-store-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-store - namespace: monitoring + namespace: thanos spec: clusterIP: None ports: @@ -15,4 +18,6 @@ spec: port: 10902 targetPort: 10902 selector: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos diff --git a/examples/all/manifests/thanos-store-serviceMonitor.yaml b/examples/all/manifests/thanos-store-serviceMonitor.yaml index 39518c74..8d3a4bc3 100644 --- a/examples/all/manifests/thanos-store-serviceMonitor.yaml +++ b/examples/all/manifests/thanos-store-serviceMonitor.yaml @@ -2,10 +2,13 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: thanos-store - namespace: monitoring + namespace: thanos spec: endpoints: - port: http selector: matchLabels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/examples/all/manifests/thanos-store-statefulSet.yaml b/examples/all/manifests/thanos-store-statefulSet.yaml index e957a26c..4a7e3307 100644 --- a/examples/all/manifests/thanos-store-statefulSet.yaml +++ b/examples/all/manifests/thanos-store-statefulSet.yaml @@ -2,19 +2,27 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-store - namespace: monitoring + namespace: thanos spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos serviceName: thanos-store template: metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: containers: - args: @@ -29,7 +37,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 8 httpGet: @@ -51,18 +59,17 @@ spec: scheme: HTTP periodSeconds: 5 volumeMounts: - - mountPath: /var/thanos/store - name: thanos-store-data + - mountPath: /var/thanos/receive + name: data readOnly: false terminationGracePeriodSeconds: 120 volumes: null volumeClaimTemplates: - metadata: - name: thanos-store-data + name: data spec: accessModes: - ReadWriteOnce resources: requests: - storage: 50Gi - storageClassName: standard + storage: 10Gi diff --git a/examples/development-minio/deployment.yaml b/examples/development-minio/deployment.yaml index dde8d32b..60378e50 100644 --- a/examples/development-minio/deployment.yaml +++ b/examples/development-minio/deployment.yaml @@ -2,8 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: name: minio - namespace: monitoring spec: + selector: + matchLabels: + app.kubernetes.io/name: minio strategy: type: Recreate template: @@ -14,9 +16,10 @@ spec: containers: - name: minio image: minio/minio - args: - - server - - /storage + command: + - /bin/sh + - -c + - "mkdir -p /storage/thanos && /usr/bin/minio server /storage" env: - name: MINIO_ACCESS_KEY value: "minio" diff --git a/examples/development-minio/pvc.yaml b/examples/development-minio/pvc.yaml index 5c9d704e..0175c586 100644 --- a/examples/development-minio/pvc.yaml +++ b/examples/development-minio/pvc.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: minio - namespace: monitoring + namespace: thanos labels: app.kubernetes.io/name: minio spec: diff --git a/examples/development-minio/secret.yaml b/examples/development-minio/secret.yaml index 84e8ef4b..68e841c6 100644 --- a/examples/development-minio/secret.yaml +++ b/examples/development-minio/secret.yaml @@ -1,16 +1,14 @@ -# type: s3 -# config: -# bucket: thanos -# endpoint: minio.monitoring.svc.cluster.local:9000 -# insecure: true -# access_key: minio -# secret_key: minio123 - apiVersion: v1 kind: Secret metadata: name: thanos-objectstorage - namespace: monitoring type: Opaque -data: - thanos.yaml: dHlwZTogczMKY29uZmlnOgogIGJ1Y2tldDogdGhhbm9zCiAgZW5kcG9pbnQ6IG1pbmlvLm1vbml0b3Jpbmcuc3ZjLmNsdXN0ZXIubG9jYWw6OTAwMAogIGluc2VjdXJlOiB0cnVlCiAgYWNjZXNzX2tleTogbWluaW8KICBzZWNyZXRfa2V5OiBtaW5pbzEyMwo= +stringData: + thanos.yaml: |- + type: s3 + config: + bucket: thanos + endpoint: minio:9000 + insecure: true + access_key: minio + secret_key: minio123 diff --git a/examples/development-minio/service.yaml b/examples/development-minio/service.yaml index fa5039dd..a6cf645d 100644 --- a/examples/development-minio/service.yaml +++ b/examples/development-minio/service.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Service metadata: name: minio - namespace: monitoring spec: type: ClusterIP ports: diff --git a/examples/thanos-receive.jsonnet b/examples/thanos-receive.jsonnet new file mode 100644 index 00000000..872f7bf7 --- /dev/null +++ b/examples/thanos-receive.jsonnet @@ -0,0 +1,21 @@ +local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; +local sts = k.apps.v1.statefulSet; +local deployment = k.apps.v1.deployment; +local t = (import 'kube-thanos/kube-thanos-receive.libsonnet'); + +t.receive { + local tr = self, + name:: 'thanos-receive', + namespace:: 'observability', + version:: '0.10.1', + image:: 'quay.io/thanos/thanos:v' + tr.version, + replicas:: 3, + replicationFactor:: 3, + objectStorageConfig:: { + name: 'thanos-objectstorage', + key: 'thanos.yaml', + }, + pvcTemplate+:: { + size: '50G', + }, +} diff --git a/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet b/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet index 44672868..f9e98700 100644 --- a/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-bucket.libsonnet @@ -1,72 +1,79 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - bucket+: { - local tb = self, - name:: 'thanos-bucket', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - labels+:: { - 'app.kubernetes.io/name': tb.name, - }, - objectStorageConfig:: $.thanos.objectStorageConfig, - ports:: { - http: 8080, - }, + local tb = self, - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + objectStorageConfig: error 'must provide objectStorageConfig', - service.new( - tb.name, - tb.labels, - [ports.newNamed('http', tb.ports.http, 'http')], - ) + - service.mixin.metadata.withNamespace(tb.namespace) + - service.mixin.metadata.withLabels(tb.labels), - - deployment: - local deployment = k.apps.v1.deployment; - local container = deployment.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - - local c = - container.new(tb.name, tb.image) + - container.withArgs([ - 'bucket', - 'web', - '--objstore.config=$(OBJSTORE_CONFIG)', - ]) + - container.withEnv([ - containerEnv.fromSecretRef( - 'OBJSTORE_CONFIG', - tb.objectStorageConfig.name, - tb.objectStorageConfig.key, - ), - ]) + - container.withPorts([ - { name: 'http', containerPort: tb.ports.http }, - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(30) + - container.mixin.livenessProbe.withFailureThreshold(4) + - container.mixin.livenessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(20) + - container.mixin.readinessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': tb.config.name, + 'app.kubernetes.io/version': 'v' + tb.config.version, + 'app.kubernetes.io/component': 'thanos-bucket', + }, - deployment.new(tb.name, 1, c, $.thanos.bucket.deployment.metadata.labels) + - deployment.mixin.metadata.withNamespace(tb.namespace) + - deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': tb.name }) + - deployment.mixin.spec.selector.withMatchLabels(tb.labels) + - deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120), + podLabelSelector:: { + [labelName]: tb.config.commonLabels[labelName] + for labelName in std.objectFields(tb.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) }, }, + + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; + + service.new( + tb.config.name, + tb.config.podLabelSelector, + [ports.newNamed('http', 8080, 'http')], + ) + + service.mixin.metadata.withNamespace(tb.config.namespace) + + service.mixin.metadata.withLabels(tb.config.commonLabels), + + deployment: + local deployment = k.apps.v1.deployment; + local container = deployment.mixin.spec.template.spec.containersType; + local containerEnv = container.envType; + + local c = + container.new('thanos-bucket', tb.config.image) + + container.withArgs([ + 'bucket', + 'web', + '--objstore.config=$(OBJSTORE_CONFIG)', + ]) + + container.withEnv([ + containerEnv.fromSecretRef( + 'OBJSTORE_CONFIG', + tb.config.objectStorageConfig.name, + tb.config.objectStorageConfig.key, + ), + ]) + + container.withPorts([ + { name: 'http', containerPort: tb.service.spec.ports[0].port }, + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(30) + + container.mixin.livenessProbe.withFailureThreshold(4) + + container.mixin.livenessProbe.httpGet.withPort(tb.service.spec.ports[0].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(20) + + container.mixin.readinessProbe.httpGet.withPort(tb.service.spec.ports[0].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + + deployment.new(tb.config.name, 1, c, tb.config.commonLabels) + + deployment.mixin.metadata.withNamespace(tb.config.namespace) + + deployment.mixin.metadata.withLabels(tb.config.commonLabels) + + deployment.mixin.spec.selector.withMatchLabels(tb.config.podLabelSelector) + + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120), } diff --git a/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet b/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet index 2d75b733..4edab065 100644 --- a/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-compactor.libsonnet @@ -1,86 +1,187 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - compactor+: { - local tc = self, - name:: 'thanos-compactor', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - objectStorageConfig:: $.thanos.objectStorageConfig, + local tc = self, - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + objectStorageConfig: error 'must provide objectStorageConfig', - service.new( - tc.name, - $.thanos.compactor.statefulSet.metadata.labels, - [ - ports.newNamed('http', 10902, 'http'), - ], - ) + - service.mixin.metadata.withNamespace(tc.namespace) + - service.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.compactor.service.metadata.name }), + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': tc.config.name, + 'app.kubernetes.io/version': 'v' + tc.config.version, + 'app.kubernetes.io/component': 'thanos-compactor', + }, + + podLabelSelector:: { + [labelName]: tc.config.commonLabels[labelName] + for labelName in std.objectFields(tc.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) + }, + }, + + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; + + service.new( + tc.config.name, + tc.config.podLabelSelector, + [ + ports.newNamed('http', 10902, 'http'), + ], + ) + + service.mixin.metadata.withNamespace(tc.config.namespace) + + service.mixin.metadata.withLabels(tc.config.commonLabels), + + statefulSet: + local statefulSet = k.apps.v1.statefulSet; + local volume = statefulSet.mixin.spec.template.spec.volumesType; + local container = statefulSet.mixin.spec.template.spec.containersType; + local containerEnv = container.envType; + local containerVolumeMount = container.volumeMountsType; + + local c = + container.new('thanos-compactor', tc.config.image) + + container.withArgs([ + 'compact', + '--wait', + '--objstore.config=$(OBJSTORE_CONFIG)', + '--data-dir=/var/thanos/compactor', + '--debug.accept-malformed-index', + ]) + + container.withEnv([ + containerEnv.fromSecretRef( + 'OBJSTORE_CONFIG', + tc.config.objectStorageConfig.name, + tc.config.objectStorageConfig.key, + ), + ]) + + container.withPorts([ + { name: 'http', containerPort: tc.service.spec.ports[0].port }, + ]) + + container.withVolumeMounts([ + containerVolumeMount.new('data', '/var/thanos/compactor', false), + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(30) + + container.mixin.livenessProbe.withFailureThreshold(4) + + container.mixin.livenessProbe.httpGet.withPort(tc.service.spec.ports[0].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(20) + + container.mixin.readinessProbe.httpGet.withPort(tc.service.spec.ports[0].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); - statefulSet: - local statefulSet = k.apps.v1.statefulSet; - local volume = statefulSet.mixin.spec.template.spec.volumesType; - local container = statefulSet.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - local containerVolumeMount = container.volumeMountsType; + statefulSet.new(tc.config.name, 1, c, [], tc.config.commonLabels) + + statefulSet.mixin.metadata.withNamespace(tc.config.namespace) + + statefulSet.mixin.metadata.withLabels(tc.config.commonLabels) + + statefulSet.mixin.spec.withServiceName(tc.service.metadata.name) + + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + + statefulSet.mixin.spec.template.spec.withVolumes([ + volume.fromEmptyDir('data'), + ]) + + statefulSet.mixin.spec.selector.withMatchLabels(tc.config.podLabelSelector) + + { + spec+: { + volumeClaimTemplates: null, + }, + }, + + withServiceMonitor:: { + local tc = self, + serviceMonitor: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata+: { + name: tc.config.name, + namespace: tc.config.namespace, + }, + spec: { + selector: { + matchLabels: tc.config.commonLabels, + }, + endpoints: [ + { port: 'http' }, + ], + }, + }, + }, + + withVolumeClaimTemplate:: { + local tc = self, + config+:: { + volumeClaimTemplate: error 'must provide volumeClaimTemplate', + }, + statefulSet+: { + spec+: { + template+: { + spec+: { + volumes: null, + }, + }, + volumeClaimTemplates: [tc.config.volumeClaimTemplate { + metadata+: { + name: 'data', + }, + }], + }, + }, + }, + + withRetention:: { + local tc = self, + config+:: { + retentionResolutionRaw: error 'must provide retentionResolutionRaw', + retentionResolution5m: error 'must provide retentionResolution5m', + retentionResolution1h: error 'must provide retentionResolution1h', + }, + + statefulSet+: { + spec+: { + template+: { + spec+: { + containers: [ + if c.name == 'thanos-compactor' then c { + args+: [ + '--retention.resolution-raw=' + tc.config.retentionResolutionRaw, + '--retention.resolution-5m=' + tc.config.retentionResolution5m, + '--retention.resolution-1h=' + tc.config.retentionResolution1h, + ], + } else c + for c in super.containers + ], + }, + }, + }, + }, + }, - local c = - container.new($.thanos.compactor.statefulSet.metadata.name, tc.image) + - container.withArgs([ - 'compact', - '--wait', - '--retention.resolution-raw=16d', - '--retention.resolution-5m=42d', - '--retention.resolution-1h=180d', - '--objstore.config=$(OBJSTORE_CONFIG)', - '--data-dir=/var/thanos/compactor', - ]) + - container.withEnv([ - containerEnv.fromSecretRef( - 'OBJSTORE_CONFIG', - tc.objectStorageConfig.name, - tc.objectStorageConfig.key, - ), - ]) + - container.withPorts([ - { name: 'http', containerPort: $.thanos.compactor.service.spec.ports[0].port }, - ]) + - container.withVolumeMounts([ - containerVolumeMount.new('thanos-compactor-data', '/var/thanos/compactor', false), - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(30) + - container.mixin.livenessProbe.withFailureThreshold(4) + - container.mixin.livenessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(20) + - container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + withDownsamplingDisabled:: { + local tc = self, - statefulSet.new(tc.name, 1, c, [], $.thanos.compactor.statefulSet.metadata.labels) + - statefulSet.mixin.metadata.withNamespace(tc.namespace) + - statefulSet.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.compactor.statefulSet.metadata.name }) + - statefulSet.mixin.spec.withServiceName($.thanos.compactor.service.metadata.name) + - statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + - statefulSet.mixin.spec.template.spec.withVolumes([ - volume.fromEmptyDir('thanos-compactor-data'), - ]) + - statefulSet.mixin.spec.selector.withMatchLabels($.thanos.compactor.statefulSet.metadata.labels) + - { + statefulSet+: { + spec+: { + template+: { spec+: { - volumeClaimTemplates:: null, + containers: [ + if c.name == 'thanos-compactor' then c { + args+: [ + '--downsampling.disable', + ], + } else c + for c in super.containers + ], }, }, + }, }, }, } diff --git a/jsonnet/kube-thanos/kube-thanos-querier.libsonnet b/jsonnet/kube-thanos/kube-thanos-querier.libsonnet index f87acc92..86d42834 100644 --- a/jsonnet/kube-thanos/kube-thanos-querier.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-querier.libsonnet @@ -1,81 +1,117 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - namespace:: 'monitoring', - image:: error 'must set thanos image', + local tq = self, - querier+: { - local tq = self, - name:: 'thanos-querier', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - replicas:: 1, - replicaLabel:: 'prometheus_replica', + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + replicas: error 'must provide replicas', + replicaLabels: error 'must provide replica labels', + stores: error 'must provide store addresses', - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': tq.config.name, + 'app.kubernetes.io/version': 'v' + tq.config.version, + 'app.kubernetes.io/component': 'thanos-querier', + }, + + podLabelSelector:: { + [labelName]: tq.config.commonLabels[labelName] + for labelName in std.objectFields(tq.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) + }, + }, + + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; + + service.new( + tq.config.name, + tq.config.podLabelSelector, + [ + ports.newNamed('grpc', 10901, 'grpc'), + ports.newNamed('http', 9090, 'http'), + ] + ) + + service.mixin.metadata.withNamespace(tq.config.namespace) + + service.mixin.metadata.withLabels(tq.config.commonLabels), - service.new( - tq.name, - $.thanos.querier.deployment.metadata.labels, - [ - ports.newNamed('grpc', 10901, 'grpc'), - ports.newNamed('http', 9090, 'http'), - ] - ) + - service.mixin.metadata.withNamespace(tq.namespace) + - service.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.querier.service.metadata.name }), + deployment: + local deployment = k.apps.v1.deployment; + local container = deployment.mixin.spec.template.spec.containersType; + local affinity = deployment.mixin.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecutionType; + local matchExpression = affinity.mixin.podAffinityTerm.labelSelector.matchExpressionsType; - deployment: - local deployment = k.apps.v1.deployment; - local container = deployment.mixin.spec.template.spec.containersType; - local affinity = deployment.mixin.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecutionType; - local matchExpression = affinity.mixin.podAffinityTerm.labelSelector.matchExpressionsType; + local c = + container.new('thanos-query', tq.config.image) + + container.withArgs([ + 'query', + '--grpc-address=0.0.0.0:%d' % tq.service.spec.ports[0].port, + '--http-address=0.0.0.0:%d' % tq.service.spec.ports[1].port, + ] + [ + '--query.replica-label=%s' % labelName + for labelName in tq.config.replicaLabels + ] + [ + '--store=%s' % store + for store in tq.config.stores + ]) + + container.withPorts([ + { name: 'grpc', containerPort: tq.service.spec.ports[0].port }, + { name: 'http', containerPort: tq.service.spec.ports[1].port }, + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(30) + + container.mixin.livenessProbe.withFailureThreshold(4) + + container.mixin.livenessProbe.httpGet.withPort(tq.service.spec.ports[1].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(20) + + container.mixin.readinessProbe.httpGet.withPort(tq.service.spec.ports[1].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); - local c = - container.new($.thanos.querier.deployment.metadata.name, tq.image) + - container.withArgs([ - 'query', - '--query.replica-label=%s' % tq.replicaLabel, - '--grpc-address=0.0.0.0:%d' % $.thanos.querier.service.spec.ports[0].port, - '--http-address=0.0.0.0:%d' % $.thanos.querier.service.spec.ports[1].port, - ]) + - container.withPorts([ - { name: 'grpc', containerPort: $.thanos.querier.service.spec.ports[0].port }, - { name: 'http', containerPort: $.thanos.querier.service.spec.ports[1].port }, - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(30) + - container.mixin.livenessProbe.withFailureThreshold(4) + - container.mixin.livenessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(20) + - container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + deployment.new(tq.config.name, tq.config.replicas, c, tq.config.commonLabels) + + deployment.mixin.metadata.withNamespace(tq.config.namespace) + + deployment.mixin.metadata.withLabels(tq.config.commonLabels) + + deployment.mixin.spec.selector.withMatchLabels(tq.config.podLabelSelector) + + deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + + deployment.mixin.spec.template.spec.affinity.podAntiAffinity.withPreferredDuringSchedulingIgnoredDuringExecution([ + affinity.new() + + affinity.withWeight(100) + + affinity.mixin.podAffinityTerm.withNamespaces(tq.config.namespace) + + affinity.mixin.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') + + affinity.mixin.podAffinityTerm.labelSelector.withMatchExpressions([ + matchExpression.new() + + matchExpression.withKey('app.kubernetes.io/name') + + matchExpression.withOperator('In') + + matchExpression.withValues([tq.deployment.metadata.labels['app.kubernetes.io/name']]), + ]), + ]), - deployment.new(tq.name, tq.replicas, c, $.thanos.querier.deployment.metadata.labels) + - deployment.mixin.metadata.withNamespace(tq.namespace) + - deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.querier.deployment.metadata.name }) + - deployment.mixin.spec.selector.withMatchLabels($.thanos.querier.deployment.metadata.labels) + - deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + - deployment.mixin.spec.template.spec.affinity.podAntiAffinity.withPreferredDuringSchedulingIgnoredDuringExecution([ - affinity.new() + - affinity.withWeight(100) + - affinity.mixin.podAffinityTerm.withNamespaces(tq.namespace) + - affinity.mixin.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') + - affinity.mixin.podAffinityTerm.labelSelector.withMatchExpressions([ - matchExpression.new() + - matchExpression.withKey('app.kubernetes.io/name') + - matchExpression.withOperator('In') + - matchExpression.withValues([$.thanos.querier.deployment.metadata.labels['app.kubernetes.io/name']]), - ]), - ]), + withServiceMonitor:: { + local tq = self, + serviceMonitor: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata+: { + name: tq.config.name, + namespace: tq.config.namespace, + }, + spec: { + selector: { + matchLabels: tq.config.commonLabels, + }, + endpoints: [ + { port: 'http' }, + ], + }, }, }, } diff --git a/jsonnet/kube-thanos/kube-thanos-receive-pvc.libsonnet b/jsonnet/kube-thanos/kube-thanos-receive-pvc.libsonnet deleted file mode 100644 index a56566fc..00000000 --- a/jsonnet/kube-thanos/kube-thanos-receive-pvc.libsonnet +++ /dev/null @@ -1,45 +0,0 @@ -local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; - -{ - thanos+:: { - receive+: { - local tr = self, - pvc+:: { - class: 'standard', - size: error 'must set PVC size for Thanos receive', - }, - - statefulSet+: - local sts = k.apps.v1.statefulSet; - local pvc = sts.mixin.spec.volumeClaimTemplatesType; - - { - spec+: { - template+: { - spec+: { - volumes: null, - }, - }, - volumeClaimTemplates::: [ - { - metadata: { - name: $.thanos.receive.statefulSet.metadata.name + '-data', - }, - spec: { - accessModes: [ - 'ReadWriteOnce', - ], - storageClassName: tr.pvc.class, - resources: { - requests: { - storage: tr.pvc.size, - }, - }, - }, - }, - ], - }, - }, - }, - }, -} diff --git a/jsonnet/kube-thanos/kube-thanos-receive.libsonnet b/jsonnet/kube-thanos/kube-thanos-receive.libsonnet index 7429d583..51834b90 100644 --- a/jsonnet/kube-thanos/kube-thanos-receive.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-receive.libsonnet @@ -1,125 +1,235 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - receive: { - local tr = self, - name:: 'thanos-receive', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - replicas:: 3, - objectStorageConfig:: $.thanos.objectStorageConfig, - - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; - - service.new( - tr.name, - $.thanos.receive.statefulSet.metadata.labels, - [ - ports.newNamed('grpc', 10901, 10901), - ports.newNamed('http', 10902, 10902), - ports.newNamed('remote-write', 19291, 19291), - ] - ) + - service.mixin.metadata.withNamespace(tr.namespace) + - service.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.receive.service.metadata.name }) + - service.mixin.spec.withClusterIp('None'), - - statefulSet: - local sts = k.apps.v1.statefulSet; - local affinity = sts.mixin.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecutionType; - local matchExpression = affinity.mixin.podAffinityTerm.labelSelector.matchExpressionsType; - local volume = sts.mixin.spec.template.spec.volumesType; - local container = sts.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - local containerVolumeMount = container.volumeMountsType; - - local c = - container.new($.thanos.receive.statefulSet.metadata.name, tr.image) + - container.withArgs([ - 'receive', - '--grpc-address=0.0.0.0:%d' % $.thanos.receive.service.spec.ports[0].port, - '--http-address=0.0.0.0:%d' % $.thanos.receive.service.spec.ports[1].port, - '--remote-write.address=0.0.0.0:%d' % $.thanos.receive.service.spec.ports[2].port, - '--objstore.config=$(OBJSTORE_CONFIG)', - '--tsdb.path=/var/thanos/receive', - '--label=replica="$(NAME)"', - '--label=receive="true"', - ]) + - container.withEnv([ - containerEnv.fromFieldPath('NAME', 'metadata.name'), - containerEnv.fromSecretRef( - 'OBJSTORE_CONFIG', - tr.objectStorageConfig.name, - tr.objectStorageConfig.key, - ), - ]) + - container.withPorts([ - { name: 'grpc', containerPort: $.thanos.receive.service.spec.ports[0].port }, - { name: 'http', containerPort: $.thanos.receive.service.spec.ports[1].port }, - { name: 'remote-write', containerPort: $.thanos.receive.service.spec.ports[2].port }, - ]) + - container.withVolumeMounts([ - containerVolumeMount.new(tr.name + '-data', '/var/thanos/receive', false), - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(30) + - container.mixin.livenessProbe.withFailureThreshold(8) + - container.mixin.livenessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(20) + - container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); - - sts.new(tr.name, tr.replicas, c, [], $.thanos.receive.statefulSet.metadata.labels) + - sts.mixin.metadata.withNamespace(tr.namespace) + - sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.receive.statefulSet.metadata.name }) + - sts.mixin.spec.withServiceName($.thanos.receive.service.metadata.name) + - sts.mixin.spec.selector.withMatchLabels($.thanos.receive.statefulSet.metadata.labels) + - sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + - sts.mixin.spec.template.spec.affinity.podAntiAffinity.withPreferredDuringSchedulingIgnoredDuringExecution([ - affinity.new() + - affinity.withWeight(100) + - affinity.mixin.podAffinityTerm.withNamespaces(tr.namespace) + - affinity.mixin.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') + - affinity.mixin.podAffinityTerm.labelSelector.withMatchExpressions([ - matchExpression.new() + - matchExpression.withKey('app.kubernetes.io/name') + - matchExpression.withOperator('In') + - matchExpression.withValues([$.thanos.receive.statefulSet.metadata.labels['app.kubernetes.io/name']]), - ]), - ]) + - sts.mixin.spec.template.spec.withVolumes([ - volume.fromEmptyDir('data'), - ]) + - { + local tr = self, + + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + replicas: error 'must provide replicas', + replicationFactor: error 'must provide replication factor', + objectStorageConfig: error 'must provide objectStorageConfig', + + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': tr.config.name, + 'app.kubernetes.io/version': 'v' + tr.config.version, + 'app.kubernetes.io/component': 'thanos-receive', + }, + + podLabelSelector:: { + [labelName]: tr.config.commonLabels[labelName] + for labelName in std.objectFields(tr.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) + }, + }, + + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; + + service.new( + tr.config.name, + tr.config.podLabelSelector, + [ + ports.newNamed('grpc', 10901, 10901), + ports.newNamed('http', 10902, 10902), + ports.newNamed('remote-write', 19291, 19291), + ] + ) + + service.mixin.metadata.withNamespace(tr.config.namespace) + + service.mixin.metadata.withLabels(tr.config.commonLabels) + + service.mixin.spec.withClusterIp('None'), + + statefulSet: + local sts = k.apps.v1.statefulSet; + local affinity = sts.mixin.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecutionType; + local matchExpression = affinity.mixin.podAffinityTerm.labelSelector.matchExpressionsType; + local volume = sts.mixin.spec.template.spec.volumesType; + local container = sts.mixin.spec.template.spec.containersType; + local containerEnv = container.envType; + local containerVolumeMount = container.volumeMountsType; + + local replicationFactor = if tr.config.replicationFactor <= tr.config.replicas then tr.config.replicationFactor else error 'replication factor must be less or equal to number of replicas'; + local localEndpointFlag = '--receive.local-endpoint=http://$(NAME).%s.$(NAMESPACE).svc.cluster.local:%d/api/v1/receive' % [tr.config.name, tr.service.spec.ports[2].port]; + + local c = + container.new('thanos-receive', tr.config.image) + + container.withArgs([ + 'receive', + '--grpc-address=0.0.0.0:%d' % tr.service.spec.ports[0].port, + '--http-address=0.0.0.0:%d' % tr.service.spec.ports[1].port, + '--remote-write.address=0.0.0.0:%d' % tr.service.spec.ports[2].port, + '--receive.replication-factor=%d' % replicationFactor, + '--objstore.config=$(OBJSTORE_CONFIG)', + '--tsdb.path=/var/thanos/receive', + '--label=replica="$(NAME)"', + '--label=receive="true"', + localEndpointFlag, + ]) + + container.withEnv([ + containerEnv.fromFieldPath('NAME', 'metadata.name'), + containerEnv.fromFieldPath('NAMESPACE', 'metadata.namespace'), + containerEnv.fromSecretRef( + 'OBJSTORE_CONFIG', + tr.config.objectStorageConfig.name, + tr.config.objectStorageConfig.key, + ), + ]) + + container.withPorts([ + { name: 'grpc', containerPort: tr.service.spec.ports[0].port }, + { name: 'http', containerPort: tr.service.spec.ports[1].port }, + { name: 'remote-write', containerPort: tr.service.spec.ports[2].port }, + ]) + + container.withVolumeMounts([ + containerVolumeMount.new('data', '/var/thanos/receive', false), + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(30) + + container.mixin.livenessProbe.withFailureThreshold(8) + + container.mixin.livenessProbe.httpGet.withPort(tr.service.spec.ports[1].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(20) + + container.mixin.readinessProbe.httpGet.withPort(tr.service.spec.ports[1].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + + sts.new(tr.config.name, tr.config.replicas, c, [], tr.config.commonLabels) + + sts.mixin.metadata.withNamespace(tr.config.namespace) + + sts.mixin.metadata.withLabels(tr.config.commonLabels) + + sts.mixin.spec.withServiceName(tr.service.metadata.name) + + sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + + sts.mixin.spec.template.spec.withVolumes([ + volume.fromEmptyDir('data'), + ]) + + sts.mixin.spec.template.spec.affinity.podAntiAffinity.withPreferredDuringSchedulingIgnoredDuringExecution([ + affinity.new() + + affinity.withWeight(100) + + affinity.mixin.podAffinityTerm.withNamespaces(tr.config.namespace) + + affinity.mixin.podAffinityTerm.withTopologyKey('kubernetes.io/hostname') + + affinity.mixin.podAffinityTerm.labelSelector.withMatchExpressions([ + matchExpression.new() + + matchExpression.withKey('app.kubernetes.io/instance') + + matchExpression.withOperator('In') + + matchExpression.withValues([tr.statefulSet.metadata.labels['app.kubernetes.io/instance']]), + ]), + ]) + + sts.mixin.spec.selector.withMatchLabels(tr.config.podLabelSelector) + + { + spec+: { + volumeClaimTemplates: null, + }, + }, + + withServiceMonitor:: { + local tr = self, + serviceMonitor: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata+: { + name: tr.config.name, + namespace: tr.config.namespace, + }, + spec: { + selector: { + matchLabels: tr.config.commonLabels, + }, + endpoints: [ + { port: 'http' }, + ], + }, + }, + }, + + withPodDisruptionBudget:: { + local tr = self, + podDisruptionBudget: + local pdb = k.policy.v1beta1.podDisruptionBudget; + pdb.new() + + pdb.mixin.spec.withMinAvailable(tr.config.replicas - (std.floor(tr.config.replicationFactor / 2))) + + pdb.mixin.spec.selector.withMatchLabels(tr.config.podLabelSelector) + + pdb.mixin.metadata.withName(tr.config.name) + + pdb.mixin.metadata.withNamespace(tr.config.namespace), + }, + + withVolumeClaimTemplate:: { + local tr = self, + config+:: { + volumeClaimTemplate: error 'must provide volumeClaimTemplate', + }, + + statefulSet+: { + spec+: { + template+: { + spec+: { + volumes: null, + }, + }, + volumeClaimTemplates: [tr.config.volumeClaimTemplate { + metadata+: { + name: 'data', + }, + }], + }, + }, + }, + + withRetention:: { + local tr = self, + config+:: { + retention: error 'must provide retention', + }, + + statefulSet+: { + spec+: { + template+: { spec+: { - volumeClaimTemplates:: null, + containers: [ + if c.name == 'thanos-receive' then c { + args+: [ + '--tsdb.retention=' + tr.config.retention, + ], + } else c + for c in super.containers + ], }, }, + }, }, + }, + + withHashringConfigMap:: { + local tr = self, + config+:: { + hashringConfigMapName: error 'must provide hashringConfigMapName', + }, + + statefulSet+: { + spec+: { + template+: { + spec+: { + containers: [ + if c.name == 'thanos-receive' then c { + args+: [ + '--receive.hashrings-file=/var/lib/thanos-receive/hashrings.json', + ], + volumeMounts+: [ + { name: 'hashring-config', mountPath: '/var/lib/thanos-receive' }, + ], + } else c + for c in super.containers + ], - querier+: { - deployment+: { - spec+: { - template+: { - spec+: { - containers: [ - super.containers[0] - { args+: [ - '--store=dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [ - $.thanos.receive.service.metadata.name, - $.thanos.receive.service.metadata.namespace, - ], - ] }, - ], - }, + local volume = k.apps.v1.statefulSet.mixin.spec.template.spec.volumesType, + volumes: [ + volume.withName('hashring-config') + + volume.mixin.configMap.withName(tr.config.hashringConfigMapName), + ], }, }, }, diff --git a/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet b/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet index 04c29de7..35495115 100644 --- a/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-ruler.libsonnet @@ -1,130 +1,151 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - namespace:: 'monitoring', - image:: error 'must set thanos image', + local tr = self, - ruler+: { - local tr = self, - name:: 'thanos-ruler', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - replicas:: 1, - labels+:: { - 'app.kubernetes.io/name': tr.name, - }, - ports:: { - grpc: 10901, - http: 10902, - }, - objectStorageConfig:: $.thanos.objectStorageConfig, - ruleFiles:: [], - alertmanagersURLs:: [], + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + replicas: error 'must provide replicas', + objectStorageConfig: error 'must provide objectStorageConfig', + ruleFiles: [], + alertmanagersURLs: [], + queriers: [], - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': tr.config.name, + 'app.kubernetes.io/version': 'v' + tr.config.version, + 'app.kubernetes.io/component': 'thanos-ruler', + }, - service.new( - tr.name, - tr.labels, - [ - ports.newNamed('grpc', tr.ports.grpc, 'grpc'), - ports.newNamed('http', tr.ports.http, 'http'), - ], - ) + - service.mixin.metadata.withNamespace(tr.namespace) + - service.mixin.metadata.withLabels(tr.labels) + - service.mixin.spec.withClusterIp('None'), + podLabelSelector:: { + [labelName]: tr.config.commonLabels[labelName] + for labelName in std.objectFields(tr.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) + }, + }, - statefulSet: - local statefulSet = k.apps.v1.statefulSet; - local volume = statefulSet.mixin.spec.template.spec.volumesType; - local container = statefulSet.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - local containerVolumeMount = container.volumeMountsType; + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; - local c = - container.new(tr.name, tr.image) + - container.withArgs( - [ - 'rule', - '--grpc-address=0.0.0.0:%d' % tr.ports.grpc, - '--http-address=0.0.0.0:%d' % tr.ports.http, - '--objstore.config=$(OBJSTORE_CONFIG)', - '--data-dir=/var/thanos/ruler', - '--label=ruler_replica="$(NAME)"', - '--alert.label-drop="ruler_replica"', - '--query=dnssrv+_http._tcp.%s.%s.svc.cluster.local' % [ - $.thanos.querier.service.metadata.name, - $.thanos.querier.service.metadata.namespace, - ], - ] + - (['--rule-file=%s' % path for path in tr.ruleFiles]) + - (['--alertmanagers.url=%s' % url for url in tr.alertmanagersURLs]) - ) + - container.withEnv([ - containerEnv.fromFieldPath('NAME', 'metadata.name'), - containerEnv.fromSecretRef( - 'OBJSTORE_CONFIG', - tr.objectStorageConfig.name, - tr.objectStorageConfig.key, - ), - ]) + - container.withVolumeMounts([ - containerVolumeMount.new('thanos-ruler-data', '/var/thanos/ruler', false), - ]) + - container.withPorts([ - { name: 'grpc', containerPort: tr.ports.grpc }, - { name: 'http', containerPort: tr.ports.http }, - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(5) + - container.mixin.livenessProbe.withFailureThreshold(24) + - container.mixin.livenessProbe.httpGet.withPort(tr.ports.http) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withInitialDelaySeconds(10) + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(18) + - container.mixin.readinessProbe.httpGet.withPort(tr.ports.http) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + service.new( + tr.config.name, + tr.config.podLabelSelector, + [ + ports.newNamed('grpc', 10901, 'grpc'), + ports.newNamed('http', 10902, 'http'), + ], + ) + + service.mixin.metadata.withNamespace(tr.config.namespace) + + service.mixin.metadata.withLabels(tr.config.commonLabels) + + service.mixin.spec.withClusterIp('None'), - statefulSet.new(tr.name, tr.replicas, c, [], $.thanos.ruler.statefulSet.metadata.labels) + - statefulSet.mixin.metadata.withNamespace(tr.namespace) + - statefulSet.mixin.metadata.withLabels({ 'app.kubernetes.io/name': tr.name }) + - statefulSet.mixin.spec.withServiceName($.thanos.ruler.service.metadata.name) + - statefulSet.mixin.spec.selector.withMatchLabels(tr.labels) + - statefulSet.mixin.spec.template.spec.withVolumes([ - volume.fromEmptyDir('thanos-ruler-data'), - ]) + { - spec+: { - volumeClaimTemplates:: null, - }, + statefulSet: + local statefulSet = k.apps.v1.statefulSet; + local volume = statefulSet.mixin.spec.template.spec.volumesType; + local container = statefulSet.mixin.spec.template.spec.containersType; + local containerEnv = container.envType; + local containerVolumeMount = container.volumeMountsType; + + local c = + container.new('thanos-ruler', tr.config.image) + + container.withArgs( + [ + 'rule', + '--grpc-address=0.0.0.0:%d' % tr.service.spec.ports[0].port, + '--http-address=0.0.0.0:%d' % tr.service.spec.ports[1].port, + '--objstore.config=$(OBJSTORE_CONFIG)', + '--data-dir=/var/thanos/ruler', + '--label=ruler_replica="$(NAME)"', + '--alert.label-drop="ruler_replica"', + ] + + (['--query=%s' % querier for querier in tr.config.queriers]) + + (['--rule-file=%s' % path for path in tr.config.ruleFiles]) + + (['--alertmanagers.url=%s' % url for url in tr.config.alertmanagersURLs]) + ) + + container.withEnv([ + containerEnv.fromFieldPath('NAME', 'metadata.name'), + containerEnv.fromSecretRef( + 'OBJSTORE_CONFIG', + tr.config.objectStorageConfig.name, + tr.config.objectStorageConfig.key, + ), + ]) + + container.withVolumeMounts([ + containerVolumeMount.new('data', '/var/thanos/ruler', false), + ]) + + container.withPorts([ + { name: 'grpc', containerPort: tr.service.spec.ports[0].port }, + { name: 'http', containerPort: tr.service.spec.ports[1].port }, + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(5) + + container.mixin.livenessProbe.withFailureThreshold(24) + + container.mixin.livenessProbe.httpGet.withPort(tr.service.spec.ports[1].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withInitialDelaySeconds(10) + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(18) + + container.mixin.readinessProbe.httpGet.withPort(tr.service.spec.ports[1].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + + statefulSet.new(tr.config.name, tr.config.replicas, c, [], tr.config.commonLabels) + + statefulSet.mixin.metadata.withNamespace(tr.config.namespace) + + statefulSet.mixin.metadata.withLabels(tr.config.commonLabels) + + statefulSet.mixin.spec.withServiceName(tr.service.metadata.name) + + statefulSet.mixin.spec.selector.withMatchLabels(tr.config.podLabelSelector) + + statefulSet.mixin.spec.template.spec.withVolumes([ + volume.fromEmptyDir('data'), + ]) + { + spec+: { + volumeClaimTemplates: null, + }, + }, + + withServiceMonitor:: { + local tr = self, + serviceMonitor: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata+: { + name: tr.config.name, + namespace: tr.config.namespace, + }, + spec: { + selector: { + matchLabels: tr.config.commonLabels, }, + endpoints: [ + { port: 'http' }, + ], + }, }, + }, - querier+: { - deployment+: { - spec+: { - template+: { - spec+: { - containers: [ - super.containers[0] - { args+: [ - '--query.replica-label=ruler_replica', - '--store=dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [ - $.thanos.ruler.service.metadata.name, - $.thanos.ruler.service.metadata.namespace, - ], - ] }, - ], - }, + withVolumeClaimTemplate:: { + local tr = self, + config+:: { + volumeClaimTemplate: error 'must provide volumeClaimTemplate', + }, + statefulSet+: { + spec+: { + template+: { + spec+: { + volumes: null, }, }, + volumeClaimTemplates: [tr.config.volumeClaimTemplate { + metadata+: { + name: 'data', + }, + }], }, }, }, diff --git a/jsonnet/kube-thanos/kube-thanos-servicemonitors.libsonnet b/jsonnet/kube-thanos/kube-thanos-servicemonitors.libsonnet deleted file mode 100644 index 9f3eafb4..00000000 --- a/jsonnet/kube-thanos/kube-thanos-servicemonitors.libsonnet +++ /dev/null @@ -1,94 +0,0 @@ -{ - thanos+:: { - querier+: { - serviceMonitor+: { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata+: { - name: $.thanos.querier.name, - namespace: $.thanos.querier.namespace, - }, - spec: { - selector: { - matchLabels: $.thanos.querier.service.metadata.labels, - }, - endpoints: [ - { port: 'http' }, - ], - }, - }, - }, - store+: { - serviceMonitor+: { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata+: { - name: $.thanos.store.name, - namespace: $.thanos.store.namespace, - }, - spec: { - selector: { - matchLabels: $.thanos.store.service.metadata.labels, - }, - endpoints: [ - { port: 'http' }, - ], - }, - }, - }, - receive+: { - serviceMonitor+: { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata+: { - name: $.thanos.receive.name, - namespace: $.thanos.receive.namespace, - }, - spec: { - selector: { - matchLabels: $.thanos.receive.service.metadata.labels, - }, - endpoints: [ - { port: 'http' }, - ], - }, - }, - }, - compactor+: { - serviceMonitor+: { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata+: { - name: $.thanos.compactor.name, - namespace: $.thanos.compactor.namespace, - }, - spec: { - selector: { - matchLabels: $.thanos.compactor.service.metadata.labels, - }, - endpoints: [ - { port: 'http' }, - ], - }, - }, - }, - ruler+: { - serviceMonitor+: { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - metadata+: { - name: $.thanos.ruler.name, - namespace: $.thanos.ruler.namespace, - }, - spec: { - selector: { - matchLabels: $.thanos.ruler.service.metadata.labels, - }, - endpoints: [ - { port: 'http' }, - ], - }, - }, - }, - }, -} diff --git a/jsonnet/kube-thanos/kube-thanos-sidecar.libsonnet b/jsonnet/kube-thanos/kube-thanos-sidecar.libsonnet deleted file mode 100644 index a4605ce1..00000000 --- a/jsonnet/kube-thanos/kube-thanos-sidecar.libsonnet +++ /dev/null @@ -1,32 +0,0 @@ -// This example adds the --store flag to the querier to find sidecars. -// The used values are the defaults in kube-prometheus. -// If you need to modify this, you most likely want to -// copy this file into your project and change it accordingly. - -{ - thanos+:: { - querier+: { - local tq = self, - sidecarName:: 'prometheus-k8s', - sidecarNamespace:: tq.namespace, - - deployment+: { - spec+: { - template+: { - spec+: { - containers: [ - super.containers[0] - { args+: [ - '--store=dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [ - tq.sidecarName, - tq.sidecarNamespace, - ], - ] }, - ], - }, - }, - }, - }, - }, - }, -} diff --git a/jsonnet/kube-thanos/kube-thanos-store-pvc.libsonnet b/jsonnet/kube-thanos/kube-thanos-store-pvc.libsonnet deleted file mode 100644 index 9b4a1142..00000000 --- a/jsonnet/kube-thanos/kube-thanos-store-pvc.libsonnet +++ /dev/null @@ -1,45 +0,0 @@ -local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; - -{ - thanos+:: { - store+: { - local ts = self, - pvc+:: { - class: 'standard', - size: error 'must set PVC size for Thanos store', - }, - - statefulSet+: - local sts = k.apps.v1.statefulSet; - local pvc = sts.mixin.spec.volumeClaimTemplatesType; - - { - spec+: { - template+: { - spec+: { - volumes: null, - }, - }, - volumeClaimTemplates::: [ - { - metadata: { - name: $.thanos.store.statefulSet.metadata.name + '-data', - }, - spec: { - accessModes: [ - 'ReadWriteOnce', - ], - storageClassName: ts.pvc.class, - resources: { - requests: { - storage: ts.pvc.size, - }, - }, - }, - }, - ], - }, - }, - }, - }, -} diff --git a/jsonnet/kube-thanos/kube-thanos-store.libsonnet b/jsonnet/kube-thanos/kube-thanos-store.libsonnet index e7e6c70c..ee851ebf 100644 --- a/jsonnet/kube-thanos/kube-thanos-store.libsonnet +++ b/jsonnet/kube-thanos/kube-thanos-store.libsonnet @@ -1,113 +1,141 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet'; { - thanos+:: { - namespace:: 'monitoring', - objectStorageConfig+:: { - name: error 'must set an object storage secret name', - key: error 'must set an object storage secret key', + local ts = self, + + config:: { + name: error 'must provide name', + namespace: error 'must provide namespace', + version: error 'must provide version', + image: error 'must provide image', + replicas: error 'must provide replicas', + objectStorageConfig: error 'must provide objectStorageConfig', + + commonLabels:: { + 'app.kubernetes.io/name': 'thanos', + 'app.kubernetes.io/instance': ts.config.name, + 'app.kubernetes.io/version': 'v' + ts.config.version, + 'app.kubernetes.io/component': 'thanos-store', }, - store: { - local ts = self, - name:: 'thanos-store', - namespace:: $.thanos.namespace, - image:: $.thanos.image, - replicas:: 1, - objectStorageConfig:: $.thanos.objectStorageConfig, + podLabelSelector:: { + [labelName]: ts.config.commonLabels[labelName] + for labelName in std.objectFields(ts.config.commonLabels) + if !std.setMember(labelName, ['app.kubernetes.io/version']) + }, + }, - service: - local service = k.core.v1.service; - local ports = service.mixin.spec.portsType; + service: + local service = k.core.v1.service; + local ports = service.mixin.spec.portsType; - service.new( - ts.name, - $.thanos.store.statefulSet.metadata.labels, - [ - ports.newNamed('grpc', 10901, 10901), - ports.newNamed('http', 10902, 10902), - ] - ) + - service.mixin.metadata.withNamespace(ts.namespace) + - service.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.store.service.metadata.name }) + - service.mixin.spec.withClusterIp('None'), + service.new( + ts.config.name, + ts.config.podLabelSelector, + [ + ports.newNamed('grpc', 10901, 10901), + ports.newNamed('http', 10902, 10902), + ] + ) + + service.mixin.metadata.withNamespace(ts.config.namespace) + + service.mixin.metadata.withLabels(ts.config.commonLabels) + + service.mixin.spec.withClusterIp('None'), - statefulSet: - local sts = k.apps.v1.statefulSet; - local volume = sts.mixin.spec.template.spec.volumesType; - local container = sts.mixin.spec.template.spec.containersType; - local containerEnv = container.envType; - local containerVolumeMount = container.volumeMountsType; + statefulSet: + local sts = k.apps.v1.statefulSet; + local volume = sts.mixin.spec.template.spec.volumesType; + local container = sts.mixin.spec.template.spec.containersType; + local containerEnv = container.envType; + local containerVolumeMount = container.volumeMountsType; - local c = - container.new(ts.name, ts.image) + - container.withArgs([ - 'store', - '--data-dir=/var/thanos/store', - '--grpc-address=0.0.0.0:%d' % $.thanos.store.service.spec.ports[0].port, - '--http-address=0.0.0.0:%d' % $.thanos.store.service.spec.ports[1].port, - '--objstore.config=$(OBJSTORE_CONFIG)', - ]) + - container.withEnv([ - containerEnv.fromSecretRef( - 'OBJSTORE_CONFIG', - ts.objectStorageConfig.name, - ts.objectStorageConfig.key, - ), - ]) + - container.withPorts([ - { name: 'grpc', containerPort: $.thanos.store.service.spec.ports[0].port }, - { name: 'http', containerPort: $.thanos.store.service.spec.ports[1].port }, - ]) + - container.withVolumeMounts([ - containerVolumeMount.new(ts.name + '-data', '/var/thanos/store', false), - ]) + - container.mixin.livenessProbe + - container.mixin.livenessProbe.withPeriodSeconds(30) + - container.mixin.livenessProbe.withFailureThreshold(8) + - container.mixin.livenessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) + - container.mixin.livenessProbe.httpGet.withScheme('HTTP') + - container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + - container.mixin.readinessProbe + - container.mixin.readinessProbe.withPeriodSeconds(5) + - container.mixin.readinessProbe.withFailureThreshold(20) + - container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) + - container.mixin.readinessProbe.httpGet.withScheme('HTTP') + - container.mixin.readinessProbe.httpGet.withPath('/-/ready'); + local c = + container.new(ts.config.name, ts.config.image) + + container.withArgs([ + 'store', + '--data-dir=/var/thanos/store', + '--grpc-address=0.0.0.0:%d' % ts.service.spec.ports[0].port, + '--http-address=0.0.0.0:%d' % ts.service.spec.ports[1].port, + '--objstore.config=$(OBJSTORE_CONFIG)', + ]) + + container.withEnv([ + containerEnv.fromSecretRef( + 'OBJSTORE_CONFIG', + ts.config.objectStorageConfig.name, + ts.config.objectStorageConfig.key, + ), + ]) + + container.withPorts([ + { name: 'grpc', containerPort: ts.service.spec.ports[0].port }, + { name: 'http', containerPort: ts.service.spec.ports[1].port }, + ]) + + container.withVolumeMounts([ + containerVolumeMount.new('data', '/var/thanos/receive', false), + ]) + + container.mixin.livenessProbe + + container.mixin.livenessProbe.withPeriodSeconds(30) + + container.mixin.livenessProbe.withFailureThreshold(8) + + container.mixin.livenessProbe.httpGet.withPort(ts.service.spec.ports[1].port) + + container.mixin.livenessProbe.httpGet.withScheme('HTTP') + + container.mixin.livenessProbe.httpGet.withPath('/-/healthy') + + container.mixin.readinessProbe + + container.mixin.readinessProbe.withPeriodSeconds(5) + + container.mixin.readinessProbe.withFailureThreshold(20) + + container.mixin.readinessProbe.httpGet.withPort(ts.service.spec.ports[1].port) + + container.mixin.readinessProbe.httpGet.withScheme('HTTP') + + container.mixin.readinessProbe.httpGet.withPath('/-/ready'); - sts.new(ts.name, ts.replicas, c, [], $.thanos.store.statefulSet.metadata.labels) + - sts.mixin.metadata.withNamespace(ts.namespace) + - sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': ts.name }) + - sts.mixin.spec.withServiceName($.thanos.store.service.metadata.name) + - sts.mixin.spec.selector.withMatchLabels($.thanos.store.statefulSet.metadata.labels) + - sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + - sts.mixin.spec.template.spec.withVolumes([ - volume.fromEmptyDir(ts.name + '-data'), - ]) + - { - spec+: { - volumeClaimTemplates:: null, - }, + sts.new(ts.config.name, ts.config.replicas, c, [], ts.config.commonLabels) + + sts.mixin.metadata.withNamespace(ts.config.namespace) + + sts.mixin.metadata.withLabels(ts.config.commonLabels) + + sts.mixin.spec.withServiceName(ts.service.metadata.name) + + sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + + sts.mixin.spec.template.spec.withVolumes([ + volume.fromEmptyDir('data'), + ]) + + sts.mixin.spec.selector.withMatchLabels(ts.config.podLabelSelector) + + { + spec+: { + volumeClaimTemplates: null, + }, + }, + + withServiceMonitor:: { + local ts = self, + serviceMonitor: { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + metadata+: { + name: ts.config.name, + namespace: ts.config.namespace, + }, + spec: { + selector: { + matchLabels: ts.config.commonLabels, }, + endpoints: [ + { port: 'http' }, + ], + }, }, + }, - querier+: { - deployment+: { - spec+: { - template+: { - spec+: { - containers: [ - super.containers[0] - { args+: [ - '--store=dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [ - $.thanos.store.service.metadata.name, - $.thanos.store.service.metadata.namespace, - ], - ] }, - ], - }, + withVolumeClaimTemplate:: { + local ts = self, + config+:: { + volumeClaimTemplate: error 'must provide volumeClaimTemplate', + }, + statefulSet+: { + spec+: { + template+: { + spec+: { + volumes: null, }, }, + volumeClaimTemplates: [ts.config.volumeClaimTemplate { + metadata+: { + name: 'data', + }, + }], }, }, }, diff --git a/jsonnet/kube-thanos/thanos.libsonnet b/jsonnet/kube-thanos/thanos.libsonnet new file mode 100644 index 00000000..2a8df86b --- /dev/null +++ b/jsonnet/kube-thanos/thanos.libsonnet @@ -0,0 +1,8 @@ +{ + bucket: (import 'kube-thanos-bucket.libsonnet'), + compactor: (import 'kube-thanos-compactor.libsonnet'), + querier: (import 'kube-thanos-querier.libsonnet'), + receive: (import 'kube-thanos-receive.libsonnet'), + ruler: (import 'kube-thanos-ruler.libsonnet'), + store: (import 'kube-thanos-store.libsonnet'), +} diff --git a/jsonnetfile.lock.json b/jsonnetfile.lock.json index bbeb9e3a..d39edbf6 100644 --- a/jsonnetfile.lock.json +++ b/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "grafana-builder" } }, - "version": "f4c59f64f80442f871a06c91edf74d014b82acaf", - "sum": "ELsYwK+kGdzX1mee2Yy+/b2mdO4Y503BOCDkFzwmGbE=" + "version": "e92000a708ded5bde1dd332a8e7457d0b39b7928", + "sum": "slxrtftVDiTlQK22ertdfrg4Epnq97gdrLI63ftUfaE=" }, { "name": "grafonnet", @@ -19,8 +19,8 @@ "subdir": "grafonnet" } }, - "version": "69bc267211790a1c3f4ea6e6211f3e8ffe22f987", - "sum": "BjHfWzqSAgtAKEVD6ipoYOkb8XT5wSBIboY4ZLwhlOU=" + "version": "c459106d2d2b583dd3a83f6c75eb52abee3af764", + "sum": "CeM3LRgUCUJTolTdMnerfMPGYmhClx7gX5ajrQVEY2Y=" }, { "name": "ksonnet", diff --git a/manifests/thanos-querier-deployment.yaml b/manifests/thanos-querier-deployment.yaml index 64337613..49c9d84e 100644 --- a/manifests/thanos-querier-deployment.yaml +++ b/manifests/thanos-querier-deployment.yaml @@ -2,18 +2,26 @@ apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/name: thanos-querier - name: thanos-querier - namespace: monitoring + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 + name: thanos-query + namespace: thanos spec: - replicas: 3 + replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos template: metadata: labels: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: affinity: podAntiAffinity: @@ -24,19 +32,20 @@ spec: - key: app.kubernetes.io/name operator: In values: - - thanos-querier + - thanos namespaces: - - monitoring + - thanos topologyKey: kubernetes.io/hostname weight: 100 containers: - args: - query - - --query.replica-label=prometheus_replica - --grpc-address=0.0.0.0:10901 - --http-address=0.0.0.0:9090 - - --store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc.cluster.local - image: quay.io/thanos/thanos:v0.9.0 + - --query.replica-label=prometheus_replica + - --query.replica-label=ruler_replica + - --store=dnssrv+_grpc._tcp.thanos-store.thanos.svc.cluster.local + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 4 httpGet: @@ -44,7 +53,7 @@ spec: port: 9090 scheme: HTTP periodSeconds: 30 - name: thanos-querier + name: thanos-query ports: - containerPort: 10901 name: grpc diff --git a/manifests/thanos-querier-service.yaml b/manifests/thanos-querier-service.yaml index 5ca6108b..700c6a64 100644 --- a/manifests/thanos-querier-service.yaml +++ b/manifests/thanos-querier-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-querier - name: thanos-querier - namespace: monitoring + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 + name: thanos-query + namespace: thanos spec: ports: - name: grpc @@ -14,4 +17,6 @@ spec: port: 9090 targetPort: http selector: - app.kubernetes.io/name: thanos-querier + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos diff --git a/manifests/thanos-querier-serviceMonitor.yaml b/manifests/thanos-querier-serviceMonitor.yaml new file mode 100644 index 00000000..8d032b01 --- /dev/null +++ b/manifests/thanos-querier-serviceMonitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: thanos-query + namespace: thanos +spec: + endpoints: + - port: http + selector: + matchLabels: + app.kubernetes.io/component: thanos-querier + app.kubernetes.io/instance: thanos-query + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/manifests/thanos-store-service.yaml b/manifests/thanos-store-service.yaml index 10a762ca..3e27d984 100644 --- a/manifests/thanos-store-service.yaml +++ b/manifests/thanos-store-service.yaml @@ -2,9 +2,12 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-store - namespace: monitoring + namespace: thanos spec: clusterIP: None ports: @@ -15,4 +18,6 @@ spec: port: 10902 targetPort: 10902 selector: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos diff --git a/manifests/thanos-store-serviceMonitor.yaml b/manifests/thanos-store-serviceMonitor.yaml new file mode 100644 index 00000000..8d3a4bc3 --- /dev/null +++ b/manifests/thanos-store-serviceMonitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: thanos-store + namespace: thanos +spec: + endpoints: + - port: http + selector: + matchLabels: + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 diff --git a/manifests/thanos-store-statefulSet.yaml b/manifests/thanos-store-statefulSet.yaml index d93c80d8..4a7e3307 100644 --- a/manifests/thanos-store-statefulSet.yaml +++ b/manifests/thanos-store-statefulSet.yaml @@ -2,19 +2,27 @@ apiVersion: apps/v1 kind: StatefulSet metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 name: thanos-store - namespace: monitoring + namespace: thanos spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos serviceName: thanos-store template: metadata: labels: - app.kubernetes.io/name: thanos-store + app.kubernetes.io/component: thanos-store + app.kubernetes.io/instance: thanos-store + app.kubernetes.io/name: thanos + app.kubernetes.io/version: v0.10.1 spec: containers: - args: @@ -29,7 +37,7 @@ spec: secretKeyRef: key: thanos.yaml name: thanos-objectstorage - image: quay.io/thanos/thanos:v0.9.0 + image: quay.io/thanos/thanos:v0.10.1 livenessProbe: failureThreshold: 8 httpGet: @@ -51,10 +59,17 @@ spec: scheme: HTTP periodSeconds: 5 volumeMounts: - - mountPath: /var/thanos/store - name: thanos-store-data + - mountPath: /var/thanos/receive + name: data readOnly: false terminationGracePeriodSeconds: 120 - volumes: - - emptyDir: {} - name: thanos-store-data + volumes: null + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi