Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[development] Add HA Prometheus lab setup for dev M3 docker compose deployment #2206

Merged
merged 7 commits into from
Mar 12, 2020
Merged

[development] Add HA Prometheus lab setup for dev M3 docker compose deployment #2206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
06290a0
[development] Add HA Prometheus lab deployment for dev M3 docker comp…
robskillington Mar 12, 2020
97a620b
Merge branch 'master' into r/start-m3-improvements-for-ha-prometheus-…
robskillington Mar 12, 2020
7bb64d4
Address feedback
robskillington Mar 12, 2020
ca210fe
Update readme and use single prometheus config, add ability to add se…
robskillington Mar 12, 2020
3fd216e
Update
robskillington Mar 12, 2020
4c67f88
Setup placement for two aggregators if using HA aggregators
robskillington Mar 12, 2020
8faed8a
Remove downsample stanza from non-aggregator config
robskillington Mar 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,14 @@ install-vendor-m3:
install-vendor-m3-remove-bad-dep:
([ -d $(VENDOR)/$(bad_trace_dep) ] && rm -rf $(VENDOR)/$(bad_trace_dep)) || (echo "No bad trace dep" > /dev/null)

.PHONY: docker-dev-prep
docker-dev-prep:
mkdir -p ./bin/config

# Hacky way to find all configs and put into ./bin/config/
find ./src | fgrep config | fgrep ".yml" | xargs -I{} cp {} ./bin/config/
find ./src | fgrep config | fgrep ".yaml" | xargs -I{} cp {} ./bin/config/

define SERVICE_RULES

.PHONY: $(SERVICE)
Expand All @@ -141,11 +149,7 @@ $(SERVICE)-linux-amd64:

.PHONY: $(SERVICE)-docker-dev
$(SERVICE)-docker-dev: clean-build $(SERVICE)-linux-amd64
mkdir -p ./bin/config

# Hacky way to find all configs and put into ./bin/config/
find ./src | fgrep config | fgrep ".yml" | xargs -I{} cp {} ./bin/config/
find ./src | fgrep config | fgrep ".yaml" | xargs -I{} cp {} ./bin/config/
make docker-dev-prep

# Build development docker image
docker build -t $(SERVICE):dev -t quay.io/m3dbtest/$(SERVICE):dev-$(USER) -f ./docker/$(SERVICE)/development.Dockerfile ./bin
Expand Down
2 changes: 2 additions & 0 deletions docker/grafana/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ COPY ./integrations/grafana/m3query_dashboard.json /tmp/grafana_dashboards/m3que
COPY ./integrations/grafana/m3coordinator_dashboard.json /tmp/grafana_dashboards/m3coordinator_dashboard.json
COPY ./integrations/grafana/m3db_dashboard.json /tmp/grafana_dashboards/m3db_dashboard.json
COPY ./integrations/grafana/temporal_function_comparison.json /tmp/grafana_dashboards/temporal_function_comparison.json
COPY ./integrations/grafana/m3aggregator_dashboard.json /tmp/grafana_dashboards/m3aggregator_dashboard.json
COPY ./integrations/grafana/m3aggregator_end_to_end_details.json /tmp/grafana_dashboards/m3aggregator_end_to_end_details.json

# Need to replace datasource template variable with name of actual data source so auto-import
# JustWorksTM. Use a temporary directory to host the dashboards since the default
Expand Down
2 changes: 1 addition & 1 deletion docker/m3aggregator/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ RUN cd /go/src/github.com/m3db/m3/ && \
FROM alpine:latest
LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"

EXPOSE 5000/tcp 6000/tcp 60001/tcp 7203/tcp 9000-9004/tcp
EXPOSE 5000/tcp 6000/tcp 6001/tcp

RUN apk add --no-cache curl jq

Expand Down
10 changes: 10 additions & 0 deletions docker/m3aggregator/development.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM alpine:latest
LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"

EXPOSE 5000/tcp 6000/tcp 6001/tcp

ADD ./m3aggregator /bin/m3aggregator
ADD ./config/m3aggregator.yml /etc/m3aggregator/m3aggregator.yml

ENTRYPOINT [ "/bin/m3aggregator" ]
CMD [ "-f", "/etc/m3aggregator/m3aggregator.yml" ]
10 changes: 10 additions & 0 deletions docker/m3collector/development.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM alpine:latest
LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"

EXPOSE 7206/tcp 7207/tcp

ADD ./m3collector /bin/m3collector
ADD ./config/m3collector.yml /etc/m3collector/m3collector.yml

ENTRYPOINT [ "/bin/m3collector" ]
CMD [ "-f", "/etc/m3collector/m3collector.yml" ]
10 changes: 10 additions & 0 deletions docker/m3query/development.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM alpine:latest
LABEL maintainer="The M3DB Authors <m3db@googlegroups.com>"

EXPOSE 7201/tcp 7203/tcp

ADD ./m3query /bin/m3query
ADD ./config/m3query-local-etcd.yml /etc/m3query/m3query.yml

ENTRYPOINT [ "/bin/m3query" ]
CMD [ "-f", "/etc/m3query/m3query.yml" ]
78 changes: 50 additions & 28 deletions integrations/grafana/m3aggregator_dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,30 @@
{
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "5.2.4"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": "5.0.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "5.0.0"
},
{
"type": "panel",
"id": "singlestat",
"name": "Singlestat",
"version": "5.0.0"
}
],
"annotations": {
"list": [
{
Expand All @@ -15,13 +41,12 @@
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"iteration": 1575560854098,
"iteration": 1582905705847,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -952,7 +977,7 @@
},
{
"collapsed": false,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -1319,7 +1344,7 @@
},
{
"collapsed": false,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -2077,7 +2102,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -2648,7 +2673,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -3210,7 +3235,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -3586,7 +3611,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -4142,7 +4167,7 @@
},
{
"collapsed": false,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -5160,7 +5185,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -5736,7 +5761,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -6386,7 +6411,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -7451,7 +7476,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -8771,7 +8796,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -9237,7 +9262,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -9433,7 +9458,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -9993,7 +10018,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -10543,7 +10568,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -10829,7 +10854,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -10952,7 +10977,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -11058,7 +11083,7 @@
},
{
"collapsed": true,
"datasource": null,
"datasource": "$datasource",
"gridPos": {
"h": 1,
"w": 24,
Expand Down Expand Up @@ -11697,14 +11722,11 @@
"list": [
{
"current": {
"tags": [],
"text": "M3DB",
"value": "M3DB"
"text": "M3Query - Prometheus",
"value": "M3Query - Prometheus"
},
"hide": 0,
"includeAll": false,
"label": null,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
Expand Down Expand Up @@ -11895,7 +11917,7 @@
"$__all"
]
},
"datasource": "M3DB",
"datasource": "$datasource",
"definition": "m3aggregator_aggregator_flush_handler_placement_update{backend=\"m3msg\"}",
"hide": 0,
"includeAll": true,
Expand Down Expand Up @@ -11925,7 +11947,7 @@
"$__all"
]
},
"datasource": "M3DB",
"datasource": "$datasource",
"definition": "m3aggregator_aggregator_flush_handler_placement_update{backend=\"m3msg\"}",
"hide": 0,
"includeAll": true,
Expand Down
26 changes: 26 additions & 0 deletions integrations/grafana/m3aggregator_end_to_end_details.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,30 @@
{
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "5.2.4"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": "5.0.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "5.0.0"
},
{
"type": "panel",
"id": "singlestat",
"name": "Singlestat",
"version": "5.0.0"
}
],
"annotations": {
"list": [
{
Expand Down
Loading