Support multiple monitor nodes (pingcap#1137)

* Support multiple monitor nodes

roles/check_config_static/tasks/main.yml

@@ -1,9 +1,9 @@
 ---
 # Common Tasks
 
-- name: Ensure only one monitoring host exists
-  fail: msg="One monitoring host should be specified."
-  when: groups['monitoring_servers'] | length != 1
+- name: Ensure monitoring_servers exists
+  fail: msg="monitoring_servers should be specified."
+  when: groups['monitoring_servers'] | length < 1
 
 - name: Ensure monitored_servers exists
   fail: msg="monitored_servers should be specified."

roles/grafana/templates/data_source.json.j2

@@ -1,9 +1,20 @@
-{% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) if groups.monitoring_servers else "" -%}
-{% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port if groups.monitoring_servers else ''-%}
+{% if groups.monitoring_servers | length == groups.grafana_servers | length -%}
+  {% set index = [] -%}
+  {% for host in groups.monitoring_servers -%}
+    {% if inventory_hostname == hostvars[host].inventory_hostname -%}
+      {% set _ = index.append(loop.index0) -%}
+    {% endif -%}
+  {% endfor -%}
+  {% set metric_host = hostvars[groups.monitoring_servers[index.0]].ansible_host | default(hostvars[groups.monitoring_servers[index.0]].inventory_hostname) -%}
+  {% set metric_port = hostvars[groups.monitoring_servers[index.0]].prometheus_port -%}
+{% else -%}
+  {% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) -%}
+  {% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port -%}
+{% endif -%}
 {
     "name":"{{ cluster_name }}",
     "type":"prometheus",
     "access":"proxy",
     "url":"http://{{ metric_host }}:{{ metric_port }}/",
     "basicAuth":false
-}
+}

roles/prometheus/templates/prometheus.yml.j2

@@ -28,37 +28,49 @@ rule_files:
   - 'lightning.rules.yml'
 {% endif %}
 
-{% set alertmanager_host = hostvars[groups.alertmanager_servers[0]].ansible_host | default(hostvars[groups.alertmanager_servers[0]].inventory_hostname)
-    if groups.get('alertmanager_servers', []) else '' -%}
-{% set alertmanager_port = hostvars[groups.alertmanager_servers[0]].alertmanager_port if alertmanager_host else '' -%}
-
-{% if alertmanager_target|default("") %}
+{% if alertmanager_target|default("") -%}
 alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - '{{ alertmanager_target }}'
-{% elif alertmanager_host %}
+{% elif groups.alertmanager_servers -%}
+{% if groups.monitoring_servers | length == groups.alertmanager_servers | length -%}
+  {% set index = [] -%}
+  {% for host in groups.monitoring_servers -%}
+    {% if inventory_hostname == hostvars[host].inventory_hostname -%}
+      {% set _ = index.append(loop.index0) -%}
+    {% endif -%}
+  {% endfor -%}
+  {% set alertmanager_host = hostvars[groups.alertmanager_servers[index.0]].ansible_host | default(hostvars[groups.alertmanager_servers[index.0]].inventory_hostname) -%}
+  {% set alertmanager_port = hostvars[groups.alertmanager_servers[index.0]].alertmanager_port -%}
 alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - '{{ alertmanager_host }}:{{ alertmanager_port }}'
-{% else %}
+{% endif -%}
+{% else -%}
 # alerting:
 #  alertmanagers:
 #  - static_configs:
 #    - targets:
 #      - 'alertmanager_host:9093'
-{% endif %}
+{% endif -%}
 
-{% set pushgateway_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname)
-    if groups.get('monitoring_servers', []) else '' -%}
-{% set pushgateway_port = hostvars[groups.monitoring_servers[0]].pushgateway_port if pushgateway_host else '' -%}
+{% set pushgateway_addrs = [] -%}
+{% for host in groups.monitoring_servers -%}
+  {% set pushgateway_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set pushgateway_port = hostvars[host].pushgateway_port -%}
+  {% set _ = pushgateway_addrs.append("%s:%s" % (pushgateway_ip, pushgateway_port)) -%}
+{% endfor -%}
 
-{% set grafana_host = hostvars[groups.grafana_servers[0]].ansible_host | default(hostvars[groups.grafana_servers[0]].inventory_hostname)
-    if groups.get('grafana_servers', []) else '' -%}
-{% set grafana_port = hostvars[groups.grafana_servers[0]].grafana_port if grafana_host else '' -%}
+{% set grafana_addrs = [] -%}
+{% for host in groups.grafana_servers -%}
+  {% set grafana_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set grafana_port = hostvars[host].grafana_port -%}
+  {% set _ = grafana_addrs.append("%s:%s" % (grafana_ip, grafana_port)) -%}
+{% endfor -%}
 
 {% set blackbox_host = hostvars[groups.monitored_servers[0]].ansible_host | default(hostvars[groups.monitored_servers[0]].inventory_hostname) if groups.get('monitored_servers', []) else '' -%}
 {% set blackbox_port = hostvars[groups.monitored_servers[0]].blackbox_exporter_port if blackbox_host else '' -%}
@@ -106,12 +118,15 @@ alerting:
 {% endfor -%}
 
 scrape_configs:
-{% if pushgateway_host %}
+{% if pushgateway_addrs %}
   - job_name: 'overwritten-cluster'
     scrape_interval: 15s
     honor_labels: true # don't overwrite job & instance labels
     static_configs:
-      - targets: ['{{ pushgateway_host }}:{{ pushgateway_port }}']
+    - targets:
+{% for pushgateway_addr in pushgateway_addrs %}
+      - '{{ pushgateway_addr }}'
+{% endfor %}
 
   - job_name: "blackbox_exporter_http"
     scrape_interval: 30s
@@ -120,7 +135,9 @@ scrape_configs:
       module: [http_2xx]
     static_configs:
     - targets:
-      - 'http://{{ pushgateway_host }}:{{ pushgateway_port }}/metrics'
+{% for pushgateway_addr in pushgateway_addrs %}
+      - 'http://{{ pushgateway_addr }}/metrics'
+{% endfor %}
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
@@ -282,11 +299,15 @@ scrape_configs:
       labels:
         group: 'pd'
     - targets:
-      - '{{ pushgateway_host }}:{{ pushgateway_port }}'
+{% for pushgateway_addr in pushgateway_addrs %}
+      - '{{ pushgateway_addr }}'
+{% endfor %}
       labels:
         group: 'pushgateway'
     - targets:
-      - '{{ grafana_host }}:{{ grafana_port }}'
+{% for grafana_addr in grafana_addrs %}
+      - '{{ grafana_addr }}'
+{% endfor %}
       labels:
         group: 'grafana'
     - targets:

rolling_update_monitor.yml

@@ -395,7 +395,7 @@
 
     - name: import grafana dashboards - prepare config
       delegate_to: localhost
-      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json
+      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json
       vars:
         - ansible_become: false
         - ansible_connection: local
@@ -426,7 +426,7 @@
 
     - name: import grafana dashboards - run import script
       delegate_to: localhost
-      shell: "python grafana-config-copy.py"
+      shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json"
       args:
         chdir: "{{ playbook_dir }}/scripts"
       vars:

scripts/grafana-config-copy.py

@@ -3,6 +3,7 @@
 from __future__ import print_function, \
     unicode_literals
 
+import sys
 import urllib
 import urllib2
 import base64
@@ -23,7 +24,7 @@
 ]
 
 if not dests:
-    with open("./dests.json") as fp:
+    with open(sys.argv[1]) as fp:
         dests = json.load(fp)
 
 src = dict(

start.yml

@@ -518,7 +518,7 @@
 
     - name: import grafana dashboards - prepare config
       delegate_to: localhost
-      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json
+      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json
       vars:
         - ansible_become: false
         - ansible_connection: local
@@ -549,7 +549,7 @@
 
     - name: import grafana dashboards - run import script
       delegate_to: localhost
-      shell: "python grafana-config-copy.py"
+      shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json"
       args:
         chdir: "{{ playbook_dir }}/scripts"
       vars: