Optimization rolling update pd (#883) (#894)

* Optimization rolling update pd

common_tasks/get_pd_name_tls.yml

@@ -0,0 +1,20 @@
+---
+
+- name: get PD name
+  uri:
+    url: "https://{{ pd_addr }}/pd/api/v1/members"
+    validate_certs: no
+    client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
+    client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
+    method: GET
+    return_content: yes
+    status_code: 200
+  register: pd_info
+
+- set_fact:
+    pd_name_list: "{{ pd_info.json.members | json_query(query) }}"
+  vars:
+    query: '[?client_urls==[`http://{{ pd_addr }}`]].name'
+
+- set_fact:
+    pd_name: "{{ pd_name_list[0] }}"

excessive_rolling_update.yml

@@ -54,8 +54,101 @@
         - current_version.stdout_lines[0].replace(' ','').split(':')[1] < "v2.0.1"
         - tidb_version >= "v2.1.0" or tidb_version == "latest"
 
+- hosts: pd_servers[0]
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - pd
+  tasks:
+    - name: Check pd cluster status
+      uri:
+        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        method: GET
+        return_content: yes
+        status_code: 200
+      register: pd_status
+      when: not enable_tls|default(false)
+
+    - name: Check pd cluster status when enable_tls
+      uri:
+        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        validate_certs: no
+        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
+        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
+        method: GET
+        return_content: yes
+        status_code: 200
+      register: pd_status_tls
+      when: enable_tls|default(false)
+
+    - name: Failed when one node of pd is unhealthy
+      fail:
+        msg: "Some pd nodes are unhealthy"
+      when:
+        - not enable_tls|default(false)
+        - "'false' in pd_status.content"
+
+    - name: Failed when one node of pd is unhealthy when enable_tls
+      fail:
+        msg: "Some pd nodes are unhealthy"
+      when:
+        - enable_tls|default(false)
+        - "'false' in pd_status_tls.content"
+
+- hosts: pd_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - pd
+  tasks:
+    - set_fact:
+        pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"
+
+    - include_tasks: "common_tasks/get_pd_leader.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_leader_tls.yml"
+      when: enable_tls|default(false)
+
+    - set_fact:
+        pd_leader_name: "{{ pd_leader_info.json.name }}"
+
+    - include_tasks: "common_tasks/get_pd_name.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_name_tls.yml"
+      when: enable_tls|default(false)
+
+    - name: Set pd follower list
+      add_host:
+        name: "{{ inventory_hostname }}"
+        ansible_host: "{{ ansible_host }}"
+        ansible_ssh_host: "{{ ansible_ssh_host }}"
+        groups: pd_servers_followers
+        deploy_dir: "{{ deploy_dir }}"
+        pd_client_port: "{{ pd_client_port }}"
+        pd_peer_port: "{{ pd_peer_port }}"
+        pd_data_dir: "{{ pd_data_dir }}"
+        pd_log_dir: "{{ pd_log_dir }}"
+        pd_cert_dir: "{{ pd_cert_dir }}"
+      when: pd_leader_name != pd_name
+
+    - name: Set pd leader list
+      add_host:
+        name: "{{ inventory_hostname }}"
+        ansible_host: "{{ ansible_host }}"
+        ansible_ssh_host: "{{ ansible_ssh_host }}"
+        groups: pd_servers_leader
+        deploy_dir: "{{ deploy_dir }}"
+        pd_client_port: "{{ pd_client_port }}"
+        pd_peer_port: "{{ pd_peer_port }}"
+        pd_data_dir: "{{ pd_data_dir }}"
+        pd_log_dir: "{{ pd_log_dir }}"
+        pd_cert_dir: "{{ pd_cert_dir }}"
+      when: pd_leader_name == pd_name
+
 - name: rolling update PD cluster
-  hosts: pd_servers
+  hosts: pd_servers_followers, pd_servers_leader
   any_errors_fatal: true
   serial: 1
   tags:
@@ -66,6 +159,10 @@
         pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"
 
     - include_tasks: "common_tasks/get_pd_name.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_name_tls.yml"
+      when: enable_tls|default(false)
 
     - name: display PD name
       debug:
@@ -142,6 +239,29 @@
       delay: 5
       when: enable_tls|default(false)
 
+    - name: wait until the PD cluster is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        return_content: yes
+      register: pd_cluster_status
+      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
+
+    - name: wait until the PD cluster is available when enable_tls
+      uri:
+        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        validate_certs: no
+        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
+        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
+        return_content: yes
+      register: pd_cluster_status
+      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
+      retries: 12
+      delay: 5
+      when: enable_tls|default(false)
+
 
 - name: rolling update TiKV cluster
   hosts: tikv_servers

rolling_update.yml

@@ -54,8 +54,101 @@
         - current_version.stdout_lines[0].replace(' ','').split(':')[1] < "v2.0.1"
         - tidb_version >= "v2.1.0" or tidb_version == "latest"
 
+- hosts: pd_servers[0]
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - pd
+  tasks:
+    - name: Check pd cluster status
+      uri:
+        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        method: GET
+        return_content: yes
+        status_code: 200
+      register: pd_status
+      when: not enable_tls|default(false)
+
+    - name: Check pd cluster status when enable_tls
+      uri:
+        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        validate_certs: no
+        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
+        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
+        method: GET
+        return_content: yes
+        status_code: 200
+      register: pd_status_tls
+      when: enable_tls|default(false)
+
+    - name: Failed when one node of pd is unhealthy
+      fail:
+        msg: "Some pd nodes are unhealthy"
+      when: 
+        - not enable_tls|default(false)
+        - "'false' in pd_status.content"
+
+    - name: Failed when one node of pd is unhealthy when enable_tls
+      fail:
+        msg: "Some pd nodes are unhealthy"
+      when:
+        - enable_tls|default(false)
+        - "'false' in pd_status_tls.content"
+
+- hosts: pd_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - pd
+  tasks:
+    - set_fact:
+        pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"
+
+    - include_tasks: "common_tasks/get_pd_leader.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_leader_tls.yml"
+      when: enable_tls|default(false)
+
+    - set_fact:
+        pd_leader_name: "{{ pd_leader_info.json.name }}"
+
+    - include_tasks: "common_tasks/get_pd_name.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_name_tls.yml"
+      when: enable_tls|default(false)
+
+    - name: Set pd follower list
+      add_host:
+        name: "{{ inventory_hostname }}"
+        ansible_host: "{{ ansible_host }}"
+        ansible_ssh_host: "{{ ansible_ssh_host }}"
+        groups: pd_servers_followers
+        deploy_dir: "{{ deploy_dir }}"
+        pd_client_port: "{{ pd_client_port }}"
+        pd_peer_port: "{{ pd_peer_port }}"
+        pd_data_dir: "{{ pd_data_dir }}"
+        pd_log_dir: "{{ pd_log_dir }}"
+        pd_cert_dir: "{{ pd_cert_dir }}"
+      when: pd_leader_name != pd_name
+
+    - name: Set pd leader list
+      add_host:
+        name: "{{ inventory_hostname }}"
+        ansible_host: "{{ ansible_host }}"
+        ansible_ssh_host: "{{ ansible_ssh_host }}"
+        groups: pd_servers_leader
+        deploy_dir: "{{ deploy_dir }}"
+        pd_client_port: "{{ pd_client_port }}"
+        pd_peer_port: "{{ pd_peer_port }}"
+        pd_data_dir: "{{ pd_data_dir }}"
+        pd_log_dir: "{{ pd_log_dir }}"
+        pd_cert_dir: "{{ pd_cert_dir }}"
+      when: pd_leader_name == pd_name
+
 - name: rolling update PD cluster
-  hosts: pd_servers
+  hosts: pd_servers_followers, pd_servers_leader
   any_errors_fatal: true
   serial: 1
   tags:
@@ -66,6 +159,10 @@
         pd_addr: "{{ ansible_host }}:{{ pd_client_port }}"
 
     - include_tasks: "common_tasks/get_pd_name.yml"
+      when: not enable_tls|default(false)
+
+    - include_tasks: "common_tasks/get_pd_name_tls.yml"
+      when: enable_tls|default(false)
 
     - name: display PD name
       debug:
@@ -142,6 +239,29 @@
       delay: 5
       when: enable_tls|default(false)
 
+    - name: wait until the PD cluster is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        return_content: yes
+      register: pd_cluster_status
+      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
+
+    - name: wait until the PD cluster is available when enable_tls
+      uri:
+        url: "https://{{ ansible_host }}:{{ pd_client_port }}/pd/health"
+        validate_certs: no
+        client_cert: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem"
+        client_key: "{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem"
+        return_content: yes
+      register: pd_cluster_status
+      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content
+      retries: 12
+      delay: 5
+      when: enable_tls|default(false)
+
 
 - name: rolling update TiKV cluster
   hosts: tikv_servers