From 2ed211ba15271071eb186625bdca25fb5fcb8b65 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Wed, 3 Nov 2021 21:52:32 +0200 Subject: [PATCH 01/20] Fix-CI: python was upgraded in CI to 3.10 and pathlib is now included in python base making this dependency break the CI (#8153) --- tests/scripts/md-table/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/md-table/requirements.txt b/tests/scripts/md-table/requirements.txt index 754da38d8da..ecf5aac53a7 100644 --- a/tests/scripts/md-table/requirements.txt +++ b/tests/scripts/md-table/requirements.txt @@ -1,4 +1,4 @@ pyaml jinja2 -pathlib -pydblite \ No newline at end of file +pathlib ; python_version < '3.10' +pydblite From 1c3d082b8dce3e05fd4f905b4e15798071d3d97f Mon Sep 17 00:00:00 2001 From: Florian Ruynat <16313165+floryut@users.noreply.github.com> Date: Thu, 4 Nov 2021 18:38:04 +0100 Subject: [PATCH 02/20] fix calico crds hashes for 3.20.2 (#8157) --- roles/download/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index f4bae3ce520..882185526f3 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -390,7 +390,7 @@ calicoctl_binary_checksums: v3.19.3: ec3cfbd2dccbd614ac353be8c9abf8e336d8700fbd2b9b76da1c3c4c14a6dfe2 v3.18.5: 2080b2cd8efb71df98dce8678edaba8fce9252801c4b8f987609a4f4d6ddb4e2 calico_crds_archive_checksums: - v3.20.2: aaeaebf3d11d88a09d8564c0655932fd2a80c158b71325c80a324199a847ed8a + v3.20.2: b75b9965197073d2ae510275f61779ad38ff961ffe95a2fc3e0f452f94949c40 v3.19.3: 7066d0e6b0136920f82a75a5bd2d595e9f69bd3ab823403e920906569ec6be07 v3.18.5: ed7065c5a90b71cf7b3b525d5107a4573bd051c3ff004a56ab6017c222b3e2d6 From a0be7f0e264009b7ad987cdb0a44f734178c99ce Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Thu, 4 Nov 2021 22:10:23 +0200 Subject: [PATCH 03/20] heketi: fix deployment logic that was broken by the ansible 3.4 upgrade (#8118) --- .../network-storage/heketi/roles/provision/tasks/secret.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml index 3249c87b483..c455b6f6ddc 100644 --- a/contrib/network-storage/heketi/roles/provision/tasks/secret.yml +++ b/contrib/network-storage/heketi/roles/provision/tasks/secret.yml @@ -5,7 +5,7 @@ changed_when: false - name: "Kubernetes Apps | Deploy cluster role binding." - when: "clusterrolebinding_state.stdout | length > 0" + when: "clusterrolebinding_state.stdout | length == 0" command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" - name: Get clusterrolebindings again @@ -31,7 +31,7 @@ mode: 0644 - name: "Deploy Heketi config secret" - when: "secret_state.stdout | length > 0" + when: "secret_state.stdout | length == 0" command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json" - name: Get the heketi-config-secret secret again @@ -41,5 +41,5 @@ - name: Make sure the heketi-config-secret secret exists now assert: - that: "secret_state.stdout != \"\"" + that: "secret_state.stdout | length > 0" msg: "Heketi config secret is not present." From 8d553f7e91705280b445893ecdeb7f92774a3927 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Fri, 5 Nov 2021 09:57:52 +0200 Subject: [PATCH 04/20] Mitogen: deprecate the use of mitogen and remove coverage from CI (#8147) --- .gitlab-ci/packet.yml | 11 ----------- .gitlab-ci/terraform.yml | 4 ---- Makefile | 4 +++- ansible.cfg | 1 - docs/ansible.md | 2 +- docs/mitogen.md | 17 +++++++++++++++++ tests/scripts/testcases_run.sh | 7 ------- 7 files changed, 21 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 30d6ba3f34e..9b432a19a89 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -39,8 +39,6 @@ packet_centos7-flannel-containerd-addons-ha: extends: .packet_pr stage: deploy-part2 when: on_success - variables: - MITOGEN_ENABLE: "true" allow_failure: true packet_centos8-crio: @@ -52,8 +50,6 @@ packet_ubuntu18-crio: extends: .packet_pr stage: deploy-part2 when: manual - variables: - MITOGEN_ENABLE: "true" packet_ubuntu16-canal-kubeadm-ha: stage: deploy-part2 @@ -89,8 +85,6 @@ packet_debian10-containerd: stage: deploy-part2 extends: .packet_pr when: on_success - variables: - MITOGEN_ENABLE: "true" packet_debian11-calico: stage: deploy-part2 @@ -214,15 +208,12 @@ packet_centos7-weave-upgrade-ha: when: on_success variables: UPGRADE_TEST: basic - MITOGEN_ENABLE: "false" # Calico HA Wireguard packet_ubuntu20-calico-ha-wireguard: stage: deploy-part2 extends: .packet_pr when: manual - variables: - MITOGEN_ENABLE: "true" packet_debian9-calico-upgrade: stage: deploy-part3 @@ -230,7 +221,6 @@ packet_debian9-calico-upgrade: when: on_success variables: UPGRADE_TEST: graceful - MITOGEN_ENABLE: "false" packet_debian9-calico-upgrade-once: stage: deploy-part3 @@ -238,7 +228,6 @@ packet_debian9-calico-upgrade-once: when: on_success variables: UPGRADE_TEST: graceful - MITOGEN_ENABLE: "false" packet_ubuntu18-calico-ha-recover: stage: deploy-part3 diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index d12ca381521..91874091f55 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -146,10 +146,6 @@ tf-validate-upcloud: OS_INTERFACE: public OS_IDENTITY_API_VERSION: "3" TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df" - # Since ELASTX is in Stockholm, Mitogen helps with latency - MITOGEN_ENABLE: "false" - # Mitogen doesn't support interpreter discovery yet - ANSIBLE_PYTHON_INTERPRETER: "/usr/bin/python3" tf-elastx_cleanup: stage: unit-tests diff --git a/Makefile b/Makefile index 2093b820b0f..793e763dc12 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ mitogen: - ansible-playbook -c local mitogen.yml -vv + @echo Mitogen support is deprecated. + @echo Please run the following command manually: + @echo ansible-playbook -c local mitogen.yml -vv clean: rm -rf dist/ rm *.retry diff --git a/ansible.cfg b/ansible.cfg index 2132064985d..c17fe48fdcd 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -3,7 +3,6 @@ pipelining=True ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null #control_path = ~/.ssh/ansible-%%r@%%h:%%p [defaults] -strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .) force_valid_group_names = ignore diff --git a/docs/ansible.md b/docs/ansible.md index 044397f9f0f..cc51d261729 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -251,7 +251,7 @@ For more information about Ansible and bastion hosts, read ## Mitogen -You can use [mitogen](/docs/mitogen.md) to speed up kubespray. +Mitogen support is deprecated, please see [mitogen related docs](/docs/mitogen.md) for useage and reasons for deprecation. ## Beyond ansible 2.9 diff --git a/docs/mitogen.md b/docs/mitogen.md index 89b108a6c50..8505845fb41 100644 --- a/docs/mitogen.md +++ b/docs/mitogen.md @@ -1,5 +1,7 @@ # Mitogen +*Warning:* Mitogen support is now deprecated in kubespray due to upstream not releasing an updated version to support ansible 4.x (ansible-base 2.11.x) and above. The CI support has been stripped for mitogen and we are no longer validating any support or regressions for it. The supporting mitogen install playbook and integration documentation will be removed in a later version. + [Mitogen for Ansible](https://mitogen.networkgenomics.com/ansible_detailed.html) allow a 1.25x - 7x speedup and a CPU usage reduction of at least 2x, depending on network conditions, modules executed, and time already spent by targets on useful work. Mitogen cannot improve a module once it is executing, it can only ensure the module executes as quickly as possible. ## Install @@ -8,6 +10,21 @@ ansible-playbook mitogen.yml ``` +Ensure to enable mitogen use by environment varialbles: + +```ShellSession +export ANSIBLE_STRATEGY=mitogen_linear +export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy +``` + +... or `ansible.cfg` setup: + +```ini +[defaults] +strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy +strategy=mitogen_linear +``` + ## Limitation If you are experiencing problems, please see the [documentation](https://mitogen.networkgenomics.com/ansible_detailed.html#noteworthy-differences). diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 55cf7c3c75a..2461d29c6c6 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -50,13 +50,6 @@ test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR} -# Install mitogen ansible plugin -if [ "${MITOGEN_ENABLE}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} mitogen.yml - export ANSIBLE_STRATEGY=mitogen_linear - export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy -fi - # Create cluster ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml From c94291558d93ebf5106a85157fde827edbe0c09c Mon Sep 17 00:00:00 2001 From: Marcus Fenner Date: Fri, 5 Nov 2021 15:53:53 +0100 Subject: [PATCH 05/20] Fix containerd install for fcos (#8107) * Fix containerd install for fcos * rm orphaned runc and containerd binaries --- roles/container-engine/containerd/tasks/main.yml | 13 +++++++++++++ roles/container-engine/runc/defaults/main.yml | 2 +- roles/container-engine/runc/tasks/main.yml | 13 +++++++++++++ roles/kubespray-defaults/defaults/main.yaml | 2 +- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index 4a76a192acb..7088f340d73 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -57,6 +57,19 @@ - --strip-components=1 notify: restart containerd +- name: containerd | Remove orphaned binary + file: + path: "/usr/bin/{{ item }}" + state: absent + when: containerd_bin_dir != "/usr/bin" + ignore_errors: true # noqa ignore-errors + with_items: + - containerd + - containerd-shim + - containerd-shim-runc-v1 + - containerd-shim-runc-v2 + - ctr + - name: containerd | Generate systemd service for containerd template: src: containerd.service.j2 diff --git a/roles/container-engine/runc/defaults/main.yml b/roles/container-engine/runc/defaults/main.yml index 9c2fafffdf6..af8aa0837d7 100644 --- a/roles/container-engine/runc/defaults/main.yml +++ b/roles/container-engine/runc/defaults/main.yml @@ -1,5 +1,5 @@ --- -runc_bin_dir: /usr/bin/ +runc_bin_dir: "{{ bin_dir }}" runc_package_name: runc diff --git a/roles/container-engine/runc/tasks/main.yml b/roles/container-engine/runc/tasks/main.yml index be1014d7924..94f97c0baf1 100644 --- a/roles/container-engine/runc/tasks/main.yml +++ b/roles/container-engine/runc/tasks/main.yml @@ -1,8 +1,14 @@ --- +- name: runc | set is_ostree + set_fact: + is_ostree: "{{ ostree.stat.exists }}" + - name: runc | Uninstall runc package managed by package manager package: name: "{{ runc_package_name }}" state: absent + when: + - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar")) - name: runc | Download runc binary include_tasks: "../../../download/tasks/download_file.yml" @@ -15,3 +21,10 @@ dest: "{{ runc_bin_dir }}/runc" mode: 0755 remote_src: true + +- name: runc | Remove orphaned binary + file: + path: /usr/bin/runc + state: absent + when: runc_bin_dir != "/usr/bin" + ignore_errors: true # noqa ignore-errors diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index bb00672845f..282cc6507aa 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -67,7 +67,7 @@ haproxy_config_dir: "/etc/haproxy" # Directory where the binaries will be installed bin_dir: /usr/local/bin docker_bin_dir: /usr/bin -containerd_bin_dir: /usr/bin +containerd_bin_dir: "{{ bin_dir }}" etcd_data_dir: /var/lib/etcd # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) From 6e5b9e0ebfa5d0c037356da8825be883e24069fc Mon Sep 17 00:00:00 2001 From: Pasquale Toscano Date: Fri, 5 Nov 2021 15:59:54 +0100 Subject: [PATCH 06/20] Fix Kubelet and Containerd when using cgroupfs as cgroup driver (#8123) --- docs/vars.md | 2 +- .../containerd/defaults/main.yml | 16 +++++++++------- .../container-engine/containerd/tasks/facts.yml | 6 ++++++ .../container-engine/containerd/tasks/main.yml | 4 ++++ .../containerd/templates/config.toml.j2 | 10 +++++++++- roles/kubernetes/node/defaults/main.yml | 12 ++++++++++-- roles/kubernetes/node/tasks/facts.yml | 17 +++++++++++++++++ .../templates/kubelet-config.v1beta1.yaml.j2 | 2 +- 8 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 roles/container-engine/containerd/tasks/facts.yml diff --git a/docs/vars.md b/docs/vars.md index 51129cfc220..be812042abd 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -119,7 +119,7 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m ``--insecure-registry=myregistry.mydomain:5000`` * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install. * *containerd_default_runtime* - Sets the default Containerd runtime used by the Kubernetes CRI plugin. -* *containerd_runtimes* - Sets the Containerd runtime attributes used by the Kubernetes CRI plugin. +* *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin. [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars. * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index bfab4aaa815..f01f85f38ce 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -7,13 +7,15 @@ containerd_oom_score: 0 containerd_default_runtime: "runc" # containerd_snapshotter: "native" -containerd_runtimes: - - name: runc - type: "io.containerd.runc.v2" - engine: "" - root: "" - options: - systemdCgroup: "true" +containerd_runc_runtime: + name: runc + type: "io.containerd.runc.v2" + engine: "" + root: "" + options: + systemCgroup: "true" + +containerd_additional_runtimes: [] # Example for Kata Containers as additional runtime: # - name: kata # type: "io.containerd.kata.v2" diff --git a/roles/container-engine/containerd/tasks/facts.yml b/roles/container-engine/containerd/tasks/facts.yml new file mode 100644 index 00000000000..987b784580a --- /dev/null +++ b/roles/container-engine/containerd/tasks/facts.yml @@ -0,0 +1,6 @@ +--- + +- name: set kubelet_config_extra_args options when cgroupfs is used + set_fact: + containerd_runc_runtime: "{{ containerd_runc_runtime | combine({'options':{'systemCgroup':'false'}}) }}" + when: not containerd_use_systemd_cgroup diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index 7088f340d73..39779e78cb5 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -1,4 +1,8 @@ --- +- import_tasks: facts.yml + tags: + - facts + - name: Fail containerd setup if distribution is not supported fail: msg: "{{ ansible_distribution }} is not supported by containerd." diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 35c4f933a2a..48f3628e069 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -22,7 +22,15 @@ oom_score = {{ containerd_oom_score }} default_runtime_name = "{{ containerd_default_runtime | default('runc') }}" snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] -{% for runtime in containerd_runtimes %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}] + runtime_type = "{{ containerd_runc_runtime.type }}" + runtime_engine = "{{ containerd_runc_runtime.engine}}" + runtime_root = "{{ containerd_runc_runtime.root }}" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}.options] +{% for key, value in containerd_runc_runtime.options.items() %} + {{ key }} = {{ value }} +{% endfor %} +{% for runtime in containerd_additional_runtimes %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}] runtime_type = "{{ runtime.type }}" runtime_engine = "{{ runtime.engine }}" diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 686e2e6096e..d42fa555ad0 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -14,11 +14,14 @@ kube_resolv_conf: "/etc/resolv.conf" # Set to empty to avoid cgroup creation kubelet_enforce_node_allocatable: "\"\"" -# Set runtime cgroups +# Set runtime and kubelet cgroups when using systemd as cgroup driver (default) kubelet_runtime_cgroups: "/systemd/system.slice" -# Set kubelet cgroups kubelet_kubelet_cgroups: "/systemd/system.slice" +# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver +kubelet_runtime_cgroups_cgroupfs: "/system.slice/containerd.service" +kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.slice" + ### fail with swap on (default true) kubelet_fail_swap_on: true @@ -66,6 +69,11 @@ kubelet_max_pods: 110 ## Support parameters to be passed to kubelet via kubelet-config.yaml kubelet_config_extra_args: {} +## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver +kubelet_config_extra_args_cgroupfs: + systemCgroups: /system.slice + cgroupRoot: / + ## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters kubelet_node_config_extra_args: {} diff --git a/roles/kubernetes/node/tasks/facts.yml b/roles/kubernetes/node/tasks/facts.yml index b7b3ad01180..d4bd428f585 100644 --- a/roles/kubernetes/node/tasks/facts.yml +++ b/roles/kubernetes/node/tasks/facts.yml @@ -27,6 +27,23 @@ {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%} when: container_manager == 'containerd' +- name: set kubelet_cgroup_driver + set_fact: + kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}" + when: kubelet_cgroup_driver is undefined + +- name: set kubelet_cgroups options when cgroupfs is used + set_fact: + kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}" + kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}" + when: kubelet_cgroup_driver == 'cgroupfs' + +- name: set kubelet_config_extra_args options when cgroupfs is used + vars: + set_fact: + kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}" + when: kubelet_cgroup_driver == 'cgroupfs' + - name: os specific vars include_vars: "{{ item }}" with_first_found: diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 index 673c6178831..13ed5f4c424 100644 --- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -23,7 +23,7 @@ enforceNodeAllocatable: {% endfor %} {% endif %} staticPodPath: {{ kube_manifest_dir }} -cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} +cgroupDriver: {{ kubelet_cgroup_driver }} containerLogMaxFiles: {{ kubelet_logfiles_max_nr }} containerLogMaxSize: {{ kubelet_logfiles_max_size }} maxPods: {{ kubelet_max_pods }} From b7eb1cf9364d71c889ddffe83c2ff208dfd5d890 Mon Sep 17 00:00:00 2001 From: Antoine Gatineau <43171889+infra-monkey@users.noreply.github.com> Date: Fri, 5 Nov 2021 17:43:52 +0100 Subject: [PATCH 07/20] cert-manager: add trusted internal ca when configured (#8135) * cert-manager: add trusted internal ca when configured * wrong check for inventory variable * Update documentation --- docs/cert_manager.md | 14 ++++++++++++ .../sample/group_vars/k8s_cluster/addons.yml | 4 ++++ .../templates/cert-manager.yml.j2 | 22 +++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/docs/cert_manager.md b/docs/cert_manager.md index 34378a56a69..4ed28afc224 100644 --- a/docs/cert_manager.md +++ b/docs/cert_manager.md @@ -88,6 +88,20 @@ Certificates issued by public ACME servers are typically trusted by client’s c - [DNS01 Challenges](https://cert-manager.io/v1.5-docs/configuration/acme/dns01/) - [ACME FAQ](https://cert-manager.io/v1.5-docs/faq/acme/) +#### ACME With An Internal Certificate Authority + +The ACME Issuer with an internal certificate authority requires cert-manager to trust the certificate authority. This trust must be done at the cert-manager deployment level. +To add a trusted certificate authority to cert-manager, add it's certificate to `group_vars/k8s-cluster/addons.yml`: + +```yaml +cert_manager_trusted_internal_ca: | + -----BEGIN CERTIFICATE----- + [REPLACE with your CA certificate] + -----END CERTIFICATE----- +``` + +Once the CA is trusted, you can define your issuer normally. + ### Create New TLS Root CA Certificate and Key #### Install Cloudflare PKI/TLS `cfssl` Toolkit diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index 5f5e37f443c..2e077dd805e 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -129,6 +129,10 @@ ingress_alb_enabled: false # Cert manager deployment cert_manager_enabled: false # cert_manager_namespace: "cert-manager" +# cert_manager_trusted_internal_ca: | +# -----BEGIN CERTIFICATE----- +# [REPLACE with your CA certificate] +# -----END CERTIFICATE----- # MetalLB deployment metallb_enabled: false diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 index 200ab268016..3f51b19ad66 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/cert-manager.yml.j2 @@ -875,6 +875,17 @@ spec: resources: {} --- +{% if cert_manager_trusted_internal_ca is defined %} +apiVersion: v1 +data: + internal-ca.pem: | + {{ cert_manager_trusted_internal_ca | indent(width=4, indentfirst=False) }} +kind: ConfigMap +metadata: + name: ca-internal-truststore + namespace: {{ cert_manager_namespace }} +--- +{% endif %} # Source: cert-manager/templates/deployment.yaml apiVersion: apps/v1 kind: Deployment @@ -928,6 +939,17 @@ spec: fieldPath: metadata.namespace resources: {} +{% if cert_manager_trusted_internal_ca is defined %} + volumeMounts: + - mountPath: /etc/ssl/certs/internal-ca.pem + name: ca-internal-truststore + subPath: internal-ca.pem + volumes: + - configMap: + defaultMode: 420 + name: ca-internal-truststore + name: ca-internal-truststore +{% endif %} --- # Source: cert-manager/templates/webhook-deployment.yaml apiVersion: apps/v1 From 58390c79d0f7cd8c92a4c47814453a123481e092 Mon Sep 17 00:00:00 2001 From: Emin AKTAS Date: Sat, 6 Nov 2021 12:26:50 +0300 Subject: [PATCH 08/20] Bump crun version 1.2 to 1.3 (#8162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Emin Aktaş Co-authored-by: Yasin Taha Erol Co-authored-by: Necatican Yıldırım Co-authored-by: Yasin Taha Erol Co-authored-by: Necatican Yıldırım --- roles/download/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 882185526f3..0be6cc30eab 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -51,7 +51,7 @@ image_arch: "{{host_architecture | default('amd64')}}" # Versions kubeadm_version: "{{ kube_version }}" etcd_version: v3.5.0 -crun_version: 1.2 +crun_version: 1.3 runc_version: v1.0.2 kata_containers_version: 2.2.0 gvisor_version: 20210921 @@ -425,6 +425,7 @@ crun_checksums: 0.20.1: 9fac3040c95adbeced9110ceb79fd49556dd5027e39f98473c3c3e1f7edf5d16 0.21: b96cbdf549b69d20ce5dc81c300a138e5c1fd3d11555674043143ace8303c9a7 1.2: 2228a8e0e0f10920b230f9b8bc7c4fd951b603b278ccf0ebdba794339a49c33b + 1.3: 020a2e74d48f1e52f888a31b8bf873a1a99e9f89713ac9ff9403e14b2b9d5c18 arm64: 0.16: 0 0.17: 0 @@ -435,6 +436,7 @@ crun_checksums: 0.20.1: bcbb1ad85cbd953c9c2eb8d8651fee2bbc949516c4c6ac4fd03a9dffc7d2ff53 0.21: 7207d328978ee478be6dcf673ada0674305a624f57ee7ae1660c688751feb725 1.2: 3aee1057196b40b9786a08c875569c9046e58f97d29333b454359668b6088fb1 + 1.3: c0955cf6d3d832c0249bbaa71ed235abb35b8ca45fe07f2bd4501a00afb9bdc4 kata_containers_binary_checksums: arm: From 8922c45556e7b7dd83224efbf2063c16b4d87a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Torres=20Cogollo?= Date: Sun, 7 Nov 2021 11:22:51 +0100 Subject: [PATCH 09/20] Added ArgoCD kubernetes-app (#7895) * Added ArgoCD kubernetes-app * Update argocd_version to latest --- .../sample/group_vars/k8s_cluster/addons.yml | 13 ++++ .../kubernetes-apps/argocd/defaults/main.yml | 5 ++ roles/kubernetes-apps/argocd/tasks/main.yml | 77 +++++++++++++++++++ .../argocd/templates/argocd-namespace.yml.j2 | 7 ++ roles/kubernetes-apps/meta/main.yml | 7 ++ roles/kubespray-defaults/defaults/main.yaml | 1 + 6 files changed, 110 insertions(+) create mode 100644 roles/kubernetes-apps/argocd/defaults/main.yml create mode 100644 roles/kubernetes-apps/argocd/tasks/main.yml create mode 100644 roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index 2e077dd805e..013f30bf6d5 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -180,6 +180,19 @@ metallb_speaker_enabled: true # peer_asn: 64513 # my_asn: 4200000000 + +argocd_enabled: false +# argocd_version: v2.1.6 +# argocd_namespace: argocd +# Default password: +# - https://argoproj.github.io/argo-cd/getting_started/#4-login-using-the-cli +# --- +# The initial password is autogenerated to be the pod name of the Argo CD API server. This can be retrieved with the command: +# kubectl get pods -n argocd -l app.kubernetes.io/name=argocd-server -o name | cut -d'/' -f 2 +# --- +# Use the following var to set admin password +# argocd_admin_password: "password" + # The plugin manager for kubectl krew_enabled: false krew_root_dir: "/usr/local/krew" diff --git a/roles/kubernetes-apps/argocd/defaults/main.yml b/roles/kubernetes-apps/argocd/defaults/main.yml new file mode 100644 index 00000000000..39014108bcd --- /dev/null +++ b/roles/kubernetes-apps/argocd/defaults/main.yml @@ -0,0 +1,5 @@ +--- +argocd_enabled: false +argocd_version: v2.1.6 +argocd_namespace: argocd +# argocd_admin_password: diff --git a/roles/kubernetes-apps/argocd/tasks/main.yml b/roles/kubernetes-apps/argocd/tasks/main.yml new file mode 100644 index 00000000000..e80e63e694a --- /dev/null +++ b/roles/kubernetes-apps/argocd/tasks/main.yml @@ -0,0 +1,77 @@ +--- +- name: Kubernetes Apps | Install yq + become: yes + get_url: + url: "https://github.com/mikefarah/yq/releases/download/v4.11.2/yq_linux_amd64" + dest: "{{ bin_dir }}/yq" + mode: '0755' + +- name: Kubernetes Apps | Set ArgoCD template list + set_fact: + argocd_templates: + - name: namespace + file: argocd-namespace.yml + - name: install + file: argocd-install.yml + namespace: "{{ argocd_namespace }}" + url: "https://raw.githubusercontent.com/argoproj/argo-cd/{{argocd_version}}/manifests/install.yaml" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Download ArgoCD remote manifests + become: yes + get_url: + url: "{{ item.url }}" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Set ArgoCD namespace for remote manifests + become: yes + command: | + {{ bin_dir }}/yq eval-all -i '.metadata.namespace="{{argocd_namespace}}"' {{ kube_config_dir }}/{{ item.file }} + with_items: "{{ argocd_templates | selectattr('url', 'defined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Create ArgoCD manifests from templates + become: yes + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: "{{ argocd_templates | selectattr('url', 'undefined') | list }}" + loop_control: + label: "{{ item.file }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +- name: Kubernetes Apps | Install ArgoCD + become: yes + kube: + name: ArgoCD + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.file }}" + state: latest + with_items: "{{ argocd_templates }}" + when: + - "inventory_hostname == groups['kube_control_plane'][0]" + +# https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it +- name: Kubernetes Apps | Set ArgoCD custom admin password + become: yes + shell: | + {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n {{argocd_namespace}} patch secret argocd-secret -p \ + '{ + "stringData": { + "admin.password": "{{argocd_admin_password|password_hash('bcrypt')}}", + "admin.passwordMtime": "'$(date +%FT%T%Z)'" + } + }' + when: + - argocd_admin_password is defined + - "inventory_hostname == groups['kube_control_plane'][0]" diff --git a/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 b/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 new file mode 100644 index 00000000000..99962f13f25 --- /dev/null +++ b/roles/kubernetes-apps/argocd/templates/argocd-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{argocd_namespace}} + labels: + app: argocd diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 8ed80387d53..4650b38c162 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -110,3 +110,10 @@ dependencies: - inventory_hostname == groups['kube_control_plane'][0] tags: - metallb + + - role: kubernetes-apps/argocd + when: + - argocd_enabled + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - argocd diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 282cc6507aa..ef14e9b3297 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -377,6 +377,7 @@ ingress_alb_enabled: false cert_manager_enabled: false expand_persistent_volumes: false metallb_enabled: false +argocd_enabled: false # containerd official CLI tool nerdctl_enabled: false From cb7c30a4f154567ca11a6b83aba9f6f0efdb0ea0 Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Sun, 7 Nov 2021 23:48:52 -0800 Subject: [PATCH 10/20] Fix cloud_provider check (#8164) This fixes the preinstall check for cloud_provider option based on inventory/sample/group_vars/all/all.yml --- roles/kubernetes/preinstall/tasks/0020-verify-settings.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 6b666ba0649..524027c18cf 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -183,8 +183,8 @@ - name: check cloud_provider value assert: - that: cloud_provider in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', 'external'] - msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere', or external" + that: cloud_provider in ['gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', 'external'] + msg: "If set the 'cloud_provider' var must be set either to 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci' or 'external'" when: - cloud_provider is defined - not ignore_assert_errors From 04711d3b00593d6c6267c2aecd9af1f631fdd661 Mon Sep 17 00:00:00 2001 From: zhengtianbao Date: Mon, 8 Nov 2021 02:00:52 -0600 Subject: [PATCH 11/20] Replace path_join to support Ansible 2.9 (#8160) --- roles/reset/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 737e6b15174..94e1fbe7039 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -373,7 +373,7 @@ - dns - name: reset | include file with reset tasks specific to the network_plugin if exists - include_tasks: "{{ (role_path,'../network_plugin',kube_network_plugin,'tasks/reset.yml') | path_join | realpath }}" + include_tasks: "{{ (role_path,'../network_plugin',kube_network_plugin,'tasks/reset.yml') | community.general.path_join | realpath }}" when: - kube_network_plugin in ['flannel', 'cilium', 'kube-router', 'calico'] tags: From 61c2ae55498b70b1b75601f2e4ef2ac194e33ef8 Mon Sep 17 00:00:00 2001 From: Hyojun Jeon Date: Mon, 8 Nov 2021 17:06:52 +0900 Subject: [PATCH 12/20] Add vxlanEnabled spec in FelixConfiguration (#8167) --- roles/network_plugin/calico/tasks/install.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index cac20ea5e2d..e2ad4a78cd9 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -159,7 +159,8 @@ "bpfEnabled": {{ calico_bpf_enabled | bool }}, "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}", "wireguardEnabled": {{ calico_wireguard_enabled | bool }}, - "logSeverityScreen": "{{ calico_felix_log_severity_screen }}" }} + "logSeverityScreen": "{{ calico_felix_log_severity_screen }}", + "vxlanEnabled": {{ calico_vxlan_mode != 'Never' }} }} when: - inventory_hostname == groups['kube_control_plane'][0] From 6c1ab2498153ebf70b5ddc5dd4e28d11631359e5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 8 Nov 2021 11:22:58 +0100 Subject: [PATCH 13/20] Limit kubectl delete node to k8s nodes (#8101) * Limit kubectl delete node to k8s nodes This avoids the use of `kubectl delete node` when removing etcd nodes which are not part of the cluser (separate etcd) * Take errors into account when deleting node There should not be error now that we're limiting the deletion to nodes actually in the cluster * Retrying on error --- roles/remove-node/post-remove/tasks/main.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/remove-node/post-remove/tasks/main.yml b/roles/remove-node/post-remove/tasks/main.yml index 3205c008fa7..6ca8c2a687e 100644 --- a/roles/remove-node/post-remove/tasks/main.yml +++ b/roles/remove-node/post-remove/tasks/main.yml @@ -1,5 +1,10 @@ --- -- name: Delete node # noqa 301 ignore-errors +- name: Delete node command: "{{ bin_dir }}/kubectl delete node {{ kube_override_hostname|default(inventory_hostname) }}" delegate_to: "{{ groups['kube_control_plane']|first }}" - ignore_errors: true + when: inventory_hostname in groups['k8s_cluster'] + retries: 10 + # Sometimes the api-server can have a short window of indisponibility when we delete a master node + delay: 3 + register: result + until: result is not failed From 65540c5771be701fda8e90a7369acbfd939c9f49 Mon Sep 17 00:00:00 2001 From: zhengtianbao Date: Mon, 8 Nov 2021 04:54:59 -0600 Subject: [PATCH 14/20] krew: update to v0.4.2 (#8168) krew release urls changed since v0.4.2, clearly OS type and arch inside the filename. from: https://github.com/kubernetes-sigs/krew/releases/download/v0.4.1/krew.tar.gz to: https://github.com/kubernetes-sigs/krew/releases/download/v0.4.2/krew-linux_amd64.tar.gz define `host_os` like `host_architecture` determine which OS is krew installed at. --- roles/download/defaults/main.yml | 31 ++++++++++++++++--- roles/kubernetes-apps/krew/tasks/krew.yml | 2 +- .../krew/templates/krew.yml.j2 | 12 +++---- roles/kubespray-defaults/defaults/main.yaml | 11 +++++++ 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 0be6cc30eab..662cad54c60 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -91,7 +91,7 @@ ovn4nfv_ovn_image_version: "v1.0.0" ovn4nfv_k8s_plugin_image_version: "v1.1.0" helm_version: "v3.7.0" nerdctl_version: "0.12.1" -krew_version: "v0.4.1" +krew_version: "v0.4.2" # Get kubernetes major version (i.e. 1.17.4 => 1.17) kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" @@ -118,7 +118,7 @@ kata_containers_download_url: "https://github.com/kata-containers/kata-container gvisor_runsc_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc" gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1" nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" -krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz" +krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz" crictl_checksums: @@ -395,7 +395,28 @@ calico_crds_archive_checksums: v3.18.5: ed7065c5a90b71cf7b3b525d5107a4573bd051c3ff004a56ab6017c222b3e2d6 krew_archive_checksums: - v0.4.1: a26deea175f70264260d59a4e061778a892f8a8e301ac261660dd7d24c551c99 + linux: + arm: + v0.4.2: 115f503e35ef7f63f00a9b01236d80a9f94862ec684010a81c3a3b51bdca1351 + arm64: + v0.4.2: 7581be80d803536acc63cceb20065023b96f07fd7eb9f4ee495dce0294a866eb + amd64: + v0.4.2: 203bfd8006b304c1e58d9e96f9afdc5f4a055e0fbd7ee397fac9f36bf202e721 + darwin: + arm: + v0.4.2: 0 + arm64: + v0.4.2: a69d48f8cad7d87b379071129cde3ee4abcaaa1c3f3692bc80887178b2cc7d33 + amd64: + v0.4.2: 47c6b5b647c5de679a2302444f75a36a70530fa4751cb655e0edd5da56a5f110 + windows: + arm: + v0.4.2: 0 + arm64: + v0.4.2: 0 + amd64: + v0.4.2: 3150ff0291ac876ebe4fe0e813ee90a18aa2bc0510c3adcfae6117dec44ef269 + helm_archive_checksums: arm: @@ -506,7 +527,7 @@ kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" -krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}" +krew_archive_checksum: "{{ krew_archive_checksums[host_os][image_arch][krew_version] }}" containerd_archive_checksum: "{{ containerd_archive_checksums[image_arch][containerd_version] }}" # Containers @@ -1106,7 +1127,7 @@ downloads: enabled: "{{ krew_enabled }}" file: true version: "{{ krew_version }}" - dest: "{{ local_release_dir }}/krew.tar.gz" + dest: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz" sha256: "{{ krew_archive_checksum }}" url: "{{ krew_download_url }}" unarchive: true diff --git a/roles/kubernetes-apps/krew/tasks/krew.yml b/roles/kubernetes-apps/krew/tasks/krew.yml index ebd98212090..7e8161fc31d 100644 --- a/roles/kubernetes-apps/krew/tasks/krew.yml +++ b/roles/kubernetes-apps/krew/tasks/krew.yml @@ -15,7 +15,7 @@ dest: "{{ local_release_dir }}/krew.yml" - name: Krew | Install krew # noqa 301 305 - shell: "{{ local_release_dir }}/krew-linux_{{ image_arch }} install --archive={{ local_release_dir }}/krew.tar.gz --manifest={{ local_release_dir }}/krew.yml" + shell: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} install --archive={{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz --manifest={{ local_release_dir }}/krew.yml" environment: KREW_ROOT: "{{ krew_root_dir }}" KREW_DEFAULT_INDEX_URI: "{{ krew_default_index_uri | default('') }}" diff --git a/roles/kubernetes-apps/krew/templates/krew.yml.j2 b/roles/kubernetes-apps/krew/templates/krew.yml.j2 index 9c036a93109..8235067adb4 100644 --- a/roles/kubernetes-apps/krew/templates/krew.yml.j2 +++ b/roles/kubernetes-apps/krew/templates/krew.yml.j2 @@ -26,7 +26,7 @@ spec: https://krew.sigs.k8s.io/docs/user-guide/quickstart/. platforms: - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew files: @@ -38,7 +38,7 @@ spec: matchLabels: os: darwin arch: amd64 - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew files: @@ -50,7 +50,7 @@ spec: matchLabels: os: darwin arch: arm64 - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew files: @@ -62,7 +62,7 @@ spec: matchLabels: os: linux arch: amd64 - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew files: @@ -74,7 +74,7 @@ spec: matchLabels: os: linux arch: arm - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew files: @@ -86,7 +86,7 @@ spec: matchLabels: os: linux arch: arm64 - - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz + - uri: https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz sha256: {{ krew_archive_checksum }} bin: krew.exe files: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index ef14e9b3297..99ee494d76a 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -576,6 +576,17 @@ host_architecture: >- {{ ansible_architecture }} {%- endif -%} +_host_os_groups: + Linux: linux + Darwin: darwin + Win32NT: windows +host_os: >- + {%- if ansible_system in _host_os_groups -%} + {{ _host_os_groups[ansible_system] }} + {%- else -%} + {{ ansible_system }} + {%- endif -%} + # Sets the eventRecordQPS parameter in kubelet-config.yaml. The default value is 5 (see types.go) # Setting it to 0 allows unlimited requests per second. kubelet_event_record_qps: 5 From 4a8757161e1a456ae85c8e612c70b846979c0ead Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Tue, 9 Nov 2021 01:56:49 +0200 Subject: [PATCH 15/20] Docker: replace the use of containerd_version with docker_containerd_version to avoid causing conflicts when bumping containerd_version (#8130) --- docs/upgrades.md | 3 ++- docs/vars.md | 3 ++- roles/container-engine/docker/vars/debian-stretch.yml | 2 +- roles/container-engine/docker/vars/debian.yml | 2 +- roles/container-engine/docker/vars/fedora.yml | 2 +- roles/container-engine/docker/vars/redhat.yml | 2 +- roles/container-engine/docker/vars/ubuntu-16.yml | 2 +- roles/container-engine/docker/vars/ubuntu.yml | 2 +- roles/download/defaults/main.yml | 4 ---- roles/kubespray-defaults/defaults/main.yaml | 5 ++++- tests/files/packet_debian9-calico-upgrade-once.yml | 2 +- tests/files/packet_debian9-calico-upgrade.yml | 2 +- tests/files/packet_debian9-macvlan.yml | 2 +- tests/files/packet_ubuntu16-canal-kubeadm-ha.yml | 2 +- tests/files/packet_ubuntu16-canal-sep.yml | 2 +- tests/files/packet_ubuntu16-flannel-ha.yml | 2 +- tests/files/packet_ubuntu16-kube-router-sep.yml | 2 +- tests/files/packet_ubuntu16-kube-router-svc-proxy.yml | 2 +- tests/files/packet_ubuntu16-weave-sep.yml | 2 +- 19 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index ad0983b1a19..f5fe3804906 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -7,7 +7,8 @@ You can also individually control versions of components by explicitly defining versions. Here are all version vars for each component: * docker_version -* containerd_version +* docker_containerd_version (relevant when `container_manager` == `docker`) +* containerd_version (relevant when `container_manager` == `containerd`) * kube_version * etcd_version * calico_version diff --git a/docs/vars.md b/docs/vars.md index be812042abd..f0d7747d213 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -18,7 +18,8 @@ Some variables of note include: * *docker_version* - Specify version of Docker to used (should be quoted string). Must match one of the keys defined for *docker_versioned_pkg* in `roles/container-engine/docker/vars/*.yml`. -* *containerd_version* - Specify version of Containerd to use +* *containerd_version* - Specify version of containerd to use when setting `container_manager` to `containerd` +* *docker_containerd_version* - Specify which version of containerd to use when setting `container_manager` to `docker` * *etcd_version* - Specify version of ETCD to use * *ipip* - Enables Calico ipip encapsulation by default * *kube_network_plugin* - Sets k8s network plugin (default Calico) diff --git a/roles/container-engine/docker/vars/debian-stretch.yml b/roles/container-engine/docker/vars/debian-stretch.yml index 3616c64e00e..f26f60b1c10 100644 --- a/roles/container-engine/docker/vars/debian-stretch.yml +++ b/roles/container-engine/docker/vars/debian-stretch.yml @@ -28,7 +28,7 @@ docker_cli_versioned_pkg: docker_package_info: pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container-engine/docker/vars/debian.yml b/roles/container-engine/docker/vars/debian.yml index 3b0c784bbdd..36b22c2d7d0 100644 --- a/roles/container-engine/docker/vars/debian.yml +++ b/roles/container-engine/docker/vars/debian.yml @@ -30,7 +30,7 @@ docker_cli_versioned_pkg: docker_package_info: pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container-engine/docker/vars/fedora.yml b/roles/container-engine/docker/vars/fedora.yml index df5d3597da6..278dfe0fe73 100644 --- a/roles/container-engine/docker/vars/fedora.yml +++ b/roles/container-engine/docker/vars/fedora.yml @@ -30,6 +30,6 @@ docker_cli_versioned_pkg: docker_package_info: enablerepo: "docker-ce" pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container-engine/docker/vars/redhat.yml b/roles/container-engine/docker/vars/redhat.yml index 8cc897cda79..5cc83529e80 100644 --- a/roles/container-engine/docker/vars/redhat.yml +++ b/roles/container-engine/docker/vars/redhat.yml @@ -33,6 +33,6 @@ docker_cli_versioned_pkg: docker_package_info: enablerepo: "docker-ce" pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container-engine/docker/vars/ubuntu-16.yml b/roles/container-engine/docker/vars/ubuntu-16.yml index 54046cbe23d..78a6ceae412 100644 --- a/roles/container-engine/docker/vars/ubuntu-16.yml +++ b/roles/container-engine/docker/vars/ubuntu-16.yml @@ -29,7 +29,7 @@ docker_cli_versioned_pkg: docker_package_info: pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/container-engine/docker/vars/ubuntu.yml b/roles/container-engine/docker/vars/ubuntu.yml index 0fdc778e31f..fb85f24066b 100644 --- a/roles/container-engine/docker/vars/ubuntu.yml +++ b/roles/container-engine/docker/vars/ubuntu.yml @@ -30,7 +30,7 @@ docker_cli_versioned_pkg: docker_package_info: pkgs: - - "{{ containerd_versioned_pkg[containerd_version | string] }}" + - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}" - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}" - "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 662cad54c60..b1f6cfaf660 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -497,18 +497,14 @@ nerdctl_archive_checksums: amd64: 0.12.1: 868dc5997c3edb0bd06f75012e71c2b15ee0885b83bad191fbe2a1d6d5f4f2ac -# TODO(cristicalin): remove compatibility entries once debian9 and ubuntu16 jobs are dropped or docker is dropped containerd_archive_checksums: arm: - latest: 0 # this is needed to make debian9 and ubuntu16 CI jobs happy 1.4.9: 0 1.5.5: 0 arm64: - latest: 0 # this is needed to make debian9 and ubuntu16 CI jobs happy 1.4.9: 0 1.5.5: 0 amd64: - latest: 0 # this is needed to make debian9 and ubuntu16 CI jobs happy 1.4.9: 346f88ad5b973960ff81b5539d4177af5941ec2e4703b479ca9a6081ff1d023b 1.5.5: 8efc527ffb772a82021800f0151374a3113ed2439922497ff08f2596a70f10f1 diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 99ee494d76a..24345b5a2bc 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -330,10 +330,13 @@ docker_plugins: [] # Experimental kubeadm etcd deployment mode. Available only for new deployment etcd_kubeadm_enabled: false -# Containerd options +# Containerd options - thse are relevant when container_manager == 'containerd' containerd_version: 1.4.9 containerd_use_systemd_cgroup: true +# Docker options - this is relevant when container_manager == 'docker' +docker_containerd_version: 1.4.9 + # Settings for containerized control plane (etcd/kubelet/secrets) # deployment type for legacy etcd mode etcd_deployment_type: docker diff --git a/tests/files/packet_debian9-calico-upgrade-once.yml b/tests/files/packet_debian9-calico-upgrade-once.yml index 9e4fa1b553c..dcf4186c795 100644 --- a/tests/files/packet_debian9-calico-upgrade-once.yml +++ b/tests/files/packet_debian9-calico-upgrade-once.yml @@ -10,4 +10,4 @@ dns_min_replicas: 1 download_run_once: true # Make docker happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_debian9-calico-upgrade.yml b/tests/files/packet_debian9-calico-upgrade.yml index dd02770246f..bc6837a11fb 100644 --- a/tests/files/packet_debian9-calico-upgrade.yml +++ b/tests/files/packet_debian9-calico-upgrade.yml @@ -9,4 +9,4 @@ deploy_netchecker: true dns_min_replicas: 1 # Make docker happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_debian9-macvlan.yml b/tests/files/packet_debian9-macvlan.yml index 9a481b2b939..accf275df12 100644 --- a/tests/files/packet_debian9-macvlan.yml +++ b/tests/files/packet_debian9-macvlan.yml @@ -14,4 +14,4 @@ macvlan_interface: "eth0" auto_renew_certificates: true # Make docker happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-canal-kubeadm-ha.yml b/tests/files/packet_ubuntu16-canal-kubeadm-ha.yml index 9861b350210..ac64817aa6e 100644 --- a/tests/files/packet_ubuntu16-canal-kubeadm-ha.yml +++ b/tests/files/packet_ubuntu16-canal-kubeadm-ha.yml @@ -10,4 +10,4 @@ deploy_netchecker: true dns_min_replicas: 1 # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-canal-sep.yml b/tests/files/packet_ubuntu16-canal-sep.yml index 44df4f948cc..01fcaff8812 100644 --- a/tests/files/packet_ubuntu16-canal-sep.yml +++ b/tests/files/packet_ubuntu16-canal-sep.yml @@ -10,4 +10,4 @@ deploy_netchecker: true dns_min_replicas: 1 # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-flannel-ha.yml b/tests/files/packet_ubuntu16-flannel-ha.yml index 5f3b19d9bec..c4af804a153 100644 --- a/tests/files/packet_ubuntu16-flannel-ha.yml +++ b/tests/files/packet_ubuntu16-flannel-ha.yml @@ -12,4 +12,4 @@ deploy_netchecker: true dns_min_replicas: 1 # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-kube-router-sep.yml b/tests/files/packet_ubuntu16-kube-router-sep.yml index e923834aa4e..91aa6cbe1ca 100644 --- a/tests/files/packet_ubuntu16-kube-router-sep.yml +++ b/tests/files/packet_ubuntu16-kube-router-sep.yml @@ -10,4 +10,4 @@ deploy_netchecker: true dns_min_replicas: 1 # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-kube-router-svc-proxy.yml b/tests/files/packet_ubuntu16-kube-router-svc-proxy.yml index 043639ad9e1..433557b9832 100644 --- a/tests/files/packet_ubuntu16-kube-router-svc-proxy.yml +++ b/tests/files/packet_ubuntu16-kube-router-svc-proxy.yml @@ -12,4 +12,4 @@ dns_min_replicas: 1 kube_router_run_service_proxy: true # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu16-weave-sep.yml b/tests/files/packet_ubuntu16-weave-sep.yml index e424a6cc847..22cfe0e516c 100644 --- a/tests/files/packet_ubuntu16-weave-sep.yml +++ b/tests/files/packet_ubuntu16-weave-sep.yml @@ -11,4 +11,4 @@ dns_min_replicas: 1 auto_renew_certificates: true # Make docker jobs happy -containerd_version: latest +docker_containerd_version: latest From 46c536d261c06d4496db4446dbafe2d00eb705ed Mon Sep 17 00:00:00 2001 From: zhengtianbao Date: Tue, 9 Nov 2021 04:43:39 -0600 Subject: [PATCH 16/20] Add krew auto completion (#8171) --- roles/kubernetes-apps/krew/tasks/krew.yml | 12 ++++++++++++ roles/reset/tasks/main.yml | 1 + 2 files changed, 13 insertions(+) diff --git a/roles/kubernetes-apps/krew/tasks/krew.yml b/roles/kubernetes-apps/krew/tasks/krew.yml index 7e8161fc31d..dab6b8057bb 100644 --- a/roles/kubernetes-apps/krew/tasks/krew.yml +++ b/roles/kubernetes-apps/krew/tasks/krew.yml @@ -19,3 +19,15 @@ environment: KREW_ROOT: "{{ krew_root_dir }}" KREW_DEFAULT_INDEX_URI: "{{ krew_default_index_uri | default('') }}" + +- name: Krew | Get krew completion + command: "{{ local_release_dir }}/krew-{{ host_os }}_{{ image_arch }} completion bash" + changed_when: False + register: krew_completion + check_mode: False + +- name: Krew | Install krew completion + copy: + dest: /etc/bash_completion.d/krew.sh + content: "{{ krew_completion.stdout }}" + become: True \ No newline at end of file diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 94e1fbe7039..6941f2150e8 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -322,6 +322,7 @@ - /etc/bash_completion.d/crictl - /etc/bash_completion.d/nerdctl - /etc/bash_completion.d/krew + - /etc/bash_completion.d/krew.sh - "{{ krew_root_dir | default('/usr/local/krew') }}" - /etc/modules-load.d/kube_proxy-ipvs.conf - /etc/modules-load.d/kubespray-br_netfilter.conf From 801268d5c1d515012b7ea981640a471b80a29af8 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Tue, 9 Nov 2021 16:59:47 +0200 Subject: [PATCH 17/20] containerd: upgrade versions 1.4.11 and 1.5.7 and make 1.4.11 the default (#8129) --- roles/download/defaults/main.yml | 6 ++++++ roles/kubespray-defaults/defaults/main.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b1f6cfaf660..8e858bb3a8f 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -500,13 +500,19 @@ nerdctl_archive_checksums: containerd_archive_checksums: arm: 1.4.9: 0 + 1.4.11: 0 1.5.5: 0 + 1.5.7: 0 arm64: 1.4.9: 0 + 1.4.11: 0 1.5.5: 0 + 1.5.7: 0 amd64: 1.4.9: 346f88ad5b973960ff81b5539d4177af5941ec2e4703b479ca9a6081ff1d023b + 1.4.11: 80c47ec5ce2cd91a15204b5f5b534892ca653e75f3fba0c451ca326bca45fb00 1.5.5: 8efc527ffb772a82021800f0151374a3113ed2439922497ff08f2596a70f10f1 + 1.5.7: 109fc95b86382065ea668005c376360ddcd8c4ec413e7abe220ae9f461e0e173 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}" cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}" diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 24345b5a2bc..99aec470e3d 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -331,7 +331,7 @@ docker_plugins: [] etcd_kubeadm_enabled: false # Containerd options - thse are relevant when container_manager == 'containerd' -containerd_version: 1.4.9 +containerd_version: 1.4.11 containerd_use_systemd_cgroup: true # Docker options - this is relevant when container_manager == 'docker' From 039205560a5a38dac7e180ec4c46ce4edd404d39 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:57:47 +0200 Subject: [PATCH 18/20] nodelocaldns: allow a secondary pod for nodelocaldns for local-HA (#8100) * nodelocaldns: allow a secondary pod for nodelocaldns for local-HA * CI: add job to test nodelocaldns secondary --- .gitlab-ci/packet.yml | 5 + docs/dns-stack.md | 16 +++ .../group_vars/k8s_cluster/k8s-cluster.yml | 3 + roles/download/defaults/main.yml | 2 +- .../kubernetes-apps/ansible/defaults/main.yml | 2 + roles/kubernetes-apps/ansible/tasks/main.yml | 1 + .../ansible/tasks/nodelocaldns.yml | 28 +++++ .../templates/nodelocaldns-config.yml.j2 | 88 ++++++++++++++- .../templates/nodelocaldns-daemonset.yml.j2 | 32 ++++-- .../nodelocaldns-second-daemonset.yml.j2 | 103 ++++++++++++++++++ roles/kubespray-defaults/defaults/main.yaml | 3 + ..._centos8-calico-nodelocaldns-secondary.yml | 15 +++ 12 files changed, 281 insertions(+), 17 deletions(-) create mode 100644 roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 create mode 100644 tests/files/packet_centos8-calico-nodelocaldns-secondary.yml diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 9b432a19a89..6e72a4cd811 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -194,6 +194,11 @@ packet_amazon-linux-2-aio: extends: .packet_pr when: manual +packet_centos8-calico-nodelocaldns-secondary: + stage: deploy-part2 + extends: .packet_pr + when: manual + packet_fedora34-kube-ovn-containerd: stage: deploy-part2 extends: .packet_periodic diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 7771c26bbdd..b6d2064a624 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -212,6 +212,22 @@ nodelocaldns_external_zones: See [dns_etchosts](#dns_etchosts-coredns) above. +### Nodelocal DNS HA + +Under some circumstances the single POD nodelocaldns implementation may not be able to be replaced soon enough and a cluster upgrade or a nodelocaldns upgrade can cause DNS requests to time out for short intervals. If for any reason your applications cannot tollerate this behavior you can enable a redundant nodelocal DNS pod on each node: + +```yaml +enable_nodelocaldns_secondary: true +``` + +**Note:** when the nodelocaldns secondary is enabled, the primary is instructed to no longer tear down the iptables rules it sets up to direct traffic to itself. In case both daemonsets have failing pods on the same node, this can cause a DNS blackout with traffic no longer being forwarded to the coredns central service as a fallback. Please ensure you account for this also if you decide to disable the nodelocaldns cache. + +There is a time delta (in seconds) allowed for the secondary nodelocaldns to survive in case both primary and secondary daemonsets are updated at the same time. It is advised to tune this variable after you have performed some tests in your own environment. + +```yaml +nodelocaldns_secondary_skew_seconds: 5 +``` + ## Limitations * Kubespray has yet ways to configure Kubedns addon to forward requests SkyDns can diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 4248832eb59..dbd66d3ddd0 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -166,9 +166,12 @@ dns_mode: coredns # manual_dns_server: 10.x.x.x # Enable nodelocal dns cache enable_nodelocaldns: true +enable_nodelocaldns_secondary: false nodelocaldns_ip: 169.254.25.10 nodelocaldns_health_port: 9254 +nodelocaldns_second_health_port: 9256 nodelocaldns_bind_metrics_host_ip: false +nodelocaldns_secondary_skew_seconds: 5 # nodelocaldns_external_zones: # - zones: # - example.com diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 8e858bb3a8f..6d19e232405 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -610,7 +610,7 @@ coredns_image_is_namespaced: "{{ (kube_version is version('v1.21.0','>=')) or (c coredns_image_repo: "{{ kube_image_repo }}{{'/coredns/coredns' if (coredns_image_is_namespaced | bool) else '/coredns' }}" coredns_image_tag: "{{ coredns_version if (coredns_image_is_namespaced | bool) else (coredns_version | regex_replace('^v', '')) }}" -nodelocaldns_version: "1.17.1" +nodelocaldns_version: "1.21.1" nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache" nodelocaldns_image_tag: "{{ nodelocaldns_version }}" diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 411260551cb..fa06b2e0d20 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -17,6 +17,8 @@ nodelocaldns_cpu_requests: 100m nodelocaldns_memory_limit: 170Mi nodelocaldns_memory_requests: 70Mi nodelocaldns_ds_nodeselector: "kubernetes.io/os: linux" +nodelocaldns_prometheus_port: 9253 +nodelocaldns_secondary_prometheus_port: 9255 # Limits for dns-autoscaler dns_autoscaler_cpu_requests: 20m diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 75ee477b0aa..d59f0e0b6f7 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -48,6 +48,7 @@ - "{{ coredns_manifests.results | default({}) }}" - "{{ coredns_secondary_manifests.results | default({}) }}" - "{{ nodelocaldns_manifests.results | default({}) }}" + - "{{ nodelocaldns_second_manifests.results | default({}) }}" when: - dns_mode != 'none' - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml index ce79ceed4b8..4809aa9b83d 100644 --- a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml +++ b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml @@ -43,3 +43,31 @@ tags: - nodelocaldns - coredns + +- name: Kubernetes Apps | Lay Down nodelocaldns-secondary Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: nodelocaldns, file: nodelocaldns-second-daemonset.yml, type: daemonset } + register: nodelocaldns_second_manifests + vars: + forwardTarget: >- + {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%} + {{ primaryClusterIP }} {{ secondaryclusterIP }} + {%- else -%} + {{ primaryClusterIP }} + {%- endif -%} + upstreamForwardTarget: >- + {%- if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 -%} + {{ upstream_dns_servers|join(' ') }} + {%- else -%} + /etc/resolv.conf + {%- endif -%} + when: + - enable_nodelocaldns + - enable_nodelocaldns_secondary + - inventory_hostname == groups['kube_control_plane'] | first + tags: + - nodelocaldns + - coredns diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 index 18abf8ea33b..0244c04a414 100644 --- a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 @@ -17,7 +17,7 @@ data: loop bind {{ nodelocaldns_ip }} forward . {{ block['nameservers'] | join(' ') }} - prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:9253 + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} log {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { @@ -39,7 +39,7 @@ data: forward . {{ forwardTarget }} { force_tcp } - prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:9253 + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} health {{ nodelocaldns_ip }}:{{ nodelocaldns_health_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { @@ -56,7 +56,7 @@ data: forward . {{ forwardTarget }} { force_tcp } - prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:9253 + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} } ip6.arpa:53 { errors @@ -67,7 +67,7 @@ data: forward . {{ forwardTarget }} { force_tcp } - prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:9253 + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} } .:53 { errors @@ -76,13 +76,91 @@ data: loop bind {{ nodelocaldns_ip }} forward . {{ upstreamForwardTarget }} - prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:9253 + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_prometheus_port }} {% if dns_etchosts | default(None) %} hosts /etc/coredns/hosts { fallthrough } {% endif %} } +{% if enable_nodelocaldns_secondary %} + Corefile-second: | +{% if nodelocaldns_external_zones is defined and nodelocaldns_external_zones|length > 0 %} +{% for block in nodelocaldns_external_zones %} + {{ block['zones'] | join(' ') }} { + errors + cache {{ block['cache'] | default(30) }} + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ block['nameservers'] | join(' ') }} + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + log +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endfor %} +{% endif %} + {{ dns_domain }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + health {{ nodelocaldns_ip }}:{{ nodelocaldns_second_health_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ forwardTarget }} { + force_tcp + } + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} + } + .:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} + forward . {{ upstreamForwardTarget }} + prometheus {% if nodelocaldns_bind_metrics_host_ip %}{$MY_HOST_IP}{% endif %}:{{ nodelocaldns_secondary_prometheus_port }} +{% if dns_etchosts | default(None) %} + hosts /etc/coredns/hosts { + fallthrough + } +{% endif %} + } +{% endif %} {% if dns_etchosts | default(None) %} hosts: | {{ dns_etchosts | indent(width=4, indentfirst=None) }} diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 index 7abd28ffab5..7c63e28fa91 100644 --- a/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-daemonset.yml.j2 @@ -16,7 +16,7 @@ spec: k8s-app: nodelocaldns annotations: prometheus.io/scrape: 'true' - prometheus.io/port: '9253' + prometheus.io/port: '{{ nodelocaldns_prometheus_port }}' spec: nodeSelector: {{ nodelocaldns_ds_nodeselector }} @@ -38,16 +38,16 @@ spec: requests: cpu: {{ nodelocaldns_cpu_requests }} memory: {{ nodelocaldns_memory_requests }} - args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns" ] - securityContext: - privileged: true -{% if nodelocaldns_bind_metrics_host_ip %} - env: - - name: MY_HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP -{% endif %} + args: + - -localip + - {{ nodelocaldns_ip }} + - -conf + - /etc/coredns/Corefile + - -upstreamsvc + - coredns +{% if enable_nodelocaldns_secondary %} + - -skipteardown +{% else %} ports: - containerPort: 53 name: dns @@ -58,6 +58,16 @@ spec: - containerPort: 9253 name: metrics protocol: TCP +{% endif %} + securityContext: + privileged: true +{% if nodelocaldns_bind_metrics_host_ip %} + env: + - name: MY_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +{% endif %} livenessProbe: httpGet: host: {{ nodelocaldns_ip }} diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 new file mode 100644 index 00000000000..037bf446e03 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-second-daemonset.yml.j2 @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nodelocaldns-second + namespace: kube-system + labels: + k8s-app: kube-dns + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: nodelocaldns-second + template: + metadata: + labels: + k8s-app: nodelocaldns-second + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '{{ nodelocaldns_secondary_prometheus_port }}' + spec: + nodeSelector: + {{ nodelocaldns_ds_nodeselector }} + priorityClassName: system-cluster-critical + serviceAccountName: nodelocaldns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - effect: NoSchedule + operator: "Exists" + - effect: NoExecute + operator: "Exists" + containers: + - name: node-cache + image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" + resources: + limits: + memory: {{ nodelocaldns_memory_limit }} + requests: + cpu: {{ nodelocaldns_cpu_requests }} + memory: {{ nodelocaldns_memory_requests }} + args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile", "-upstreamsvc", "coredns", "-skipteardown" ] + securityContext: + privileged: true +{% if nodelocaldns_bind_metrics_host_ip %} + env: + - name: MY_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +{% endif %} + livenessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + readinessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: {{ nodelocaldns_health_port }} + scheme: HTTP + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + - name: xtables-lock + mountPath: /run/xtables.lock + lifecycle: + preStop: + exec: + command: + - sh + - -c + - sleep {{ nodelocaldns_secondary_skew_seconds }} && kill -9 1 + volumes: + - name: config-volume + configMap: + name: nodelocaldns + items: + - key: Corefile-second + path: Corefile +{% if dns_etchosts | default(None) %} + - key: hosts + path: hosts +{% endif %} + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Implement a time skew between the main nodelocaldns and this secondary. + # Since the two nodelocaldns instances share the :53 port, we want to keep + # at least one running at any time enven if the manifests are replaced simultaneously + terminationGracePeriodSeconds: {{ nodelocaldns_secondary_skew_seconds }} + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 99aec470e3d..488e1ae5bf3 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -93,9 +93,12 @@ dns_mode: coredns # Enable nodelocal dns cache enable_nodelocaldns: true +enable_nodelocaldns_secondary: false nodelocaldns_ip: 169.254.25.10 nodelocaldns_health_port: 9254 +nodelocaldns_second_health_port: 9256 nodelocaldns_bind_metrics_host_ip: false +nodelocaldns_secondary_skew_seconds: 5 # Should be set to a cluster IP if using a custom cluster DNS manual_dns_server: "" diff --git a/tests/files/packet_centos8-calico-nodelocaldns-secondary.yml b/tests/files/packet_centos8-calico-nodelocaldns-secondary.yml new file mode 100644 index 00000000000..600ce6017ab --- /dev/null +++ b/tests/files/packet_centos8-calico-nodelocaldns-secondary.yml @@ -0,0 +1,15 @@ +--- +# Instance settings +cloud_image: centos-8 +mode: default +vm_memory: 3072Mi + +# Kubespray settings +kube_network_plugin: calico +deploy_netchecker: true +dns_min_replicas: 1 +enable_nodelocaldns_secondary: true +loadbalancer_apiserver_type: haproxy + +# required +calico_iptables_backend: "Auto" From b7ae4a2cfde75b7f110c756438c77b2103cc4761 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Tue, 9 Nov 2021 20:01:48 +0200 Subject: [PATCH 19/20] Kata-Containers: Fix kata-containers runtime (#8068) * Kata-containes: Fix for ubuntu and centos sometimes kata containers fail to start because of access errors to /dev/vhost-vsock and /dev/vhost-net * Kata-containers: use similar testing strategy as gvisor * Kata-Containers: adjust values for 2.2.0 defaults Make CI tests actually pass * Kata-Containers: bump to 2.2.2 to fix sandbox_cgroup_only issue --- .../molecule/default/converge.yml | 1 + .../molecule/default/files/10-mynet.conf | 17 ++ .../molecule/default/files/container.json | 10 + .../molecule/default/files/sandbox.json | 10 + .../molecule/default/prepare.yml | 44 ++++- .../molecule/default/tests/test_default.py | 23 ++- .../kata-containers/tasks/main.yml | 16 ++ .../templates/configuration-qemu.toml.j2 | 175 +++++++++++++++++- roles/download/defaults/main.yml | 8 +- roles/reset/tasks/main.yml | 1 + 10 files changed, 287 insertions(+), 18 deletions(-) create mode 100644 roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf create mode 100644 roles/container-engine/kata-containers/molecule/default/files/container.json create mode 100644 roles/container-engine/kata-containers/molecule/default/files/sandbox.json diff --git a/roles/container-engine/kata-containers/molecule/default/converge.yml b/roles/container-engine/kata-containers/molecule/default/converge.yml index 3456ee6f82b..a6fdf812a78 100644 --- a/roles/container-engine/kata-containers/molecule/default/converge.yml +++ b/roles/container-engine/kata-containers/molecule/default/converge.yml @@ -4,6 +4,7 @@ become: true vars: kata_containers_enabled: true + container_manager: containerd roles: - role: kubespray-defaults - role: container-engine/containerd diff --git a/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf b/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf new file mode 100644 index 00000000000..f10935b753b --- /dev/null +++ b/roles/container-engine/kata-containers/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/roles/container-engine/kata-containers/molecule/default/files/container.json b/roles/container-engine/kata-containers/molecule/default/files/container.json new file mode 100644 index 00000000000..9ada521f476 --- /dev/null +++ b/roles/container-engine/kata-containers/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "kata1" + }, + "image": { + "image": "docker.io/library/hello-world:latest" + }, + "log_path": "kata1.0.log", + "linux": {} +} diff --git a/roles/container-engine/kata-containers/molecule/default/files/sandbox.json b/roles/container-engine/kata-containers/molecule/default/files/sandbox.json new file mode 100644 index 00000000000..326a578bed6 --- /dev/null +++ b/roles/container-engine/kata-containers/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "kata1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/roles/container-engine/kata-containers/molecule/default/prepare.yml b/roles/container-engine/kata-containers/molecule/default/prepare.yml index 1afc51a047c..9299a7e2d7d 100644 --- a/roles/container-engine/kata-containers/molecule/default/prepare.yml +++ b/roles/container-engine/kata-containers/molecule/default/prepare.yml @@ -1,6 +1,48 @@ --- - name: Prepare hosts: all - gather_facts: False + become: true roles: + - role: kubespray-defaults - role: bootstrap-os + - role: adduser + user: "{{ addusers.kube }}" + tasks: + - include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: network_plugin/cni + - role: container-engine/crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: kube + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/roles/container-engine/kata-containers/molecule/default/tests/test_default.py b/roles/container-engine/kata-containers/molecule/default/tests/test_default.py index 15e80825d4c..b34136d58e5 100644 --- a/roles/container-engine/kata-containers/molecule/default/tests/test_default.py +++ b/roles/container-engine/kata-containers/molecule/default/tests/test_default.py @@ -14,17 +14,24 @@ def test_run(host): assert "kata-runtime" in cmd.stdout -def test_run_pod(host): - image = "docker.io/library/hello-world:latest" - runtime = "io.containerd.kata-qemu.v2" - - pull_command = "ctr image pull {}".format(image) +def test_run_check(host): + kataruntime = "/opt/kata/bin/kata-runtime" with host.sudo(): - cmd = host.command(pull_command) + cmd = host.command(kataruntime + " check") assert cmd.rc == 0 + assert "System is capable of running" in cmd.stdout - run_command = "ctr run --runtime {} {} kata1".format(runtime, image) + +def test_run_pod(host): + runtime = "kata-qemu" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) with host.sudo(): cmd = host.command(run_command) assert cmd.rc == 0 - assert "Hello from Docker!" in cmd.stdout + + with host.sudo(): + log_f = host.file("/tmp/kata1.0.log") + + assert log_f.exists + assert b"Hello from Docker!" in log_f.content diff --git a/roles/container-engine/kata-containers/tasks/main.yml b/roles/container-engine/kata-containers/tasks/main.yml index 8d99e5255f8..54bd25d0fed 100644 --- a/roles/container-engine/kata-containers/tasks/main.yml +++ b/roles/container-engine/kata-containers/tasks/main.yml @@ -34,3 +34,19 @@ mode: 0755 with_items: - qemu + +- name: kata-containers | Load vhost kernel modules + modprobe: + state: present + name: "{{ item }}" + with_items: + - vhost_vsock + - vhost_net + +- name: kata-containers | Persist vhost kernel modules + copy: + dest: /etc/modules-load.d/kubespray-kata-containers.conf + mode: 0644 + content: | + vhost_vsock + vhost_net diff --git a/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 b/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 index 334a2d977bb..f64647bdfc8 100644 --- a/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 +++ b/roles/container-engine/kata-containers/templates/configuration-qemu.toml.j2 @@ -12,10 +12,33 @@ [hypervisor.qemu] path = "/opt/kata/bin/qemu-system-x86_64" +{% if kata_containers_version is version('2.2.0', '>=') %} +kernel = "/opt/kata/share/kata-containers/vmlinux.container" +{% else %} kernel = "/opt/kata/share/kata-containers/vmlinuz.container" +{% endif %} image = "/opt/kata/share/kata-containers/kata-containers.img" machine_type = "q35" +# Enable confidential guest support. +# Toggling that setting may trigger different hardware features, ranging +# from memory encryption to both memory and CPU-state encryption and integrity. +# The Kata Containers runtime dynamically detects the available feature set and +# aims at enabling the largest possible one. +# Default false +# confidential_guest = true + +# List of valid annotation names for the hypervisor +# Each member of the list is a regular expression, which is the base name +# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" +enable_annotations = [] + +# List of valid annotations values for the hypervisor +# Each member of the list is a path pattern as described by glob(3). +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/opt/kata/bin/qemu-system-x86_64"] +valid_hypervisor_paths = ["/opt/kata/bin/qemu-system-x86_64"] + # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc. @@ -37,6 +60,11 @@ firmware = "" # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` machine_accelerators="" +# CPU features +# comma-separated list of cpu features to pass to the cpu +# For example, `cpu_features = "pmu=off,vmx=off" +cpu_features="pmu=off" + # Default number of vCPUs per SB/VM: # unspecified or 0 --> will be set to 1 # < 0 --> will be set to the actual number of physical cores @@ -58,6 +86,7 @@ default_vcpus = 1 # `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of # vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, # unless you know what are you doing. +# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. default_maxvcpus = 0 # Bridges can be used to hot plug devices. @@ -103,15 +132,24 @@ default_memory = {{ kata_containers_qemu_default_memory }} disable_block_device_use = false # Shared file system type: -# - virtio-9p (default) -# - virtio-fs +# - virtio-fs (default) +# - virtio-9p +{% if kata_containers_version is version('2.2.0', '>=') %} +shared_fs = "virtio-fs" +{% else %} shared_fs = "virtio-9p" +{% endif %} # Path to vhost-user-fs daemon. -virtio_fs_daemon = "/opt/kata/bin/virtiofsd" +virtio_fs_daemon = "/opt/kata/libexec/kata-qemu/virtiofsd" + +# List of valid annotations values for the virtiofs daemon +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/opt/kata/libexec/kata-qemu/virtiofsd"] +valid_virtio_fs_daemon_paths = ["/opt/kata/libexec/kata-qemu/virtiofsd"] # Default size of DAX cache in MiB -virtio_fs_cache_size = 1024 +virtio_fs_cache_size = 0 # Extra args for virtiofsd daemon # @@ -119,7 +157,7 @@ virtio_fs_cache_size = 1024 # ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"] # # see `virtiofsd -h` for possible options. -virtio_fs_extra_args = [] +virtio_fs_extra_args = ["--thread-pool-size=1"] # Cache mode: # @@ -189,16 +227,40 @@ enable_vhost_user_store = false # simulated block device nodes for vhost-user devices to live. vhost_user_store_path = "/var/run/kata-containers/vhost-user" +# Enable vIOMMU, default false +# Enabling this will result in the VM having a vIOMMU device +# This will also add the following options to the kernel's +# command line: intel_iommu=on,iommu=pt +#enable_iommu = true + +# Enable IOMMU_PLATFORM, default false +# Enabling this will result in the VM device having iommu_platform=on set +#enable_iommu_platform = true + +# List of valid annotations values for the vhost user store path +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/var/run/kata-containers/vhost-user"] +valid_vhost_user_store_paths = ["/var/run/kata-containers/vhost-user"] + # Enable file based guest memory support. The default is an empty string which # will disable this feature. In the case of virtio-fs, this is enabled # automatically and '/dev/shm' is used as the backing folder. # This option will be ignored if VM templating is enabled. #file_mem_backend = "" +# List of valid annotations values for the file_mem_backend annotation +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: [""] +valid_file_mem_backends = [""] + # Enable swap of vm memory. Default false. # The behaviour is undefined if mem_prealloc is also set to true #enable_swap = true +# -pflash can add image file to VM. The arguments of it should be in format +# of ["/path/to/flash0.img", "/path/to/flash1.img"] +pflashes = [] + # This option changes the default hypervisor and kernel parameters # to enable debug output where available. This extra output is added # to the proxy logs, but only when proxy debug is also enabled. @@ -257,6 +319,11 @@ enable_debug = {{ kata_containers_qemu_debug }} # all practical purposes. #entropy_source= "/dev/urandom" +# List of valid annotations values for entropy_source +# The default if not set is empty (all annotations rejected.) +# Your distribution recommends: ["/dev/urandom","/dev/random",""] +valid_entropy_sources = ["/dev/urandom","/dev/random",""] + # Path to OCI hook binaries in the *guest rootfs*. # This does not affect host-side hooks which must instead be added to # the OCI spec passed to the runtime. @@ -273,6 +340,47 @@ enable_debug = {{ kata_containers_qemu_debug }} # Warnings will be logged if any error is encountered will scanning for hooks, # but it will not abort container execution. #guest_hook_path = "/usr/share/oci/hooks" +# +# Use rx Rate Limiter to control network I/O inbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) to discipline traffic. +# Default 0-sized value means unlimited rate. +#rx_rate_limiter_max_rate = 0 +# Use tx Rate Limiter to control network I/O outbound bandwidth(size in bits/sec for SB/VM). +# In Qemu, we use classful qdiscs HTB(Hierarchy Token Bucket) and ifb(Intermediate Functional Block) +# to discipline traffic. +# Default 0-sized value means unlimited rate. +#tx_rate_limiter_max_rate = 0 + +# Set where to save the guest memory dump file. +# If set, when GUEST_PANICKED event occurred, +# guest memeory will be dumped to host filesystem under guest_memory_dump_path, +# This directory will be created automatically if it does not exist. +# +# The dumped file(also called vmcore) can be processed with crash or gdb. +# +# WARNING: +# Dump guest’s memory can take very long depending on the amount of guest memory +# and use much disk space. +#guest_memory_dump_path="/var/crash/kata" + +# If enable paging. +# Basically, if you want to use "gdb" rather than "crash", +# or need the guest-virtual addresses in the ELF vmcore, +# then you should enable paging. +# +# See: https://www.qemu.org/docs/master/qemu-qmp-ref.html#Dump-guest-memory for details +#guest_memory_dump_paging=false + +# Enable swap in the guest. Default false. +# When enable_guest_swap is enabled, insert a raw file to the guest as the swap device +# if the swappiness of a container (set by annotation "io.katacontainers.container.resource.swappiness") +# is bigger than 0. +# The size of the swap device should be +# swap_in_bytes (set by annotation "io.katacontainers.container.resource.swap_in_bytes") - memory_limit_in_bytes. +# If swap_in_bytes is not set, the size should be memory_limit_in_bytes. +# If swap_in_bytes and memory_limit_in_bytes is not set, the size should +# be default_memory. +#enable_guest_swap = true [factory] # VM templating support. Once enabled, new VMs are created from template @@ -381,6 +489,16 @@ enable_debug = {{ kata_containers_qemu_debug }} # kernel_modules=[] +# Enable debug console. + +# If enabled, user can connect guest OS running inside hypervisor +# through "kata-runtime exec " command + +#debug_console_enabled = true + +# Agent connection dialing timeout value in seconds +# (default: 30) +#dial_timeout = 30 [netmon] # If enabled, the network monitoring process gets started when the @@ -433,6 +551,16 @@ disable_guest_seccomp=true # (default: disabled) #enable_tracing = true +# Set the full url to the Jaeger HTTP Thrift collector. +# The default if not set will be "http://localhost:14268/api/traces" +#jaeger_endpoint = "" + +# Sets the username to be used if basic auth is required for Jaeger. +#jaeger_user = "" + +# Sets the password to be used if basic auth is required for Jaeger. +#jaeger_password = "" + # If enabled, the runtime will not create a network namespace for shim and hypervisor processes. # This option may have some potential impacts to your host. It should only be used when you know what you're doing. # `disable_new_netns` conflicts with `enable_netmon` @@ -451,9 +579,46 @@ disable_guest_seccomp=true # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType sandbox_cgroup_only={{ kata_containers_qemu_sandbox_cgroup_only }} +# If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. +# This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. +# If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` +# These will not be exposed to the container workloads, and are only provided for potential guest services. +sandbox_bind_mounts=[] + # Enabled experimental feature list, format: ["a", "b"]. # Experimental features are features not stable enough for production, # they may break compatibility, and are prepared for a big version bump. # Supported experimental features: # (default: []) experimental=[] + +# If enabled, user can run pprof tools with shim v2 process through kata-monitor. +# (default: false) +# enable_pprof = true + +# WARNING: All the options in the following section have not been implemented yet. +# This section was added as a placeholder. DO NOT USE IT! +[image] +# Container image service. +# +# Offload the CRI image management service to the Kata agent. +# (default: false) +#service_offload = true + +# Container image decryption keys provisioning. +# Applies only if service_offload is true. +# Keys can be provisioned locally (e.g. through a special command or +# a local file) or remotely (usually after the guest is remotely attested). +# The provision setting is a complete URL that lets the Kata agent decide +# which method to use in order to fetch the keys. +# +# Keys can be stored in a local file, in a measured and attested initrd: +#provision=data:///local/key/file +# +# Keys could be fetched through a special command or binary from the +# initrd (guest) image, e.g. a firmware call: +#provision=file:///path/to/bin/fetcher/in/guest +# +# Keys can be remotely provisioned. The Kata agent fetches them from e.g. +# a HTTPS URL: +#provision=https://my-key-broker.foo/tenant/ diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 6d19e232405..0bab159cfca 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -53,7 +53,7 @@ kubeadm_version: "{{ kube_version }}" etcd_version: v3.5.0 crun_version: 1.3 runc_version: v1.0.2 -kata_containers_version: 2.2.0 +kata_containers_version: 2.2.2 gvisor_version: 20210921 # gcr and kubernetes image repo define @@ -463,15 +463,15 @@ kata_containers_binary_checksums: arm: 2.0.4: 0 2.1.1: 0 - 2.2.0: 0 + 2.2.2: 0 amd64: 2.0.4: 022a60c2d92a5ab9a5eb83d5a95154a2d06fdc2206b2a473d902ccc86766371a 2.1.1: a83591d968cd0f1adfb5025d7aa33ca1385d4b1165ff10d74602302fc3c0373f - 2.2.0: 50163e2a430e96447117f7169a4ed5a8bdea09267d62a39221d5b8b3b3f88c0e + 2.2.2: 2e3ac77b8abd4d839cf16780b57aee8f3d6e1f19489edd7d6d8069ea3cc3c18a arm64: 2.0.4: 0 2.1.1: 0 - 2.2.0: 0 + 2.2.2: 0 gvisor_runsc_binary_checksums: arm: diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 6941f2150e8..101be1c8884 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -326,6 +326,7 @@ - "{{ krew_root_dir | default('/usr/local/krew') }}" - /etc/modules-load.d/kube_proxy-ipvs.conf - /etc/modules-load.d/kubespray-br_netfilter.conf + - /etc/modules-load.d/kubespray-kata-containers.conf - /usr/libexec/kubernetes ignore_errors: true # noqa ignore-errors tags: From 0d0468e127733252895473c564fa6cd4c348d102 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Thu, 11 Nov 2021 02:11:50 +0200 Subject: [PATCH 20/20] Exercise multiple ansible versions in CI (#8172) * Ansible: separate requirements files for supported ansible versions * Ansible: allow using ansible 2.11 * CI: Exercise Ansible 2.9 and Ansible 2.11 in a basic AIO CI job * CI: Allow running a reset test outside of idempotency tests and running it in stage1 * CI: move ubuntu18-calico-aio job to stage2 and relay only on ubuntu20 with the variously supported ansible versions for stage1 * CI: add capability to install collections or roles from ansible-galaxy to mitigate missing behavior in older ansible versions --- .gitlab-ci.yml | 5 ++-- .gitlab-ci/packet.yml | 23 +++++++++++++-- ansible_version.yml | 2 +- requirements-2.10.txt | 10 +++++++ requirements-2.11.txt | 10 +++++++ requirements-2.9.txt | 8 +++++ requirements-2.9.yml | 4 +++ requirements.txt | 11 +------ ...acket_ubuntu20-calico-aio-ansible-2_11.yml | 1 + ...packet_ubuntu20-calico-aio-ansible-2_9.yml | 1 + tests/requirements-2.10.txt | 12 ++++++++ tests/requirements-2.11.txt | 12 ++++++++ tests/requirements-2.9.txt | 12 ++++++++ tests/requirements.txt | 13 +-------- tests/scripts/testcases_prepare.sh | 13 +++++++-- tests/scripts/testcases_run.sh | 29 +++++++++---------- 16 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 requirements-2.10.txt create mode 100644 requirements-2.11.txt create mode 100644 requirements-2.9.txt create mode 100644 requirements-2.9.yml mode change 100644 => 120000 requirements.txt create mode 120000 tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml create mode 120000 tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml create mode 100644 tests/requirements-2.10.txt create mode 100644 tests/requirements-2.11.txt create mode 100644 tests/requirements-2.9.txt mode change 100644 => 120000 tests/requirements.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a2b5662856..ae5a65bec07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,12 +32,13 @@ variables: RECOVER_CONTROL_PLANE_TEST: "false" RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]" TERRAFORM_VERSION: 1.0.8 + ANSIBLE_MAJOR_VERSION: "2.10" before_script: - ./tests/scripts/rebase.sh - update-alternatives --install /usr/bin/python python /usr/bin/python3 1 - - python -m pip uninstall -y ansible - - python -m pip install -r tests/requirements.txt + - python -m pip uninstall -y ansible ansible-base ansible-core + - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt - mkdir -p /.ssh .job: &job diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 6e72a4cd811..90a3ad3978f 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -23,15 +23,34 @@ extends: .packet packet_ubuntu18-calico-aio: - stage: deploy-part1 + stage: deploy-part2 extends: .packet_pr when: on_success -# Future AIO job +# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken packet_ubuntu20-calico-aio: stage: deploy-part1 extends: .packet_pr when: on_success + variables: + RESET_CHECK: "true" + +# Exericse ansible variants +packet_ubuntu20-calico-aio-ansible-2_9: + stage: deploy-part1 + extends: .packet_pr + when: on_success + variables: + ANSIBLE_MAJOR_VERSION: "2.9" + RESET_CHECK: "true" + +packet_ubuntu20-calico-aio-ansible-2_11: + stage: deploy-part1 + extends: .packet_pr + when: on_success + variables: + ANSIBLE_MAJOR_VERSION: "2.11" + RESET_CHECK: "true" # ### PR JOBS PART2 diff --git a/ansible_version.yml b/ansible_version.yml index b7fff003984..5226fd90fee 100644 --- a/ansible_version.yml +++ b/ansible_version.yml @@ -5,7 +5,7 @@ vars: minimal_ansible_version: 2.9.0 minimal_ansible_version_2_10: 2.10.11 - maximal_ansible_version: 2.11.0 + maximal_ansible_version: 2.12.0 ansible_connection: local tags: always tasks: diff --git a/requirements-2.10.txt b/requirements-2.10.txt new file mode 100644 index 00000000000..5fcbf804858 --- /dev/null +++ b/requirements-2.10.txt @@ -0,0 +1,10 @@ +ansible==3.4.0 +ansible-base==2.10.15 +cryptography==2.8 +jinja2==2.11.3 +netaddr==0.7.19 +pbr==5.4.4 +jmespath==0.9.5 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.4 +MarkupSafe==1.1.1 diff --git a/requirements-2.11.txt b/requirements-2.11.txt new file mode 100644 index 00000000000..5d535be6555 --- /dev/null +++ b/requirements-2.11.txt @@ -0,0 +1,10 @@ +ansible==4.8.0 +ansible-core==2.11.6 +cryptography==2.8 +jinja2==2.11.3 +netaddr==0.7.19 +pbr==5.4.4 +jmespath==0.9.5 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.4 +MarkupSafe==1.1.1 diff --git a/requirements-2.9.txt b/requirements-2.9.txt new file mode 100644 index 00000000000..220780881c5 --- /dev/null +++ b/requirements-2.9.txt @@ -0,0 +1,8 @@ +ansible==2.9.27 +jinja2==2.11.3 +netaddr==0.7.19 +pbr==5.4.4 +jmespath==0.9.5 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.4 +MarkupSafe==1.1.1 diff --git a/requirements-2.9.yml b/requirements-2.9.yml new file mode 100644 index 00000000000..e8034543d7a --- /dev/null +++ b/requirements-2.9.yml @@ -0,0 +1,4 @@ +--- +collections: +- name: community.general + version: '<3.0' diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ab7f293f1d6..00000000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -ansible==3.4.0 -ansible-base==2.10.11 -cryptography==2.8 -jinja2==2.11.3 -netaddr==0.7.19 -pbr==5.4.4 -jmespath==0.9.5 -ruamel.yaml==0.16.10 -ruamel.yaml.clib==0.2.4 -MarkupSafe==1.1.1 diff --git a/requirements.txt b/requirements.txt new file mode 120000 index 00000000000..5202ea4fbea --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requirements-2.10.txt \ No newline at end of file diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml new file mode 120000 index 00000000000..10064637f25 --- /dev/null +++ b/tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml @@ -0,0 +1 @@ +packet_ubuntu20-calico-aio.yml \ No newline at end of file diff --git a/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml b/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml new file mode 120000 index 00000000000..10064637f25 --- /dev/null +++ b/tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml @@ -0,0 +1 @@ +packet_ubuntu20-calico-aio.yml \ No newline at end of file diff --git a/tests/requirements-2.10.txt b/tests/requirements-2.10.txt new file mode 100644 index 00000000000..a80c3a86838 --- /dev/null +++ b/tests/requirements-2.10.txt @@ -0,0 +1,12 @@ +-r ../requirements-2.10.txt +yamllint==1.19.0 +apache-libcloud==2.2.1 +tox==3.11.1 +dopy==0.3.7 +cryptography==2.8 +ansible-lint==5.0.11 +openshift==0.8.8 +molecule==3.0.6 +molecule-vagrant==0.3 +testinfra==5.2.2 +python-vagrant==0.5.15 diff --git a/tests/requirements-2.11.txt b/tests/requirements-2.11.txt new file mode 100644 index 00000000000..8ad98b8b448 --- /dev/null +++ b/tests/requirements-2.11.txt @@ -0,0 +1,12 @@ +-r ../requirements-2.11.txt +yamllint==1.19.0 +apache-libcloud==2.2.1 +tox==3.11.1 +dopy==0.3.7 +cryptography==2.8 +ansible-lint==5.0.11 +openshift==0.8.8 +molecule==3.0.6 +molecule-vagrant==0.3 +testinfra==5.2.2 +python-vagrant==0.5.15 diff --git a/tests/requirements-2.9.txt b/tests/requirements-2.9.txt new file mode 100644 index 00000000000..2913072490d --- /dev/null +++ b/tests/requirements-2.9.txt @@ -0,0 +1,12 @@ +-r ../requirements-2.9.txt +yamllint==1.19.0 +apache-libcloud==2.2.1 +tox==3.11.1 +dopy==0.3.7 +cryptography==2.8 +ansible-lint==5.0.11 +openshift==0.8.8 +molecule==3.0.6 +molecule-vagrant==0.3 +testinfra==5.2.2 +python-vagrant==0.5.15 diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 2524ef93ce6..00000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ --r ../requirements.txt -yamllint==1.19.0 -apache-libcloud==2.2.1 -tox==3.11.1 -dopy==0.3.7 -cryptography==2.8 -ansible-lint==5.0.11 -openshift==0.8.8 -molecule==3.0.6 -molecule-vagrant==0.3 -testinfra==5.2.2 -python-vagrant==0.5.15 diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 120000 index 00000000000..5202ea4fbea --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +requirements-2.10.txt \ No newline at end of file diff --git a/tests/scripts/testcases_prepare.sh b/tests/scripts/testcases_prepare.sh index d70086a2b1d..bfaf65f1869 100755 --- a/tests/scripts/testcases_prepare.sh +++ b/tests/scripts/testcases_prepare.sh @@ -1,9 +1,18 @@ #!/bin/bash set -euxo pipefail -/usr/bin/python -m pip uninstall -y ansible -/usr/bin/python -m pip install -r tests/requirements.txt +: ${ANSIBLE_MAJOR_VERSION:=2.10} + +/usr/bin/python -m pip uninstall -y ansible ansible-base ansible-core +/usr/bin/python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt mkdir -p /.ssh mkdir -p cluster-dump mkdir -p $HOME/.ssh ansible-playbook --version + +# in some cases we may need to bring in collections or roles from ansible-galaxy +# to compensate for missing functionality in older ansible versions +if [ -f requirements-${ANSIBLE_MAJOR_VERSION}.yml ] ; then + ansible-galaxy role install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml + ansible-galaxy collection install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml +fi diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 2461d29c6c6..a6b595edb2d 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -86,27 +86,26 @@ ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} ${CI_TEST_ADDITIO ## Kubernetes conformance tests ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/100_check-k8s-conformance.yml $ANSIBLE_LOG_LEVEL -## Idempotency checks 1/5 (repeat deployment) if [ "${IDEMPOT_CHECK}" = "true" ]; then + ## Idempotency checks 1/5 (repeat deployment) ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} ${CI_TEST_ADDITIONAL_VARS} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml -fi -## Idempotency checks 2/5 (Advanced DNS checks) -if [ "${IDEMPOT_CHECK}" = "true" ]; then + ## Idempotency checks 2/5 (Advanced DNS checks) ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml -fi -## Idempotency checks 3/5 (reset deployment) -if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml -fi + if [ "${RESET_CHECK}" = "true" ]; then + ## Idempotency checks 3/5 (reset deployment) + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml + + ## Idempotency checks 4/5 (redeploy after reset) + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml -## Idempotency checks 4/5 (redeploy after reset) -if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml + ## Idempotency checks 5/5 (Advanced DNS checks) + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml + fi fi -## Idempotency checks 5/5 (Advanced DNS checks) -if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then - ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml +# Clean up at the end, this is to allow stage1 tests to include cleanup test +if [ "${RESET_CHECK}" = "true" ]; then + ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml fi