From 5f257843810d50c4d800ef8332b85f7d76b86056 Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Tue, 8 Mar 2022 10:06:01 +0100 Subject: [PATCH] [meta] update docker images (#1604) (#1608) * [meta] update docker images - use python slim base images - bump python version - bump python dependencies - bump tools version * cleanup dependencies * lint test files with new black version * link gcloud to path * do not build helm-tester docker image by default --- ...+helm-charts+{branch}+template-testing.yml | 32 ++--- elasticsearch/tests/elasticsearch_test.py | 6 +- filebeat/tests/filebeat_test.py | 101 +++++++--------- helpers/common.mk | 2 +- helpers/helm-tester/Dockerfile | 6 +- helpers/helm-tester/requirements.txt | 12 +- helpers/terraform/Dockerfile | 73 ++++++------ logstash/tests/logstash_test.py | 12 +- metricbeat/tests/metricbeat_test.py | 111 +++++++++--------- requirements.txt | 14 +-- 10 files changed, 172 insertions(+), 197 deletions(-) mode change 100755 => 100644 elasticsearch/tests/elasticsearch_test.py diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+template-testing.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+template-testing.yml index 1f09fea4a..329300e42 100644 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+template-testing.yml +++ b/.ci/jobs.t/elastic+helm-charts+{branch}+template-testing.yml @@ -4,22 +4,22 @@ display-name: elastic / helm-charts - %BRANCH% - template testing description: template testing scm: - - git: - wipe-workspace: 'True' + - git: + wipe-workspace: "True" axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: CHART - filename: helpers/matrix.yml + - axis: + type: slave + name: label + values: + - docker&&virtual + - axis: + type: yaml + name: CHART + filename: helpers/matrix.yml builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail + - shell: |- + #!/usr/local/bin/runbld + set -euo pipefail - cd ${CHART} - make test + cd ${CHART} + make build test diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py old mode 100755 new mode 100644 index 45a300f43..1c843b55f --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -1544,7 +1544,11 @@ def test_network_policy(): {"podSelector": {"matchLabels": {"app": "elasticsearch-master"}}}, ] assert transport["ports"][0]["port"] == 9300 - assert pod_selector == {"matchLabels": {"app": "elasticsearch-master",}} + assert pod_selector == { + "matchLabels": { + "app": "elasticsearch-master", + } + } def test_default_automount_sa_token(): diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 0a1c100dc..48e2f2077 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -685,29 +685,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -722,29 +716,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -761,15 +749,12 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -810,12 +795,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] config = """ deployment: @@ -838,12 +824,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] def test_adding_a_deprecated_extra_volume_with_volume_mount(): diff --git a/helpers/common.mk b/helpers/common.mk index f7debead3..734f91f2a 100644 --- a/helpers/common.mk +++ b/helpers/common.mk @@ -32,7 +32,7 @@ template: ## Render chart templates helm template ./ .PHONY: test -test: build ## Run all tests in a docker container +test: ## Run all tests in a docker container docker run --rm -i --user "$$(id -u):$$(id -g)" -v $$(pwd)/../:/app -w /app/$$(basename $$(pwd)) helm-tester make test-all .PHONY: test-all ## Run all tests diff --git a/helpers/helm-tester/Dockerfile b/helpers/helm-tester/Dockerfile index b267e047f..d77bd798b 100644 --- a/helpers/helm-tester/Dockerfile +++ b/helpers/helm-tester/Dockerfile @@ -1,7 +1,11 @@ -FROM python:3.7 +FROM python:3.10.2-slim ENV HELM_VERSION=3.8.0 +RUN apt-get update && apt-get upgrade -y && \ + apt-get install --no-install-recommends -y make wget && \ + rm -rf /var/lib/apt/lists/* + RUN wget --no-verbose https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ diff --git a/helpers/helm-tester/requirements.txt b/helpers/helm-tester/requirements.txt index 7d643131b..17aff7403 100644 --- a/helpers/helm-tester/requirements.txt +++ b/helpers/helm-tester/requirements.txt @@ -1,10 +1,2 @@ -atomicwrites==1.3.0 -attrs==19.1.0 -importlib-metadata==0.23 -more-itertools==7.2.0 -pluggy==0.13.0 -py==1.10.0 -pytest==4.1.0 -PyYAML==5.4 -six==1.12.0 -zipp==0.6.0 +pytest==7.0.1 +PyYAML==6.0 diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile index 3f7a0b95c..f5a918532 100644 --- a/helpers/terraform/Dockerfile +++ b/helpers/terraform/Dockerfile @@ -1,53 +1,48 @@ -FROM centos:7 +FROM python:3.10.2-slim -ENV VAULT_VERSION 0.9.3 -ENV TERRAFORM_VERSION=0.11.7 -ENV KUBECTL_VERSION=1.21.6 +ENV DOCKER_VERSION=20.10.12 +ENV GCLOUD_VERSION=375.0.0 ENV HELM_VERSION=3.8.0 -ENV DOCKER_VERSION=18.09.7 -ENV JQ_VERSION=1.6 +ENV KUBECTL_VERSION=1.21.6 +ENV TERRAFORM_VERSION=0.11.7 +ENV VAULT_VERSION=1.9.3 -RUN yum -y install \ - make \ - openssl \ - unzip \ - which +RUN apt-get update && apt-get upgrade -y && \ + apt-get install --no-install-recommends -y curl jq make openssl unzip && \ + rm -rf /var/lib/apt/lists/* -RUN yum -y install \ - https://packages.cloud.google.com/yum/pool/654962a43cb6bc6b47bc312424410b7c203e1beb73386d868d9e614839c82609-google-cloud-sdk-215.0.0-1.el7.noarch.rpm && \ - gcloud config set core/disable_usage_reporting true && \ - gcloud config set component_manager/disable_update_check true && \ - gcloud version +RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ + tar xvf google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ + ./google-cloud-sdk/install.sh --quiet --usage-reporting=false --path-update=true && \ + rm -fr google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \ + ln -s /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud version RUN curl -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ - unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ - chmod +x /usr/local/bin/vault && \ - rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \ - vault version + unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ + chmod +x /usr/local/bin/vault && \ + rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \ + vault version RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ - rm -f terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - terraform version + unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ + rm -f terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + terraform version RUN curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ - mv kubectl /usr/local/bin/ && \ - chmod a+x /usr/local/bin/kubectl && \ - kubectl version --client + mv kubectl /usr/local/bin/ && \ + chmod a+x /usr/local/bin/kubectl && \ + kubectl version --client RUN curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ - tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ - mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \ - helm version --client + tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ + mv linux-amd64/helm /usr/local/bin/ && \ + rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \ + helm version --client RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ - tar xfv docker* && \ - mv docker/docker /usr/local/bin && \ - rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \ - docker -v - -RUN curl -O -L https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 && \ - mv jq-linux64 /usr/local/bin/jq && \ - chmod a+x /usr/local/bin/jq && \ - jq --version + tar xfv docker* && \ + mv docker/docker /usr/local/bin && \ + rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \ + docker -v diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py index ab03cac22..112f2ab6c 100644 --- a/logstash/tests/logstash_test.py +++ b/logstash/tests/logstash_test.py @@ -243,7 +243,10 @@ def test_adding_a_extra_port(): extraPorts = r["statefulset"][name]["spec"]["template"]["spec"]["containers"][0][ "ports" ] - assert {"name": "foo", "containerPort": 30000,} in extraPorts + assert { + "name": "foo", + "containerPort": 30000, + } in extraPorts def test_adding_a_extra_init_container(): @@ -660,9 +663,10 @@ def test_adding_in_logstash_config(): s = r["statefulset"][name]["spec"]["template"]["spec"] - assert {"configMap": {"name": name + "-config"}, "name": "logstashconfig",} in s[ - "volumes" - ] + assert { + "configMap": {"name": name + "-config"}, + "name": "logstashconfig", + } in s["volumes"] assert { "mountPath": "/usr/share/logstash/config/logstash.yml", "name": "logstashconfig", diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index bcfcb5d8a..b4d8cc09b 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -631,29 +631,27 @@ def test_adding_a_secret_mount(): path: /usr/share/metricbeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/metricbeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/metricbeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/metricbeat/config/certs", - "name": "elastic-certificates", - } - not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"][ - "containers" - ][0]["volumeMounts"] - ) + assert { + "mountPath": "/usr/share/metricbeat/config/certs", + "name": "elastic-certificates", + } not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"][ + "containers" + ][ + 0 + ][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -667,29 +665,25 @@ def test_adding_a_secret_mount(): path: /usr/share/metricbeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/metricbeat/config/certs", - "name": "elastic-certificates", - } - in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][ - 0 - ]["volumeMounts"] - ) + assert { + "mountPath": "/usr/share/metricbeat/config/certs", + "name": "elastic-certificates", + } in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][ + 0 + ][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/metricbeat/config/certs", - "name": "elastic-certificates", - } - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/metricbeat/config/certs", + "name": "elastic-certificates", + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -704,15 +698,12 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/metricbeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/metricbeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/metricbeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -753,12 +744,17 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name + "-metrics"][ "spec" ]["template"]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"][ - "containers" - ][0]["volumeMounts"] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"][ + "containers" + ][ + 0 + ][ + "volumeMounts" + ] config = """ deployment: @@ -780,12 +776,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] def test_adding_a_deprecated_extra_volume_with_volume_mount(): diff --git a/requirements.txt b/requirements.txt index 35e093e54..391784a92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,3 @@ -atomicwrites==1.3.0 -attrs==19.1.0 -black==19.10b0 -importlib-metadata==0.23 -more-itertools==7.2.0 -pluggy==0.13.0 -py==1.10.0 -pytest==4.1.0 -PyYAML==5.4 -six==1.12.0 -zipp==0.6.0 +black==22.1.0 +pytest==7.0.1 +PyYAML==6.0