From 4fa6d8bd588e84086cd3a090313e5d45ebfb7996 Mon Sep 17 00:00:00 2001 From: Shelly Kagan Date: Tue, 30 Jan 2024 17:45:31 +0200 Subject: [PATCH] Update kubevirtci due to a bug in last version Signed-off-by: Shelly Kagan --- cluster-up/README.md | 20 +++- .../{k8s-1.25 => k8s-1.29}/provider.sh | 0 cluster-up/cluster/kind-1.23/README.md | 45 --------- cluster-up/cluster/kind-1.23/provider.sh | 51 ---------- .../{kind-1.27 => kind-1.28}/README.md | 0 .../{kind-1.27 => kind-1.28}/provider.sh | 6 +- cluster-up/cluster/kind/common.sh | 15 ++- cluster-up/cluster/local/README.md | 98 ------------------- cluster-up/cluster/local/provider.sh | 56 ----------- cluster-up/hack/common.sh | 4 +- 10 files changed, 38 insertions(+), 257 deletions(-) rename cluster-up/cluster/{k8s-1.25 => k8s-1.29}/provider.sh (100%) delete mode 100644 cluster-up/cluster/kind-1.23/README.md delete mode 100755 cluster-up/cluster/kind-1.23/provider.sh rename cluster-up/cluster/{kind-1.27 => kind-1.28}/README.md (100%) rename cluster-up/cluster/{kind-1.27 => kind-1.28}/provider.sh (89%) delete mode 100644 cluster-up/cluster/local/README.md delete mode 100644 cluster-up/cluster/local/provider.sh diff --git a/cluster-up/README.md b/cluster-up/README.md index 8fdcad66..a5c20347 100644 --- a/cluster-up/README.md +++ b/cluster-up/README.md @@ -1,4 +1,22 @@ -# How to use cluster-up +# cluster-up + +## Prerequisites: podman or docker + +cluster-up requires that either podman or docker be installed on the host. + +If podman is being used, it is also necessary to enable podman socket with: + +``` +sudo systemctl enable podman.socket +sudo systemctl start podman.socket +``` + +for more information see: + +https://github.com/kubevirt/kubevirtci/blob/main/PODMAN.md + + +## How to use cluster-up This directory provides a wrapper around gocli. It can be vendored into other git repos and integrated to provide in the kubevirt well-known cluster commands diff --git a/cluster-up/cluster/k8s-1.25/provider.sh b/cluster-up/cluster/k8s-1.29/provider.sh similarity index 100% rename from cluster-up/cluster/k8s-1.25/provider.sh rename to cluster-up/cluster/k8s-1.29/provider.sh diff --git a/cluster-up/cluster/kind-1.23/README.md b/cluster-up/cluster/kind-1.23/README.md deleted file mode 100644 index 05c97eab..00000000 --- a/cluster-up/cluster/kind-1.23/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# K8S 1.23.3 with mdev support in a Kind cluster - -Provides a pre-deployed k8s cluster with version 1.23.3 that runs using [kind](https://github.com/kubernetes-sigs/kind) The cluster is completely ephemeral and is recreated on every cluster restart. -The KubeVirt containers are built on the local machine and are then pushed to a registry which is exposed at -`localhost:5000`. - -## Bringing the cluster up - -The following needs to be executed as root. - -```bash -export KUBEVIRT_PROVIDER=kind-1.23 -make cluster-up -``` - -The cluster can be accessed as usual: - -```bash -$ cluster-up/kubectl.sh get nodes -NAME STATUS ROLES AGE VERSION -kind-1.23-control-plane Ready master 6m14s v1.23.3 -``` - -## Bringing the cluster down - -```bash -make cluster-down -``` - -This destroys the whole cluster. - -## Setting a custom kind version - -In order to use a custom kind image / kind version, -export KIND_NODE_IMAGE, KIND_VERSION, KUBECTL_PATH before running cluster-up. -For example in order to use kind 0.9.0 (which is based on k8s-1.19.1) use: -```bash -export KIND_NODE_IMAGE="kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600" -export KIND_VERSION="0.9.0" -export KUBECTL_PATH="/usr/bin/kubectl" -``` -This allows users to test or use custom images / different kind versions before making them official. -See https://github.com/kubernetes-sigs/kind/releases for details about node images according to the kind version. - -- In order to use `make cluster-down` please make sure the right `CLUSTER_NAME` is exported. diff --git a/cluster-up/cluster/kind-1.23/provider.sh b/cluster-up/cluster/kind-1.23/provider.sh deleted file mode 100755 index 137f677b..00000000 --- a/cluster-up/cluster/kind-1.23/provider.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -set -e - -DEFAULT_CLUSTER_NAME="kind-1.23" -DEFAULT_HOST_PORT=5000 -ALTERNATE_HOST_PORT=5001 -export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME} - -if [ $CLUSTER_NAME == $DEFAULT_CLUSTER_NAME ]; then - export HOST_PORT=$DEFAULT_HOST_PORT -else - export HOST_PORT=$ALTERNATE_HOST_PORT -fi - -function set_kind_params() { - export KIND_VERSION="${KIND_VERSION:-0.12.0}" - export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-quay.io/kubevirtci/kindest-node:v1.23.4}" - export KUBECTL_PATH="${KUBECTL_PATH:-/bin/kubectl}" -} - -function configure_registry_proxy() { - [ "$CI" != "true" ] && return - - echo "Configuring cluster nodes to work with CI mirror-proxy..." - - local -r ci_proxy_hostname="docker-mirror-proxy.kubevirt-prow.svc" - local -r kind_binary_path="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kind" - local -r configure_registry_proxy_script="${KUBEVIRTCI_PATH}/cluster/kind/configure-registry-proxy.sh" - - KIND_BIN="$kind_binary_path" PROXY_HOSTNAME="$ci_proxy_hostname" $configure_registry_proxy_script -} - -function up() { - cp $KIND_MANIFESTS_DIR/kind.yaml ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml - _add_kubeadm_cpu_manager_config_patch - _add_extra_mounts - export CONFIG_WORKER_CPU_MANAGER=true - kind_up - - configure_registry_proxy - - # remove the rancher.io kind default storageClass - _kubectl delete sc standard - - echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready" -} - -set_kind_params - -source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh diff --git a/cluster-up/cluster/kind-1.27/README.md b/cluster-up/cluster/kind-1.28/README.md similarity index 100% rename from cluster-up/cluster/kind-1.27/README.md rename to cluster-up/cluster/kind-1.28/README.md diff --git a/cluster-up/cluster/kind-1.27/provider.sh b/cluster-up/cluster/kind-1.28/provider.sh similarity index 89% rename from cluster-up/cluster/kind-1.27/provider.sh rename to cluster-up/cluster/kind-1.28/provider.sh index 6dd01f5e..69008abc 100755 --- a/cluster-up/cluster/kind-1.27/provider.sh +++ b/cluster-up/cluster/kind-1.28/provider.sh @@ -2,7 +2,7 @@ set -e -DEFAULT_CLUSTER_NAME="kind-1.27" +DEFAULT_CLUSTER_NAME="kind-1.28" DEFAULT_HOST_PORT=5000 ALTERNATE_HOST_PORT=5001 export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME} @@ -14,8 +14,8 @@ else fi function set_kind_params() { - export KIND_VERSION="${KIND_VERSION:-0.19.0}" - export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.27.1@sha256:b7d12ed662b873bd8510879c1846e87c7e676a79fefc93e17b2a52989d3ff42b}" + export KIND_VERSION="${KIND_VERSION:-0.20.0}" + export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31}" export KUBECTL_PATH="${KUBECTL_PATH:-/usr/bin/kubectl}" } diff --git a/cluster-up/cluster/kind/common.sh b/cluster-up/cluster/kind/common.sh index 7003ba2f..2ec48605 100755 --- a/cluster-up/cluster/kind/common.sh +++ b/cluster-up/cluster/kind/common.sh @@ -8,6 +8,9 @@ function detect_cri() { export CRI_BIN=${CRI_BIN:-$(detect_cri)} CONFIG_WORKER_CPU_MANAGER=${CONFIG_WORKER_CPU_MANAGER:-false} +# only setup ipFamily when the environmental variable is not empty +# avaliable value: ipv4, ipv6, dual +IPFAMILY=${IPFAMILY} # check CPU arch PLATFORM=$(uname -m) @@ -286,10 +289,20 @@ EOF fi } +function _setup_ipfamily() { + if [ $IPFAMILY != "" ]; then + cat <> ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml +networking: + ipFamily: $IPFAMILY +EOF + echo "KIND cluster ip family has been set to $IPFAMILY" + fi +} + function _prepare_kind_config() { _add_workers _add_kubeadm_config_patches - + _setup_ipfamily echo "Final KIND config:" cat ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml } diff --git a/cluster-up/cluster/local/README.md b/cluster-up/cluster/local/README.md deleted file mode 100644 index 6b897254..00000000 --- a/cluster-up/cluster/local/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Local Kubernetes Provider - -This provider allows developing against bleeding-edge Kubernetes code. The -k8s sources will be compiled and a single-node cluster will be started. - -## Environment preparation - -Since the `local` provider deploys the cluster on the host and not inside -virtual machines, you may need to adjust some settings on the node. - -Specifically, you may need to make sure that your firewall of choice doesn't -block connectivity between cluster IP and service pods. If you experience -connectivity issues, consider tweaking or disabling your firewall. Also, make -sure forwarding is enabled on the host: - -```bash -$ systemctl disable firewalld --now -$ iptables -P FORWARD ACCEPT -$ sysctl net.ipv4.conf.all.forwarding=1 -``` - -## Bringing the cluster up - -First get the k8s sources: - -```bash -go get -u -d k8s.io/kubernetes -``` - -Then compile and start the cluster: - -```bash -export KUBEVIRT_PROVIDER=local -make cluster-up -``` - -The cluster can be accessed as usual: - -```bash -$ cluster/kubectl.sh get nodes -NAME STATUS ROLES AGE VERSION -kubdev Ready 5m20s v1.12.0-beta.2 -``` - -Note: you may need to cherry-pick -[acdb1b0e9855ab671f2972f10605d20cad26284b](https://github.com/kubernetes/kubernetes/commit/acdb1b0e9855ab671f2972f10605d20cad26284b) -if it's not present in your kubernetes tree yet. - -## CNI - -By default, local provider deploys cluster with no CNI support. To make CNI -work, you should set the following variables before spinning up cluster: - -```bash -$ export NET_PLUGIN=cni -$ export CNI_CONF_DIR=/etc/cni/net.d/ -$ export CNI_BIN_DIR=/opt/cni/bin/ -``` - -Depending on your CNI of choice (for example, Flannel), you may also need to -add the following arguments to controller-manager inside -`hack/local-cluster-up.sh`: - -```bash -$ git diff -diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh -index bcf988b..9911eed 100755 ---- a/hack/local-up-cluster.sh -+++ b/hack/local-up-cluster.sh -@@ -639,6 +639,8 @@ function start_controller_manager { - --use-service-account-credentials \ - --controllers="${KUBE_CONTROLLERS}" \ - --leader-elect=false \ - --cert-dir="$CERT_DIR" \ -+ --allocate-node-cidrs=true --cluster-cidr=10.244.0.0/16 \ - --master="https://${API_HOST}:${API_SECURE_PORT}" >"${CTLRMGR_LOG}" 2>&1 & - CTLRMGR_PID=$! - } -``` - -Also, you will need to install [reference CNI plugins](https://github.com/containernetworking/plugins): - -```bash -$ go get -u -d github.com/containernetworking/plugins/ -$ cd $GOPATH/src/github.com/containernetworking/plugins/ -$ ./build.sh -$ sudo mkdir -p /opt/cni/bin/ -$ sudo cp bin/* /opt/cni/bin/ -``` - -In some cases (for example, Multus), your CNI plugin may also require presence -of `/etc/kubernetes/kubelet.conf` file. In this case, you should create a -symlink pointing to the right location: - -```bash -$ sudo mkdir /etc/kubernetes -$ sudo ln -s $GOPATH/src/kubevirt.io/kubevirt/cluster/local/certs/kubelet.kubeconfig /etc/kubernetes/kubelet.conf -``` diff --git a/cluster-up/cluster/local/provider.sh b/cluster-up/cluster/local/provider.sh deleted file mode 100644 index af10d52f..00000000 --- a/cluster-up/cluster/local/provider.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -function _cert_dir() { - echo $GOPATH/src/kubevirt.io/kubevirt/cluster/local/certs -} - -function _main_ip() { - ip -o -4 a | tr -s ' ' | cut -d' ' -f 2,4 | - grep -v -e '^lo[0-9:]*' | head -1 | - cut -d' ' -f 2 | cut -d'/' -f1 -} - -function up() { - # Make sure that local config is correct - prepare_config - - go get -d k8s.io/kubernetes - - export API_HOST_IP=$(_main_ip) - export KUBELET_HOST=$(_main_ip) - export HOSTNAME_OVERRIDE=kubdev - export ALLOW_PRIVILEGED=1 - export ALLOW_SECURITY_CONTEXT=1 - export KUBE_DNS_DOMAIN="cluster.local" - export KUBE_DNS_SERVER_IP="10.0.0.10" - export KUBE_ENABLE_CLUSTER_DNS=true - export CERT_DIR=$(_cert_dir) - ( - cd $GOPATH/src/k8s.io/kubernetes - ./hack/local-up-cluster.sh - ) -} - -function prepare_config() { - PROVIDER_CONFIG_FILE_PATH="${BASE_PATH}/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh" - cat > "$PROVIDER_CONFIG_FILE_PATH" <