Merge pull request #680 from nik-netlox/main

k3s sctpmh with seagull test case added

cicd/k3s-sctpmh-seagull/Vagrantfile

@@ -0,0 +1,71 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+workers = (ENV['WORKERS'] || "1").to_i
+#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64")
+box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s")
+box_version = "0.7.1"
+Vagrant.configure("2") do |config|
+  config.vm.box = "#{box_name}"
+  config.vm.box_version = "#{box_version}"
+
+  if Vagrant.has_plugin?("vagrant-vbguest")
+    config.vbguest.auto_update = false
+  end
+
+  config.vm.define "bastion" do |bastion|
+    bastion.vm.hostname = 'bastion'
+    bastion.vm.network :private_network, ip: "4.0.5.3", :netmask => "255.255.255.0"
+    bastion.vm.network :private_network, ip: "4.0.4.3", :netmask => "255.255.255.0"
+    bastion.vm.provision :shell, :path => "bastion.sh"
+    bastion.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 2048]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+
+  config.vm.define "loxilb1" do |loxilb|
+    loxilb.vm.hostname = 'llb1'
+    loxilb.vm.network :private_network, ip: "4.0.5.10", :netmask => "255.255.255.0"
+    loxilb.vm.network :private_network, ip: "4.0.4.10", :netmask => "255.255.255.0"
+    loxilb.vm.network :private_network, ip: "4.0.6.10", :netmask => "255.255.255.0"
+    loxilb.vm.provision :shell, :path => "loxilb1.sh"
+    loxilb.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 6000]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+    end
+  end
+
+  config.vm.define "loxilb2" do |loxilb|
+    loxilb.vm.hostname = 'llb2'
+    loxilb.vm.network :private_network, ip: "4.0.5.11", :netmask => "255.255.255.0"
+    loxilb.vm.network :private_network, ip: "4.0.4.11", :netmask => "255.255.255.0"
+    loxilb.vm.network :private_network, ip: "4.0.6.11", :netmask => "255.255.255.0"
+    loxilb.vm.provision :shell, :path => "loxilb2.sh"
+    loxilb.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 6000]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc4", "allow-all"]
+    end
+  end
+
+  config.vm.define "master" do |master|
+    master.vm.hostname = 'master'
+    master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0"
+    master.vm.network :private_network, ip: "4.0.6.150", :netmask => "255.255.255.0"
+    master.vm.provision :shell, :path => "master.sh"
+    master.vm.provider :virtualbox do |vbox|
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 4]
+        vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+        vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    end
+  end
+end

cicd/k3s-sctpmh-seagull/bastion.sh

@@ -0,0 +1,17 @@
+apt-get update
+apt-get install -y software-properties-common curl wget lksctp-tools
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN -dit -p 80:80 --name tcp_ep ghcr.io/loxilb-io/nginx:stable
+docker run -u root --cap-add SYS_ADMIN -dit --entrypoint sctp_darn --name sctp_ep loxilbio/sctp-darn:latest -H 0.0.0.0 -P 9999 -l
+
+echo "blacklist sctp" >>  /etc/modprobe.d/blacklist.conf
+echo "install sctp /bin/false" >>  /etc/modprobe.d/blacklist.conf
+
+sysctl -w net.ipv4.conf.eth1.arp_accept=1 >> /etc/sysctl.conf
+sysctl -w net.ipv4.conf.eth2.arp_accept=1 >> /etc/sysctl.conf
+
+reboot
+"

cicd/k3s-sctpmh-seagull/config.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+vagrant global-status  | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f
+vagrant up
+sleep 30
+vagrant ssh bastion -c 'sudo /vagrant/seagull.sh'

cicd/k3s-sctpmh-seagull/dummy.yml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: dummy
+  annotations:
+      loxilb.io/staticIP : "4.0.6.100"
+spec:
+ loadBalancerClass: loxilb.io/loxilb
+ type: LoadBalancer 
+ ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: dummy
+subsets:
+  - addresses:
+    - ip: 4.0.5.3
+    ports:
+    - port: 80

cicd/k3s-sctpmh-seagull/kube-loxilb.yml

@@ -0,0 +1,130 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - services
+      - services/status
+    verbs:
+      - get
+      - watch
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - discovery.k8s.io
+    resources:
+      - endpointslices
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-loxilb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-loxilb
+subjects:
+  - kind: ServiceAccount
+    name: kube-loxilb
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-loxilb
+  namespace: kube-system
+  labels:
+    app: loxilb
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loxilb
+  template:
+    metadata:
+      labels:
+        app: loxilb
+    spec:
+      hostNetwork: true
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-loxilb
+      terminationGracePeriodSeconds: 0
+      containers:
+      - name: kube-loxilb
+        image: ghcr.io/loxilb-io/kube-loxilb:latest
+        imagePullPolicy: Always
+        command:
+        - /bin/kube-loxilb
+        args:
+        - --loxiURL=http://4.0.6.10:11111,http://4.0.6.11:11111
+        - --externalCIDR=4.0.5.100/32
+        - --externalSecondaryCIDRs=4.0.4.100/24
+        #- --monitor
+        #- --setBGP
+        #- --setLBMode=1
+        #- --config=/opt/loxilb/agent/kube-loxilb.conf
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW"]

cicd/k3s-sctpmh-seagull/loxilb1.sh

@@ -0,0 +1,13 @@
+export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=4.0.5.11 --self=0 --ka=4.0.5.11:4.0.5.10
+echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc
+echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc
+
+echo $LOXILB_IP > /vagrant/loxilb-ip1

cicd/k3s-sctpmh-seagull/loxilb2.sh

@@ -0,0 +1,13 @@
+export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+apt-get update
+apt-get install -y software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+apt-get update
+apt-get install -y docker-ce
+docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest --cluster=4.0.5.10 --self=1 --ka=4.0.5.10:4.0.5.11
+echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc
+echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc
+
+echo $LOXILB_IP > /vagrant/loxilb-ip1

cicd/k3s-sctpmh-seagull/master.sh

@@ -0,0 +1,38 @@
+export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/')
+
+curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller  \
+--node-ip=${MASTER_IP} --node-external-ip=${MASTER_IP} \
+--bind-address=${MASTER_IP}" sh -
+
+echo $MASTER_IP > /vagrant/master-ip
+sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
+sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
+sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml
+sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sudo kubectl apply -f /vagrant/multus/multus-daemonset.yml
+sudo kubectl apply -f /vagrant/multus/macvlan.yml
+/vagrant/wait_ready.sh
+
+sudo apt update
+sudo apt install -y snapd
+sudo snap install go --classic
+
+git clone https://github.com/containernetworking/plugins.git
+cd plugins
+./build_linux.sh
+ls bin/macvlan
+sudo cp -f ./bin/macvlan /var/lib/rancher/k3s/data/current/bin/
+sudo ifconfig eth2 promisc
+
+sudo kubectl apply -f /vagrant/dummy.yml
+sudo kubectl apply -f /vagrant/multus/multus-seagull-pod.yml
+sudo kubectl apply -f /vagrant/multus/multus-seagull-service.yml
+/vagrant/wait_ready.sh
+
+sudo sysctl -w net.ipv4.conf.all.arp_accept=1 >> /etc/sysctl.conf
+
+echo "blacklist sctp" >>  /etc/modprobe.d/blacklist.conf
+echo "install sctp /bin/false" >>  /etc/modprobe.d/blacklist.conf
+
+reboot
+

cicd/k3s-sctpmh-seagull/multus/macvlan.yml

@@ -0,0 +1,27 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan1
+spec:
+  config: '{
+            "cniVersion": "0.3.1",
+            "type": "macvlan",
+            "master": "eth2",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "4.0.6.0/24",
+                         "rangeStart": "4.0.6.3",
+                         "rangeEnd": "4.0.6.100",
+                         "routes": [
+                            {
+                              "dst": "0.0.0.0/0"
+                            }
+                         ],
+                         "gateway": "4.0.6.149"
+                    } ]
+                ]
+            }
+        }'