Feature/worker node (#5)

* remove cluster token and instead generate a random one. * add worker node support * add worker node support * upgraded the terraform providers * update readme to show the changes. * worker nodes must depend on control plane node join
balchua · Aug 7, 2022 · c0d8320 · c0d8320
1 parent c164777
commit c0d8320
Show file tree

Hide file tree

Showing 10 changed files with 277 additions and 20 deletions.
diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl
diff --git a/README.md b/README.md
@@ -1,35 +1,40 @@
 # DigitalOcean Terraform MicroK8s
 
-**This currently works for `1.19+` channel.**
-
-**Use only with terraform v0.14**
+**Verfied using terraform v1.2.6**
 
 ~~Does not work when modifying the module after it is created.~~
 **Adding a new node now works**
+
 **Warning Reducing nodes still does not leave the cluster**
 
+**Support for worker only node, which means it will not run control plane components such as the api-server, scheduler and controller manager, available from MicroK8s v1.22**
+
 Bootstrap a Highly Available MicroK8s cluster in DigitalOcean with Terraform.
 
-For example to bootstrap a 7 node cluster.
+For example to bootstrap a 3 control plane nodes and 2 worker nodes cluster.
 
 ```hcl
 
 module "microk8s" {
-  source                       = "git::https://github.com/balchua/do-microk8s?ref=master"
-  node_count                   = "7"
+  source                       = "../"
+  cluster_name                 = "hades"
+  node_count                   = "3"
+  worker_node_count            = "2"
   os_image                     = "ubuntu-20-04-x64"
-  node_size                    = "s-4vcpu-8gb"
-  node_disksize                = "2"
+  node_size                    = "s-2vcpu-4gb"
+  worker_node_size             = "s-4vcpu-8gb"
+  node_disksize                = "30"
   region                       = "sgp1"
   dns_zone                     = "geeks.sg"
-  microk8s_channel             = "latest/edge"
+  microk8s_channel             = "latest/stable"
   cluster_token_ttl_seconds    = 3600
   digitalocean_ssh_fingerprint = var.digitalocean_ssh_fingerprint
   digitalocean_private_key     = var.digitalocean_private_key
   digitalocean_token           = var.digitalocean_token
   digitalocean_pub_key         = var.digitalocean_pub_key
 }
 
+
 ```
 
 | Fields                        | Description                              | Default values |
@@ -38,7 +43,7 @@ module "microk8s" {
 | node_count                    | The number of MicroK8s nodes to create   | 3
 | os_image                      | DigitalOcean OS images.  <br/>To get the list OS images `doctl compute image list-distribution`| ubuntu-20-04-x64
 | node_size                     | DigitalOcean droptlet sizes <br/> To get the list of droplet sizes `doctl compute size list`| s-4vcpu-8gb
-| node_disksize                 | Additional volume to add to the droplet.  Size in GB| 100 |
+| node_disksize                 | Additional volume to add to the droplet.  Size in GB| 50 |
 | region                        | DigitalOcean region <br/> To get the list of regions `doctl compute region list`| sgp1
 | dns_zone                      | The DNS zone representing your site.  Need to register your domain. | geeks.sg
 | microk8s_channel              | Specify the MicroK8s channel to use.  Refer [here](https://snapcraft.io/microk8s)| stable
@@ -47,6 +52,9 @@ module "microk8s" {
 | digitalocean_private_key      | The private key location to use when connecting to your droplets| Refer to `TF` environment variables
 | digitalocean_token            | Your DigitalOcean token| Refer to `TF` environment variables
 | digitalocean_pub_key          | The public key to use to connect to the droplet| Refer to `TF` environment variables
+| worker_node_size              | The worker node size example: `s-4vcpu-8gb` | s-4vcpu-8gb
+| worker_node_count             | The number of MicroK8s worker nodes | 2
+| worker_node_disksize          | Additional volume to add to the droplet.  Size in GB| 100 |
 
 
 ## DigitalOcean TF environment variables

diff --git a/etc_hosts.tf b/etc_hosts.tf
@@ -16,6 +16,29 @@ resource "null_resource" "provision_node_hosts_file" {
   provisioner "remote-exec" {
     inline = [
       "echo '${join("\n", formatlist("%v", digitalocean_droplet.microk8s-node.*.ipv4_address_private))}' | awk 'BEGIN{ print \"\\n\\n# Node members:\" }; { print $0 \" microk8s-node-${var.cluster_name}-\"  NR-1}' | sudo tee -a /etc/hosts > /dev/null",
+"echo '${join("\n", formatlist("%v", digitalocean_droplet.microk8s-worker-node.*.ipv4_address_private))}' | awk 'BEGIN{ print \"\\n\\n# Worker node members:\" }; { print $0 \" microk8s-worker-${var.cluster_name}-\"  NR-1}' | sudo tee -a /etc/hosts > /dev/null",      
+    ]
+  }
+}
+
+
+resource "null_resource" "provision_worker_hosts_file" {
+  count = var.worker_node_count
+  triggers = {
+    rerun = random_id.cluster_token.hex
+  }  
+  connection {
+        host        = element(digitalocean_droplet.microk8s-worker-node.*.ipv4_address, count.index)
+        user        = "root"
+        type        = "ssh"
+        private_key = file(var.digitalocean_private_key)
+        timeout     = "2m"
+  }  
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo '${join("\n", formatlist("%v", digitalocean_droplet.microk8s-node.*.ipv4_address_private))}' | awk 'BEGIN{ print \"\\n\\n# Node members:\" }; { print $0 \" microk8s-node-${var.cluster_name}-\"  NR-1}' | sudo tee -a /etc/hosts > /dev/null",
+      "echo '${join("\n", formatlist("%v", digitalocean_droplet.microk8s-worker-node.*.ipv4_address_private))}' | awk 'BEGIN{ print \"\\n\\n# Worker node members:\" }; { print $0 \" microk8s-worker-${var.cluster_name}-\"  NR-1}' | sudo tee -a /etc/hosts > /dev/null",
     ]
   }
 }
diff --git a/example/main.tf b/example/main.tf
@@ -3,12 +3,14 @@ module "microk8s" {
   source                       = "../"
   cluster_name                 = "hades"
   node_count                   = "3"
+  worker_node_count            = "2"
   os_image                     = "ubuntu-20-04-x64"
-  node_size                    = "s-1vcpu-2gb"
-  node_disksize                = "2"
+  node_size                    = "s-2vcpu-4gb"
+  worker_node_size             = "s-4vcpu-8gb"
+  node_disksize                = "30"
   region                       = "sgp1"
   dns_zone                     = "geeks.sg"
-  microk8s_channel             = "latest/edge"
+  microk8s_channel             = "latest/stable"
   cluster_token_ttl_seconds    = 3600
   digitalocean_ssh_fingerprint = var.digitalocean_ssh_fingerprint
   digitalocean_private_key     = var.digitalocean_private_key

diff --git a/microk8s-node.tf b/microk8s-node.tf
@@ -59,7 +59,10 @@ resource "null_resource" "setup_tokens" {
 
     provisioner "local-exec" {
         interpreter = ["bash", "-c"]
-        command = "echo \"1\" > /tmp/current_joining_node.txt"
+        command = <<EOT
+        echo "1" > /tmp/current_joining_node.txt
+        echo "0" > /tmp/current_joining_worker_node.txt
+        EOT
     }
 
     provisioner "file" {

diff --git a/microk8s-worker-node.tf b/microk8s-worker-node.tf
@@ -0,0 +1,88 @@
+resource "digitalocean_volume" "microk8s-worker-node" {
+  region                  = var.region
+  count                   = var.node_count
+  name                    = "microk8s-worker-fs-${count.index}"
+  size                    = var.worker_node_disksize
+  description             = "A volume to attach to the worker"
+}
+
+resource "digitalocean_droplet" "microk8s-worker-node" {
+  image              = var.os_image
+  name               = "microk8s-worker-${var.cluster_name}-${count.index}"
+  region             = var.region
+  size               = var.worker_node_size
+  count              = var.worker_node_count
+  private_networking = true
+
+ tags = [
+    digitalocean_tag.microk8s-worker.id
+  ]
+
+  ssh_keys = [
+    var.digitalocean_ssh_fingerprint,
+  ]
+  user_data = element(data.template_file.node_config.*.rendered, count.index)
+  volume_ids = [element(digitalocean_volume.microk8s-worker-node.*.id, count.index)]
+
+}
+
+# Tag to label nodes
+resource "digitalocean_tag" "microk8s-worker" {
+  name = "microk8s-worker-${var.cluster_name}"
+}
+
+
+resource "null_resource" "join_workers" {
+  count      = var.worker_node_count
+  depends_on = [null_resource.setup_tokens, null_resource.join_nodes]
+  triggers = {
+    rerun = random_id.cluster_token.hex
+    }    
+    connection {
+        host        = element(digitalocean_droplet.microk8s-worker-node.*.ipv4_address, count.index)
+        user        = "root"
+        type        = "ssh"
+        private_key = file(var.digitalocean_private_key)
+        timeout     = "20m"
+    }  
+
+    provisioner "local-exec" {
+        interpreter = ["bash", "-c"]
+        command = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done"
+    }
+
+    provisioner "file" {
+        content     = templatefile("${path.module}/templates/join-worker.sh", 
+            {
+                dns_zone = var.dns_zone
+                cluster_token = random_id.cluster_token.hex
+                main_node_ip = digitalocean_droplet.microk8s-node[0].ipv4_address_private
+            })
+        destination = "/usr/local/bin/join-worker.sh"
+    }    
+
+    provisioner "remote-exec" {
+        inline = [
+            "sh /usr/local/bin/join-worker.sh"
+        ]
+    }
+
+    provisioner "local-exec" {
+        interpreter = ["bash", "-c"]
+        command = "echo \"${count.index+1}\" > /tmp/current_joining_worker_node.txt"
+    }    
+}
+
+# Discrete DNS records for each controller's private IPv4 for ingress usage
+resource "digitalocean_record" "microk8s-worker-node" {
+  count   = var.worker_node_count
+  # DNS zone where record should be created
+  domain  = var.dns_zone
+
+  # DNS record (will be prepended to domain)
+  name    = "microk8s-worker-worker-node-${count.index}"
+  type    = "A"
+  ttl     = 300
+
+  value = element(digitalocean_droplet.microk8s-worker-node.*.ipv4_address, count.index)
+}
diff --git a/network.tf b/network.tf
@@ -2,7 +2,7 @@
 resource "digitalocean_firewall" "rules" {
   name = "microk8s-cluster-firewall-${var.cluster_name}"
 
-  tags = [digitalocean_tag.microk8s-node.id]
+  tags = [digitalocean_tag.microk8s-node.id, digitalocean_tag.microk8s-worker.id]
 
 
   # allow ssh from anywhere
@@ -16,14 +16,14 @@ resource "digitalocean_firewall" "rules" {
   inbound_rule {
     protocol    = "tcp"
     port_range  = "1-65535"
-    source_tags = [digitalocean_tag.microk8s-node.id]
+    source_tags = [digitalocean_tag.microk8s-node.id, digitalocean_tag.microk8s-worker.id]
   }
 
    # allow nodes to contact each other
   inbound_rule {
     protocol    = "udp"
     port_range  = "1-65535"
-    source_tags = [digitalocean_tag.microk8s-node.id]
+    source_tags = [digitalocean_tag.microk8s-node.id, digitalocean_tag.microk8s-worker.id]
   }
 
   # allow HTTP/HTTPS ingress from load balancer

diff --git a/templates/join-worker.sh b/templates/join-worker.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+until microk8s.status --wait-ready; 
+  do sleep 3; echo "waiting for worker status.."; 
+done
+
+
+if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then
+  echo "adding microk8s-cluster.${dns_zone} dns to CSR."
+  sed -i 's@#MOREIPS@DNS.99 = microk8s-cluster.${dns_zone}\n#MOREIPS\n@g' /var/snap/microk8s/current/certs/csr.conf.template
+  echo "done."
+  sleep 10            
+  microk8s join ${main_node_ip}:25000/${cluster_token} --worker
+else
+  echo "Join process already done. Nothing to do"
+fi