Terrafrom project to deploy testgrid server

(cherry picked from commit dfc6f2e)

testgrid/deploy/.gitignore

@@ -0,0 +1,34 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+#
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

testgrid/deploy/README.md

@@ -0,0 +1,32 @@
+# Tesgrid server installation 
+
+## Terraform
+
+### Provisioning
+
+> NOTE: Project based token doesn't have sufficient privileges for Spot instance create/destroy operations (*creation works with errors; destroy fails*). Personal token can be used to get sufficient privileges:
+
+```bash
+export TF_VAR_auth_token=<token>
+terraform apply
+```
+
+### Deprovision
+```bash
+export TF_VAR_auth_token=<token>
+terraform destroy
+```
+
+### Tested versions
+```bash
+Terraform v0.12.29
++ provider.packet v3.0.1
++ provider.template v2.2.0
+```
+
+## TODO
+
+- Persist TF state (s3 or any other shared store)
+- Install binary or docker image for `tgrun` and configure the service accordantly
+- Multi node cluster support
+- CI/CD

testgrid/deploy/main.tf

@@ -0,0 +1,32 @@
+# Configure the Packet Provider. 
+provider "packet" {
+  auth_token = var.auth_token
+}
+
+data "template_file" "tg_setup" {
+  template = file("./tg-script.sh")
+}
+
+resource "packet_spot_market_request" "req" {
+  project_id    = var.project_id
+  max_bid_price = var.max_bid
+  facilities    = var.region
+  devices_min   = 1
+  devices_max   = 1
+
+  instance_parameters {
+    hostname         = var.tg_hostname
+    billing_cycle    = "hourly"
+    operating_system = var.tg_os
+    plan             = var.instance_type
+    userdata         = data.template_file.tg_setup.rendered
+  }
+}
+
+data "packet_spot_market_request" "dreq" {
+  request_id = packet_spot_market_request.req.id
+}
+
+output "ids" {
+  value = data.packet_spot_market_request.dreq.device_ids
+}

testgrid/deploy/tg-script.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+apt update && apt upgrade -y
+
+echo "Setting up RAID0 for openebs local storage."
+apt install -y btrbk
+mkfs.btrfs -d raid0 /dev/nvme0n1 /dev/nvme1n1
+mkdir -p /var/openebs/local
+mount /dev/nvme0n1 /var/openebs/local/
+
+echo "Installing kURL"
+INSTALL_SCRIPT=/root/kurl-install.sh
+curl https://kurl.sh/f3dd2d4 > $INSTALL_SCRIPT
+sed -i 's/parse_yaml_into_bash_variables$/parse_yaml_into_bash_variables\n    PRIVATE_ADDRESS=$(\/sbin\/ifconfig bond0:0 | awk "\/inet \/ {print \\$2}")/' $INSTALL_SCRIPT 
+chmod +x $INSTALL_SCRIPT
+./$INSTALL_SCRIPT
+cp /etc/kubernetes/admin.conf /root/.kube/config
+
+echo "Instaling KubeVirt"
+kubectl create namespace kubevirt
+kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/v0.32.0/kubevirt-operator.yaml
+kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/v0.32.0/kubevirt-cr.yaml
+kubectl wait --timeout=180s --for=condition=Available -n kubevirt kv/kubevirt
+kubectl create ns cdi
+kubectl apply -f https://github.com/kubevirt/containerized-data-importer/releases/download/v1.24.0/cdi-operator.yaml
+kubectl apply -f https://github.com/kubevirt/containerized-data-importer/releases/download/v1.24.0/cdi-cr.yaml
+
+echo "Setting up tgrun service"
+cat <<SERVICE_FILE >> /lib/systemd/system/tgrun.service
+[Unit]
+Description=tgrun
+[Service]
+Type=simple
+RestartSec=5s
+StandardOutput=syslog
+StandardError=syslog
+WorkingDirectory=/root
+SyslogIdentifier=tgrund
+Environment="KUBECONFIG=/etc/kubernetes/admin.conf"
+Environment="HOME=/root"
+Environment="PATH=/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ExecStart=/root/tgrun run
+[Install]
+WantedBy=multi-user.target
+SERVICE_FILE 

testgrid/deploy/vars.tf

@@ -0,0 +1,37 @@
+variable instance_type {
+  type        = string
+  default     = "m3.large.x86"
+  description = "packet instamce type"
+}
+
+variable auth_token {
+  type        = string
+}
+
+variable region {
+  type        = list(string)
+  default     = ["sv15"]
+  description = "Packet regions to deploy testgrid"
+}
+
+variable project_id {
+  type        = string
+  default     = "bf141b98-6b6d-49c8-b7df-c261e383fc74"
+  description = "Project name, using Testing as default."
+}
+
+variable max_bid {
+  type        = string
+  default     = "0.77"
+  description = "Maximum bid price for the instance"
+}
+
+variable tg_hostname {
+    type      = string
+    default   = "testgrid-spot"
+}
+
+variable tg_os {
+  type        = string
+  default     = "ubuntu_18_04"
+}