Docker image for deploy (#1683)

* Docker image to deploy The Combine
* Add --wait option to setup_combine.py
* Remove tractor image from the "Remote Combine Server" page

deploy/.dockerignore

@@ -0,0 +1,6 @@
+Dockerfile
+__pycache__
+bin
+obj
+build
+_*.yaml

deploy/Dockerfile

@@ -0,0 +1,45 @@
+# Dockerfile to build a Kubernetes Worker container for the Combine.  The
+# image shall contain a collection of scripts to perform the following functions:
+#  - backup The Combine database and backend data files
+#  - restore The Combine database and backend data files from a previous backup
+#  - monitor specified secrets for changes and push the updated secrets to AWS
+#    S3 storage
+#  - check the current TLS secret for updates in AWS S3 storage and update the
+#    secret accordingly.
+# The scripts are written in Python.
+
+FROM ubuntu:22.04
+
+USER root
+
+RUN apt-get update && \
+  apt-get install -y python3 python3-pip nano curl openssh-client iputils-ping && \
+  apt-get autoremove && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Install kubectl and helm
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+  install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \
+  curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && \
+  chmod 700 get_helm.sh && \
+  ./get_helm.sh && \
+  rm kubectl get_helm.sh
+
+ENV HOME /root
+ENV PATH ${PATH}:${HOME}/scripts
+
+COPY requirements.txt ${HOME}
+RUN pip3 install -r ${HOME}/requirements.txt
+
+RUN mkdir ${HOME}/.ssh
+
+# Copy Python scripts for setting up and deploying the target
+# to ~/scripts
+COPY scripts/ ${HOME}/scripts
+# Copy helm templates to ~/helm
+COPY helm/ ${HOME}/helm
+# Copy Ansible configuration files to ~/ansible
+COPY ansible/ ${HOME}/ansible
+
+ENTRYPOINT [ "/bin/bash" ]

deploy/ansible/roles/k8s_install/tasks/k3s.yml

@@ -101,3 +101,13 @@
     path: "{{ kubecfg }}"
     regexp: "^(.*)default(.*)$"
     replace: '\1{{ kubecfgdir }}\2'
+
+- name: Link ~/.kube/config to {{ kubecfg }}
+  delegate_to: localhost
+  become: no
+  file:
+    state: link
+    src: "{{ kubecfg }}"
+    dest: "{{ lookup('env', 'HOME') }}/.kube/config"
+    mode: 0600
+  when: link_kubeconfig | default(false)

deploy/helm/cert-proxy-server/files/pages/index.html

@@ -47,13 +47,6 @@ <h4>
           </td>
           <td />
         </tr>
-        <tr>
-          <td />
-          <td>
-            <img src="./tractor.png" alt="TheCombine" style="width: 70%" />
-          </td>
-          <td />
-        </tr>
       </table>
     </div>
     <script>

deploy/requirements.in

@@ -0,0 +1,10 @@
+ansible
+
+# Templating/Docker.
+Jinja2
+jinja2-base64-filters
+kubernetes
+pyopenssl
+
+# Kubernetes Installation.
+pyyaml

deploy/requirements.txt

@@ -0,0 +1,86 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile requirements.in
+#
+ansible==6.0.0
+    # via -r requirements.in
+ansible-core==2.13.1
+    # via ansible
+cachetools==5.2.0
+    # via google-auth
+certifi==2022.6.15
+    # via
+    #   kubernetes
+    #   requests
+cffi==1.15.0
+    # via cryptography
+charset-normalizer==2.0.12
+    # via requests
+cryptography==37.0.2
+    # via
+    #   ansible-core
+    #   pyopenssl
+google-auth==2.8.0
+    # via kubernetes
+idna==3.3
+    # via requests
+jinja2==3.1.2
+    # via
+    #   -r requirements.in
+    #   ansible-core
+    #   jinja2-base64-filters
+jinja2-base64-filters==0.1.4
+    # via -r requirements.in
+kubernetes==23.6.0
+    # via -r requirements.in
+markupsafe==2.1.1
+    # via jinja2
+oauthlib==3.2.0
+    # via requests-oauthlib
+packaging==21.3
+    # via ansible-core
+pyasn1==0.4.8
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.2.8
+    # via google-auth
+pycparser==2.21
+    # via cffi
+pyopenssl==22.0.0
+    # via -r requirements.in
+pyparsing==3.0.9
+    # via packaging
+python-dateutil==2.8.2
+    # via kubernetes
+pyyaml==6.0
+    # via
+    #   -r requirements.in
+    #   ansible-core
+    #   kubernetes
+requests==2.28.0
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+requests-oauthlib==1.3.1
+    # via kubernetes
+resolvelib==0.8.1
+    # via ansible-core
+rsa==4.8
+    # via google-auth
+six==1.16.0
+    # via
+    #   google-auth
+    #   kubernetes
+    #   python-dateutil
+urllib3==1.26.9
+    # via
+    #   kubernetes
+    #   requests
+websocket-client==1.3.3
+    # via kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

deploy/scripts/setup_combine.py

@@ -57,6 +57,11 @@ def parse_args() -> argparse.Namespace:
         help="Invoke the 'helm install' command with the '--dry-run' option.",
         dest="dry_run",
     )
+    parser.add_argument(
+        "--wait",
+        action="store_true",
+        help="Invoke the 'helm install' command with the '--wait' option.",
+    )
     parser.add_argument(
         "--profile",
         "-p",
@@ -280,7 +285,9 @@ def main() -> None:
 
             # set the dry-run option if desired
             if args.dry_run:
-                helm_install_cmd.extend(["--dry-run"])
+                helm_install_cmd.append("--dry-run")
+            if args.wait:
+                helm_install_cmd.append("--wait")
 
             # add the profile specific configuration
             add_profile_values(

deploy/scripts/setup_target.py

@@ -0,0 +1,80 @@
+#! /usr/bin/env python3
+
+import argparse
+import os
+from pathlib import Path
+import re
+import shutil
+import tempfile
+
+
+def parse_args() -> argparse.Namespace:
+    """Define command line arguments for parser."""
+    parser = argparse.ArgumentParser(
+        description="Setup access to the target device for Ansible.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("ip", help="IPv4 address for the target device.")
+    parser.add_argument("name", help="Name of the target device.")
+    parser.add_argument(
+        "--user", default="sillsdev", help="Username for ssh connection to the target device."
+    )
+    parser.add_argument("--hosts", default="/etc/hosts", help="File for host definition.")
+    return parser.parse_args()
+
+
+def update_hosts_file(tgt_ip: str, tgt_name: str, hosts_filename: Path) -> None:
+    match = re.search(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\..(\d{1,3})$", tgt_ip)
+    if match is not None:
+        ip_pattern = tgt_ip.replace(".", r"\.")
+    else:
+        raise ValueError(f"Invalid IPv4 address: {tgt_ip}")
+    # create ip address pattern from string
+    # Update /etc/hosts
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_hosts = Path(temp_dir).resolve() / "hosts"
+        output_file = open(temp_hosts, "w")
+        hosts_file = open(hosts_filename)
+        both_in_line = re.compile(f"^{ip_pattern}[ \t]+.*{tgt_name}")
+        ip_in_line = re.compile(f"^{ip_pattern}[ \t]+.*")
+        name_in_line = re.compile(f"[ \t]{tgt_name}")
+        entry_found = False
+        for line in hosts_file:
+            line = re.sub(r"[\r\n]+$", "", line)
+            # check to see if we've already found the entry
+            if entry_found:
+                output_line = line
+            # check to see if target is in this line
+            elif both_in_line.search(line):
+                entry_found = True
+                output_line = f"{line}"
+            elif ip_in_line.search(line):
+                output_line = f"{line} {tgt_name}"
+                entry_found = True
+            elif name_in_line.search(line):
+                # replace IP address for args.name an any other hosts with that IP address
+                output_line = re.sub(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\..(\d{1,3})", tgt_ip, line)
+                entry_found = True
+            else:
+                output_line = f"{line}"
+
+            output_file.write(f"{output_line}\n")
+        if not entry_found:
+            output_file.write(f"{tgt_ip}  {tgt_name}\n")
+        hosts_file.close()
+        output_file.close()
+        shutil.copy(temp_hosts, hosts_filename)
+
+
+def main() -> None:
+    args = parse_args()
+    # Add the target IP and target name to /etc/hosts (or other hosts file)
+    update_hosts_file(args.ip, args.name, Path(args.hosts).resolve())
+    # Generate ssh keys
+    os.system("ssh-keygen")
+    # Copy ssh id to target
+    os.system(f"ssh-copy-id {args.user}@{args.name}")
+
+
+if __name__ == "__main__":
+    main()

deploy/scripts/utils.py

@@ -5,7 +5,6 @@
 from __future__ import annotations
 
 import argparse
-import re
 import subprocess
 import sys
 from typing import List
@@ -33,7 +32,7 @@ def run_cmd(
         if print_output:
             print(process_results.stdout)
         if chomp:
-            process_results.stdout = re.sub(r"[\r\n]+$", "", process_results.stdout)
+            process_results.stdout = process_results.stdout.rstrip("\r\n\t ")
         return process_results
     except subprocess.CalledProcessError as err:
         print(f"CalledProcessError returned {err.returncode}")