From 35cbcdb8390faddc64ce151d722f59fbcfd1d2ff Mon Sep 17 00:00:00 2001
From: Jim Grady <jimgrady.jg@gmail.com>
Date: Mon, 23 Nov 2020 21:53:33 -0500
Subject: [PATCH 1/2] Create backup/restore scripts for docker containers
 (#826)

* Update package versions
* Implement backup script
* Implement restore script
---
 deploy/roles/aws_access/defaults/main.yml     |   2 +-
 .../templates/combine-backup.j2               |   2 +-
 docker_deploy/group_vars/qa/main.yml          |  10 +-
 docker_deploy/group_vars/server/main.yml      |  11 +-
 docker_deploy/playbook_target_setup.yml       |   4 +
 .../roles/combine_backup/defaults/main.yml    |   8 +
 .../roles/combine_backup/tasks/main.yml       |  39 ++++
 .../templates/combine-backup.j2               | 117 ++++++++++++
 .../templates/combine-restore.j2              | 174 ++++++++++++++++++
 .../roles/combine_config/tasks/main.yml       |   2 +-
 .../templates/docker-compose.yml.j2           |   2 +-
 .../roles/combine_user/tasks/main.yml         |   9 -
 docker_deploy/vars/config_common.yml          |   2 +
 docker_deploy/vars/packages.yml               |   5 +-
 14 files changed, 367 insertions(+), 20 deletions(-)
 create mode 100644 docker_deploy/roles/combine_backup/defaults/main.yml
 create mode 100644 docker_deploy/roles/combine_backup/tasks/main.yml
 create mode 100644 docker_deploy/roles/combine_backup/templates/combine-backup.j2
 create mode 100644 docker_deploy/roles/combine_backup/templates/combine-restore.j2

diff --git a/deploy/roles/aws_access/defaults/main.yml b/deploy/roles/aws_access/defaults/main.yml
index 306ae0c3b6..361b117720 100644
--- a/deploy/roles/aws_access/defaults/main.yml
+++ b/deploy/roles/aws_access/defaults/main.yml
@@ -1,4 +1,4 @@
 ---
 aws_config_dir: "/home/{{ combine_user }}/.aws"
-aws_upload_profile: upload
+aws_backup_profile: upload
 aws_download_profile: download
diff --git a/deploy/roles/combine_backup/templates/combine-backup.j2 b/deploy/roles/combine_backup/templates/combine-backup.j2
index b5a3acd8b3..ea78a40daa 100755
--- a/deploy/roles/combine_backup/templates/combine-backup.j2
+++ b/deploy/roles/combine_backup/templates/combine-backup.j2
@@ -38,5 +38,5 @@ done
 # need to specify full path because $PATH does not contain
 # /usr/local/bin when run as a cron job
 if [ -e /usr/local/bin/aws ] ; then
-  /usr/local/bin/aws s3 cp ${BACKUP_FILE} ${AWS_FILE} --profile {{ aws_upload_profile }}
+  /usr/local/bin/aws s3 cp ${BACKUP_FILE} ${AWS_FILE} --profile {{ aws_backup_profile }}
 fi
diff --git a/docker_deploy/group_vars/qa/main.yml b/docker_deploy/group_vars/qa/main.yml
index 6aea62eac4..d857f8daff 100644
--- a/docker_deploy/group_vars/qa/main.yml
+++ b/docker_deploy/group_vars/qa/main.yml
@@ -22,7 +22,9 @@ cert_email: ""
 aws_user: "{{ combine_user }}"
 aws_group: "{{ combine_group }}"
 
-# my_default_aws_profile: ecr_read_only
-# my_aws_profiles:
-#   - ecr_read_write
-#   - s3_read_write
+aws_backup_profile: s3_read_write
+aws_ecr_profile: ecr_read_write
+
+my_aws_profiles:
+    - "{{ aws_backup_profile }}"
+    - "{{ aws_ecr_profile }}"
diff --git a/docker_deploy/group_vars/server/main.yml b/docker_deploy/group_vars/server/main.yml
index c09336af40..1632373135 100644
--- a/docker_deploy/group_vars/server/main.yml
+++ b/docker_deploy/group_vars/server/main.yml
@@ -28,6 +28,13 @@ aws_group: "{{ combine_group }}"
 #   - store backups
 # ECR:
 #   - install container images
+aws_backup_profile: s3_read_write
+aws_ecr_profile: ecr_read_only
+
 my_aws_profiles:
-    - s3_read_write
-    - ecr_read_only
+    - "{{ aws_backup_profile }}"
+    - "{{ aws_ecr_profile }}"
+
+# Define backup times (UTC)
+backup_hour:    "3"
+backup_minute:  "15"
diff --git a/docker_deploy/playbook_target_setup.yml b/docker_deploy/playbook_target_setup.yml
index 8efc05a7ba..97bd818f67 100644
--- a/docker_deploy/playbook_target_setup.yml
+++ b/docker_deploy/playbook_target_setup.yml
@@ -46,3 +46,7 @@
         name: aws_access
       tags:
         - aws
+
+    - name: setup container backups
+      import_role:
+        name: combine_backup
diff --git a/docker_deploy/roles/combine_backup/defaults/main.yml b/docker_deploy/roles/combine_backup/defaults/main.yml
new file mode 100644
index 0000000000..9aa3b8311a
--- /dev/null
+++ b/docker_deploy/roles/combine_backup/defaults/main.yml
@@ -0,0 +1,8 @@
+---
+# backup_hour:  7
+# backup_minute: 15
+
+aws_s3_backup_loc: thecombine.app/backups
+
+backend_files_subdir: ".CombineFiles"
+mongo_files_subdir: "dump"
diff --git a/docker_deploy/roles/combine_backup/tasks/main.yml b/docker_deploy/roles/combine_backup/tasks/main.yml
new file mode 100644
index 0000000000..80b5937655
--- /dev/null
+++ b/docker_deploy/roles/combine_backup/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+
+#######################################################
+#
+# Install and configure the backup script for the docker
+# containers
+
+- name: create folders for backups
+  file:
+    name: "{{ item }}"
+    owner: "{{ combine_user }}"
+    group: "{{ combine_group }}"
+    mode: 0755
+    state: directory
+  with_items:
+    - "{{ combine_backup_dir }}"
+    - "{{ combine_app_dir }}/bin"
+
+- name: install backup/restore scripts
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ combine_app_dir }}/bin/{{ item }}"
+    owner: "{{ combine_user }}"
+    group: "{{ combine_group }}"
+    mode: 0755
+  with_items:
+    - combine-backup
+    - combine-restore
+
+- name: schedule regular backups
+  cron:
+    name: combine daily backup
+    job: "{{ combine_app_dir }}/bin/combine-backup"
+    user: "{{ combine_user }}"
+    hour: "{{ backup_hour }}"
+    minute: "{{ backup_minute }}"
+  when:
+    - backup_hour is defined
+    - backup_minute is defined
diff --git a/docker_deploy/roles/combine_backup/templates/combine-backup.j2 b/docker_deploy/roles/combine_backup/templates/combine-backup.j2
new file mode 100644
index 0000000000..9643a0c921
--- /dev/null
+++ b/docker_deploy/roles/combine_backup/templates/combine-backup.j2
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+######################################################
+# Backup job for docker containers running TheCombine
+######################################################
+
+set -e
+
+VERBOSE="0"
+
+echo_verbose() {
+  if [ "${VERBOSE}" == "1" ] ; then
+    echo $@
+  fi
+}
+
+usage() {
+  cat <<USAGE
+  Usage: $0 [-h] [-v]
+     Creates a backup of the combine database and backend containers
+  Options:
+     -h: help - print this message
+     -v: verbose - print progress of backup
+USAGE
+}
+
+while [[ $# -gt 0 ]] ; do
+  arg="$1"
+
+  case ${arg} in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -v|--verbose)
+      VERBOSE="1"
+      ;;
+    -?)
+      echo "Invalid option: ${arg}."
+      usage
+      exit 2
+      ;;
+    *)
+      echo "Unrecognized argument: ${arg}"
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+echo_verbose "1. Setup some useful environment variables"
+DATE_STR=`date +%Y-%m-%d-%H-%M-%S`
+COMBINE_HOST="{{ combine_server_name | replace('.', '-') }}"
+BACKUP_FILE="combine-backup.tar.gz"
+AWS_FILE="s3://{{ aws_s3_backup_loc }}/${COMBINE_HOST}-${DATE_STR}.tar.gz"
+COMBINE_APP_DIR=${COMBINE_APP_DIR:="{{ combine_app_dir }}"}
+BACKUP_DIR=${BACKUP_DIR:="{{ combine_backup_dir }}"}
+BACKEND_FILES_SUBDIR="{{ backend_files_subdir }}"
+DB_FILES_SUBDIR="{{ mongo_files_subdir }}"
+
+cd ${COMBINE_APP_DIR}
+
+echo_verbose "2. Prepare the backup directory"
+if [ ! -e "${BACKUP_DIR}" ] ; then
+  mkdir -p ${BACKUP_DIR}
+else
+  for item in ${BACKUP_DIR}/${DB_FILES_SUBDIR} ${BACKUP_DIR}/${BACKEND_FILES_SUBDIR} ${BACKUP_DIR}/${BACKUP_FILE}
+  do
+    if [ -e "${item}" ] ; then
+      rm -rf ${item}
+    fi
+  done
+fi
+
+echo_verbose "3. Stop the current containers"
+docker-compose down
+
+echo_verbose "4. Start up just the backend and the database"
+aws ecr get-login-password --profile {{ aws_ecr_profile }} | docker login --username AWS --password-stdin {{ aws_ecr }}
+docker-compose up --detach database backend
+
+
+echo_verbose "5. Dump the database"
+docker-compose exec database mongodump --db CombineDatabase --gzip --quiet
+DB_CONTAINER=`docker ps | grep database | sed "s/.* \([^ ][^ ]*\)$/\1/"`
+docker cp ${DB_CONTAINER}:${DB_FILES_SUBDIR}/ ${BACKUP_DIR}
+
+echo_verbose "6. Copy the backend files (commands are run relative the 'app' user's home directory)"
+BE_CONTAINER=`docker ps | grep backend | sed "s/.* \([^ ][^ ]*\)$/\1/"`
+docker cp ${BE_CONTAINER}:/home/app/${BACKEND_FILES_SUBDIR}/ ${BACKUP_DIR}
+
+echo_verbose "7. create the tarball for the backup"
+cd ${BACKUP_DIR}
+tar --create --file=${BACKUP_FILE} --gzip --verbose ${BACKEND_FILES_SUBDIR} ${DB_FILES_SUBDIR}
+
+echo_verbose "8. Remove old backup files"
+# Running in ${BACKUP_DIR}
+rm -rf ${BACKEND_FILES_SUBDIR} ${DB_FILES_SUBDIR}
+ALL_BACKUPS=(`ls combine-backup*.tar.gz`)
+
+for bu in "${ALL_BACKUPS[@]}"; do
+  if [ "$bu" != "$BACKUP_FILE" ] ; then
+  	  echo "Removing $bu"
+      rm $bu
+  fi
+done
+
+echo_verbose "9. push backup to AWS S3 storage"
+#    need to specify full path because $PATH does not contain
+#    /usr/local/bin when run as a cron job
+/usr/local/bin/aws s3 cp ${BACKUP_FILE} ${AWS_FILE} --profile {{ aws_backup_profile }}
+
+echo_verbose "10. Restart the containers"
+cd ${COMBINE_APP_DIR}
+docker-compose down
+docker-compose up --detach
diff --git a/docker_deploy/roles/combine_backup/templates/combine-restore.j2 b/docker_deploy/roles/combine_backup/templates/combine-restore.j2
new file mode 100644
index 0000000000..e925f49628
--- /dev/null
+++ b/docker_deploy/roles/combine_backup/templates/combine-restore.j2
@@ -0,0 +1,174 @@
+#!/bin/bash
+
+#########################################################
+# Restore scrips for docker containers running TheCombine
+#########################################################
+
+set -e
+
+VERBOSE="0"
+BACKUP=""
+CLEAN="0"
+
+echo_verbose() {
+  if [ "${VERBOSE}" == "1" ] ; then
+    echo $@
+  fi
+}
+
+usage() {
+  cat <<USAGE
+Usage: $0 [-h] [-v] [backup_file]
+   Restores the combine database and backend containers from the specified
+   backup on AWS.  If the backup is not specified, the currently available
+   backups are listed.
+Options:
+   -c, --clean:
+        remove backend files in /home/app/.CombineFiles in the backend
+        container.  Normally, the files are restored on top of the existing
+        backend files.
+   -h, --help:
+        print this message
+   -v: verbose
+        print progress of backup
+USAGE
+}
+
+while [[ $# -gt 0 ]] ; do
+  arg="$1"
+
+  case ${arg} in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -v|--verbose)
+      VERBOSE="1"
+      ;;
+    -c|--clean)
+      CLEAN="1"
+      ;;
+    -?)
+      echo "Invalid option: ${arg}."
+      usage
+      exit 1
+      ;;
+    *)
+      BACKUP=`basename $arg`
+      ;;
+  esac
+  shift
+done
+
+echo_verbose "1. Prepare for the restore"
+
+COMBINE_HOST={{ ansible_hostname }}
+RESTORE_FILE="combine-backup.tar.gz"
+AWS_BACKUPS="s3://{{ aws_s3_backup_loc }}"
+COMBINE_APP_DIR=${COMBINE_APP_DIR:="{{ combine_app_dir }}"}
+RESTORE_DIR=${RESTORE_DIR:="{{ combine_restore_dir }}"}
+BACKEND_FILES_SUBDIR="{{ backend_files_subdir }}"
+DB_FILES_SUBDIR="{{ mongo_files_subdir }}"
+
+cd ${COMBINE_APP_DIR}
+
+echo_verbose "2. Prepare the restore directory"
+if [ ! -e "${RESTORE_DIR}" ] ; then
+  mkdir -p ${RESTORE_DIR}
+else
+  for item in ${RESTORE_DIR}/${DB_FILES_SUBDIR} ${RESTORE_DIR}/${BACKEND_FILES_SUBDIR} ${RESTORE_DIR}/${RESTORE_FILE}
+  do
+    if [ -e "${item}" ] ; then
+      rm -rf ${item}
+    fi
+  done
+fi
+
+if [ ! -x /usr/local/bin/aws ] ; then
+  echo "aws-cli v2 is not installed."
+  exit 2
+else
+  AWS_VER=`/usr/local/bin/aws --version`
+  aws_ver_pattern='aws-cli/([0-9][0-9]*).*'
+  if [[ $AWS_VER =~ $aws_ver_pattern ]] ; then
+    AWS_MAJ_VERSION=${BASH_REMATCH[1]}
+    if [ "${AWS_MAJ_VERSION}" != "2" ] ; then
+      echo "aws-cli is installed but is not version 2:"
+      echo "${AWS_VER}"
+      exit 3
+    fi
+  else
+    echo "Cannot determine the AWS version. :-("
+    exit 4
+  fi
+fi
+
+if [ -z "${BACKUP}" ] ; then
+  aws_backup_list=( $(/usr/local/bin/aws s3 ls ${AWS_BACKUPS} --recursive --profile {{ aws_backup_profile }} | sed "s/.* backups\///") )
+  aws_backups_available=${{ '{#' }}aws_backup_list[@]}
+  if [[ $aws_backups_available -eq 0 ]] ; then
+    echo "No backups available from ${AWS_BACKUPS}"
+    exit 0
+  fi
+  echo "Backup List:"
+  for key in "${!aws_backup_list[@]}" ; do
+    index=$((key+1))
+    echo -e "\t$index: ${aws_backup_list[$key]}"
+  done
+  read -p "Enter the number of the backup you would like to restore (0 = None):" backup_num
+  if [ -z "$backup_num" ] || [ "$backup_num" == "0" ] ; then
+    echo "No backup selected.  Exiting."
+    exit 0
+  fi
+  num_re='^[0-9]+$'
+  if ! [[ $backup_num =~ $num_re ]] || [[ $backup_num -gt ${aws_backups_available} ]]; then
+   echo "Invalid selection"
+   exit 5
+  fi
+  key=$((backup_num-1))
+  BACKUP=${aws_backup_list[$key]}
+fi
+echo "BACKUP == '${BACKUP}'"
+
+echo_verbose "3. Fetch the selected backup, ${BACKUP}"
+AWS_FILE="${AWS_BACKUPS}/${BACKUP}"
+/usr/local/bin/aws s3 cp ${AWS_FILE} ${RESTORE_DIR}/${RESTORE_FILE} --profile {{ aws_backup_profile }}
+
+echo_verbose "4. Unpack the backup"
+tar xzvf ${RESTORE_DIR}/${RESTORE_FILE} -C ${RESTORE_DIR}
+
+echo verbose "5. Stop the current containers"
+docker-compose down
+
+echo_verbose "6. Start up just the backend and the database"
+aws ecr get-login-password --profile {{ aws_ecr_profile }} | docker login --username AWS --password-stdin {{ aws_ecr }}
+docker-compose up --detach database backend
+
+
+echo_verbose "7. Restore the database"
+DB_CONTAINER=`docker ps | grep database | sed "s/.* \([^ ][^ ]*\)$/\1/"`
+docker cp ${RESTORE_DIR}/${DB_FILES_SUBDIR}/ ${DB_CONTAINER}:${DB_FILES_SUBDIR}
+docker-compose exec database mongorestore --drop --gzip --quiet
+docker-compose exec database rm -rf ${DB_FILES_SUBDIR}
+
+echo_verbose "8. Copy the backend files"
+# if CLEAN is set, delete the existing files
+if [ "$CLEAN" == "1" ] ; then
+  # we run the rm command inside a bash shell so that the shell will do wildcard
+  # expansion
+  docker-compose exec --user root --workdir /home/app/${BACKEND_FILES_SUBDIR} backend /bin/bash -c "rm -rf *"
+fi
+BE_CONTAINER=`docker ps | grep backend | sed "s/.* \([^ ][^ ]*\)$/\1/"`
+docker cp ${RESTORE_DIR}/${BACKEND_FILES_SUBDIR}/ ${BE_CONTAINER}:/home/app
+# change permissions for the copied files.  Since the tarball is created outside
+# of the container, the app user will not be the owner (the backend process is
+# running as "app").  In addition, it is possible that the backup is from a
+# different host with different UIDs.
+docker-compose exec --user root backend find /home/app/${BACKEND_FILES_SUBDIR} -exec chown app:app {} \;
+
+echo_verbose "9. Cleanup Restore files"
+rm -rf ${RESTORE_DIR}
+
+echo_verbose "10. Restart the containers"
+docker-compose down
+docker-compose up --detach
diff --git a/docker_deploy/roles/combine_config/tasks/main.yml b/docker_deploy/roles/combine_config/tasks/main.yml
index bea1c2230d..33ad8ad931 100644
--- a/docker_deploy/roles/combine_config/tasks/main.yml
+++ b/docker_deploy/roles/combine_config/tasks/main.yml
@@ -28,7 +28,7 @@
     group: "{{ combine_group }}"
     mode: 0644
 
-- name: create frontend environment file
+- name: create container environment files
   template:
     src: "{{ item }}.j2"
     dest: "{{ combine_app_dir }}/.{{ item }}"
diff --git a/docker_deploy/roles/combine_config/templates/docker-compose.yml.j2 b/docker_deploy/roles/combine_config/templates/docker-compose.yml.j2
index 87452d760a..ba71dbeefb 100644
--- a/docker_deploy/roles/combine_config/templates/docker-compose.yml.j2
+++ b/docker_deploy/roles/combine_config/templates/docker-compose.yml.j2
@@ -52,7 +52,7 @@ services:
     restart: unless-stopped
 
   database:
-    image: mongo:4.2
+    image: mongo:{{ mongodb_version }}
     volumes:
       - database_data:/data/db
     expose:
diff --git a/docker_deploy/roles/combine_user/tasks/main.yml b/docker_deploy/roles/combine_user/tasks/main.yml
index 7d1035fdaf..479e44db96 100644
--- a/docker_deploy/roles/combine_user/tasks/main.yml
+++ b/docker_deploy/roles/combine_user/tasks/main.yml
@@ -18,12 +18,3 @@
     shell: /bin/bash
     append: yes
     generate_ssh_key: yes
-
-- name: Add current user as authorized user for combine_user
-  authorized_key:
-    user: "{{ combine_user }}"
-    state: present
-    key: "{{ lookup('file', '{{ item }}') }}"
-  with_fileglob:
-    - "{{ lookup('env', 'HOME')}}/.ssh/*.pub"
-    
diff --git a/docker_deploy/vars/config_common.yml b/docker_deploy/vars/config_common.yml
index a3bab7835d..e93e191d06 100644
--- a/docker_deploy/vars/config_common.yml
+++ b/docker_deploy/vars/config_common.yml
@@ -10,6 +10,8 @@ frontend_env_dir: "{{ source_dir }}"
 combine_user: combine
 combine_group: app
 combine_app_dir: /opt/combine
+combine_backup_dir: "{{ combine_app_dir }}/backups"
+combine_restore_dir: "{{ combine_app_dir }}/restore"
 
 # common configuration items that are shared between the certmgr and frontend
 # containers and across target types:
diff --git a/docker_deploy/vars/packages.yml b/docker_deploy/vars/packages.yml
index 6a9970379a..20f93000c6 100644
--- a/docker_deploy/vars/packages.yml
+++ b/docker_deploy/vars/packages.yml
@@ -5,8 +5,11 @@
 # this file and rerun the specified playbook
 ###############################
 
+# MongoDB version
+mongodb_version: 4.4
+
 # Packages installed by playbook_docker.yml
-docker_compose_version: 1.26.2
+docker_compose_version: 1.27.4
 
 # Packages required for targets
 required_packages:

From 16f08d48b68410249620703332b0160452e5ca06 Mon Sep 17 00:00:00 2001
From: Jim Grady <jimgrady.jg@gmail.com>
Date: Tue, 24 Nov 2020 11:03:07 -0500
Subject: [PATCH 2/2] Bump version to 0.6.0-alpha.0 (#827)

Late update to version for development.
---
 package-lock.json | 2 +-
 package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 0503751217..e556b5e07c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "thecombine",
-  "version": "0.6.0",
+  "version": "0.6.0-alpha.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index 9e8eb9b7d6..764114c02d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "thecombine",
-  "version": "0.6.0",
+  "version": "0.6.0-alpha.0",
   "license": "MIT",
   "private": true,
   "scripts": {