diff --git a/deploy/scripts/aws_env.py b/deploy/scripts/aws_env.py new file mode 100755 index 0000000000..054d9e0ad9 --- /dev/null +++ b/deploy/scripts/aws_env.py @@ -0,0 +1,92 @@ +#! /usr/bin/env python3 +"""Set AWS Environment variables from aws cli profiles.""" + +from __future__ import annotations + +import os +import re +from typing import Dict, List, Optional + +from utils import choose_from_list, run_cmd + + +def aws_version() -> Optional[int]: + """Test if the aws cli version 2 is installed.""" + try: + result = run_cmd(["aws", "--version"], check_results=False, chomp=True) + except FileNotFoundError: + print("AWS CLI version 2 is not installed.") + return None + else: + if result.returncode == 0: + # get major version number from stdout + match = re.match(r"aws-cli/(\d+)\..*", result.stdout) + if match: + return int(match.group(1)) + return None + + +def list_aws_profiles() -> List[str]: + aws_ver = aws_version() + if aws_ver is not None and aws_ver == 2: + result = run_cmd(["aws", "configure", "list-profiles"], chomp=True) + return result.stdout.split("\n") + return [] + + +def get_profile_var(profile: str, var_name: str) -> str: + result = run_cmd(["aws", "configure", "--profile", profile, "get", var_name], chomp=True) + return result.stdout + + +def init_aws_environment() -> None: + profile_list = list_aws_profiles() + # Build a map for looking up a profile name from the access key id. This + # algorithm assumes: + # - the 'default' profile will be processed first + # - if there are profiles using the same access key id, only the last + # one will be put into the map + if len(profile_list) == 0: + return + profile_map: Dict[str, str] = {} + for profile in profile_list: + key_id = get_profile_var(profile, "aws_access_key_id") + profile_map[key_id] = profile + curr_access_key = os.getenv("AWS_ACCESS_KEY_ID", "") + if curr_access_key in profile_map: + curr_profile = profile_map[curr_access_key] + else: + curr_profile = None + aws_profile = choose_from_list("AWS Environment", curr_profile, profile_list) + if aws_profile is not None and aws_profile != curr_profile: + os.environ["AWS_PROFILE"] = aws_profile + os.environ["AWS_ACCESS_KEY_ID"] = get_profile_var(aws_profile, "aws_access_key_id") + os.environ["AWS_SECRET_ACCESS_KEY"] = get_profile_var(aws_profile, "aws_secret_access_key") + os.environ["AWS_DEFAULT_REGION"] = get_profile_var(aws_profile, "region") + result = run_cmd( + [ + "aws", + "sts", + "--profile", + aws_profile, + "get-caller-identity", + "--query", + "Account", + "--output", + "text", + ], + chomp=True, + ) + os.environ["AWS_ACCOUNT"] = result.stdout + + +if __name__ == "__main__": + init_aws_environment() + print("AWS Environment:") + for env_var in [ + "AWS_ACCOUNT", + "AWS_DEFAULT_REGION", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + ]: + print(f"{env_var}: {os.getenv(env_var, None)}") diff --git a/deploy/scripts/kube_env.py b/deploy/scripts/kube_env.py new file mode 100755 index 0000000000..e1c79985a6 --- /dev/null +++ b/deploy/scripts/kube_env.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Manage the Kubernetes environment for kubectl & helm.""" + +from __future__ import annotations + +import argparse +from typing import List, Optional + +from utils import choose_from_list, run_cmd + + +class KubernetesEnvironment: + def __init__(self, args: argparse.Namespace) -> None: + if "kubeconfig" in args and args.kubeconfig is not None: + self.kubeconfig = args.kubeconfig + else: + self.kubeconfig = None + if "context" in args and args.context is not None: + # if the user specified a context, use that one. + self.kubecontext = args.context + else: + context_list: List[str] = [] + + result = run_cmd( + ["kubectl"] + self.get_kubeconfig() + ["config", "get-contexts", "--no-headers"], + check_results=True, + ) + curr_context: Optional[str] = None + for line in result.stdout.splitlines(): + if line[0] == "*": + curr_context = line.split()[1] + context_list.append(curr_context) + else: + context_list.append(line.split()[0]) + + # If there is more than one context available, prompt the user to make sure + # that the intended context will be used. + curr_context = choose_from_list("context", curr_context, context_list) + if curr_context: + self.kubecontext = curr_context + else: + self.kubecontext = None + if "debug" in args: + self.debug = args.debug + else: + self.debug = False + + def get_kubeconfig(self) -> List[str]: + if self.kubeconfig is not None: + return ["--kubeconfig", self.kubeconfig] + return [] + + def get_helm_opts(self) -> List[str]: + """ + Create list of general helm options. + """ + helm_opts = self.get_kubeconfig() + + if self.kubecontext is not None: + helm_opts.extend(["--kube-context", self.kubecontext]) + if self.debug: + helm_opts.append("--debug") + return helm_opts + + def get_kubectl_opts(self) -> List[str]: + """ + Create list of general kubectl options. + """ + kubectl_opts = self.get_kubeconfig() + + if self.kubecontext is not None: + kubectl_opts.extend(["--context", self.kubecontext]) + return kubectl_opts + + +def add_kube_opts(parser: argparse.ArgumentParser) -> None: + """Add commandline arguments for Kubernetes tools.""" + parser.add_argument( + "--context", + help="Context in kubectl configuration file to be used.", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debugging output for helm commands.", + ) + parser.add_argument( + "--kubeconfig", + help="Specify the kubectl configuration file to be used.", + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate Helm Charts for The Combine.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + add_kube_opts(parser) + args = parser.parse_args() + + kube_env = KubernetesEnvironment(args) + print(f"kubectl {kube_env.get_kubectl_opts()} ...") + print(f"helm {kube_env.get_helm_opts()} ...") diff --git a/deploy/scripts/setup_cluster.py b/deploy/scripts/setup_cluster.py index ddfe14055e..38817d2541 100755 --- a/deploy/scripts/setup_cluster.py +++ b/deploy/scripts/setup_cluster.py @@ -11,7 +11,8 @@ from typing import Any, Dict, List from enum_types import ExitStatus, HelmAction -from utils import add_helm_opts, add_namespace, get_helm_opts, run_cmd +from kube_env import KubernetesEnvironment, add_kube_opts +from utils import add_namespace, run_cmd import yaml scripts_dir = Path(__file__).resolve().parent @@ -24,7 +25,7 @@ def parse_args() -> argparse.Namespace: description="Build containerd container images for project.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - add_helm_opts(parser) + add_kube_opts(parser) parser.add_argument( "--type", "-t", @@ -90,13 +91,15 @@ def main() -> None: for chart in yaml.safe_load(chart_list_results.stdout): curr_charts.append(chart["name"]) + # Verify the Kubernetes/Helm environment + kube_env = KubernetesEnvironment(args) # Install the required charts for chart_descr in this_cluster: chart_spec = config[chart_descr]["chart"] # add namespace if needed - add_namespace(chart_spec["namespace"]) + add_namespace(chart_spec["namespace"], kube_env.get_kubectl_opts()) # install the chart - helm_cmd = ["helm"] + get_helm_opts(args) + helm_cmd = ["helm"] + kube_env.get_helm_opts() if chart_spec["name"] in curr_charts: helm_action = HelmAction.UPGRADE else: diff --git a/deploy/scripts/setup_combine.py b/deploy/scripts/setup_combine.py index ccf52b08b2..6d3460a9c1 100755 --- a/deploy/scripts/setup_combine.py +++ b/deploy/scripts/setup_combine.py @@ -26,9 +26,11 @@ from typing import Any, Dict, List from app_release import get_release +from aws_env import init_aws_environment import combine_charts from enum_types import ExitStatus, HelmAction -from utils import add_helm_opts, add_namespace, get_helm_opts, run_cmd +from kube_env import KubernetesEnvironment, add_kube_opts +from utils import add_namespace, run_cmd import yaml scripts_dir = Path(__file__).resolve().parent @@ -41,7 +43,7 @@ def parse_args() -> argparse.Namespace: description="Generate Helm Charts for The Combine.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - add_helm_opts(parser) + add_kube_opts(parser) parser.add_argument( "--clean", action="store_true", help="Delete chart, if it exists, before installing." ) @@ -213,9 +215,13 @@ def main() -> None: else: profile = args.profile - # Create a base helm command for commands used to alter + # Verify the Kubernetes/Helm environment + kube_env = KubernetesEnvironment(args) + # Cache options for helm commands used to alter # the target cluster - helm_opts = get_helm_opts(args) + helm_opts = kube_env.get_helm_opts() + # Check AWS Environment Variables + init_aws_environment() # create list of target specific variable values target_vars = [ @@ -242,7 +248,7 @@ def main() -> None: for chart in config["profiles"][profile]["charts"]: # create the chart namespace if it does not exist chart_namespace = config["charts"][chart]["namespace"] - if add_namespace(chart_namespace): + if add_namespace(chart_namespace, kube_env.get_kubectl_opts()): installed_charts: List[str] = [] else: # get list of charts in target namespace diff --git a/deploy/scripts/setup_files/combine_config.yaml b/deploy/scripts/setup_files/combine_config.yaml index 585a7b3719..77cf31ad58 100644 --- a/deploy/scripts/setup_files/combine_config.yaml +++ b/deploy/scripts/setup_files/combine_config.yaml @@ -17,7 +17,7 @@ targets: # override values for 'thecombine' chart thecombine: global: - serverName: thecombine.local + serverName: thecombine.localhost nuc1: profile: nuc env_vars_required: true diff --git a/deploy/scripts/utils.py b/deploy/scripts/utils.py index 464e9689e4..e316b3f13f 100644 --- a/deploy/scripts/utils.py +++ b/deploy/scripts/utils.py @@ -4,7 +4,6 @@ from __future__ import annotations -import argparse import subprocess import sys from typing import List, Optional @@ -44,49 +43,61 @@ def run_cmd( sys.exit(err.returncode) -def add_namespace(namespace: str) -> bool: +def add_namespace(namespace: str, kube_opts: List[str]) -> bool: """ Create a Kubernetes namespace if and only if it does not exist. Returns True if the namespace was added. """ - lookup_results = run_cmd(["kubectl", "get", "namespace", namespace], check_results=False) + lookup_results = run_cmd( + ["kubectl"] + kube_opts + ["get", "namespace", namespace], check_results=False + ) if lookup_results.returncode != 0: - run_cmd(["kubectl", "create", "namespace", namespace]) + run_cmd(["kubectl"] + kube_opts + ["create", "namespace", namespace]) return True return False -def add_helm_opts(parser: argparse.ArgumentParser) -> None: - """ - Add commandline arguments that are shared between scripts calling helm. - - Sets up '--verbose' as the equivalent of '--debug'. +def choose_from_list( + name: str, curr_selection: Optional[str], options: List[str] +) -> Optional[str]: """ - parser.add_argument( - "--context", - help="Context in kubectl configuration file to be used.", - ) - parser.add_argument( - "--debug", - action="store_true", - help="Enable debugging output for helm commands.", - ) - parser.add_argument( - "--kubeconfig", - help="Specify the kubectl configuration file to be used.", - ) + Prompt user to choose/confirm a selection from a list. - -def get_helm_opts(args: argparse.Namespace) -> List[str]: - """ - Create list of general helm options based on argparse Namespace. + The curr_selection is automatically chosen if the options List is empty + or has the curr_selection as its only member. """ - helm_opts = [] - if args.kubeconfig: - helm_opts.extend(["--kubeconfig", args.kubeconfig]) - if args.context: - helm_opts.extend(["--kube-context", args.context]) - if args.debug: - helm_opts.append("--debug") - return helm_opts + if len(options) == 1 and curr_selection is not None and curr_selection == options[0]: + return curr_selection + if len(options) >= 1: + while True: + print(f"Choose {name} from:") + for index, option in enumerate(options): + print(f"\t{index+1}: {option}") + if curr_selection is None: + prompt_str = f"Enter {name}: " + else: + prompt_str = f"Enter {name} (Default: {curr_selection}): " + try: + reply = input(prompt_str) + except KeyboardInterrupt: + print("\nCancelled.") + sys.exit(1) + else: + if not reply: + break + elif reply in options: + curr_selection = reply + break + else: + try: + index = int(reply) + if index > 0 and index <= len(options): + curr_selection = options[index - 1] + break + else: + curr_selection = None + except ValueError: + curr_selection = None + print(f"{reply} is not in the list. Please re-enter.") + return curr_selection diff --git a/dev-requirements.txt b/dev-requirements.txt index 699e2290af..45891ffe45 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,11 +10,11 @@ attrs==22.1.0 # flake8-eradicate beautifulsoup4==4.11.1 # via mkdocs-htmlproofer-plugin -black==22.8.0 +black==22.10.0 # via -r dev-requirements.in cachetools==5.2.0 # via google-auth -certifi==2022.9.14 +certifi==2022.9.24 # via # kubernetes # requests @@ -46,9 +46,9 @@ flake8==5.0.4 # flake8-comprehensions # flake8-eradicate # pep8-naming -flake8-broken-line==0.5.0 +flake8-broken-line==0.6.0 # via -r dev-requirements.in -flake8-bugbear==22.9.11 +flake8-bugbear==22.9.23 # via -r dev-requirements.in flake8-comprehensions==3.10.0 # via -r dev-requirements.in @@ -56,14 +56,12 @@ flake8-eradicate==1.4.0 # via -r dev-requirements.in ghp-import==2.1.0 # via mkdocs -google-auth==2.11.1 +google-auth==2.12.0 # via kubernetes humanfriendly==10.0 # via -r dev-requirements.in idna==3.4 # via requests -importlib-metadata==4.12.0 - # via mkdocs isort==5.10.1 # via -r dev-requirements.in jinja2==3.1.2 @@ -90,17 +88,17 @@ mccabe==0.7.0 # via flake8 mergedeep==1.3.4 # via mkdocs -mkdocs==1.3.1 +mkdocs==1.4.0 # via # mkdocs-htmlproofer-plugin # mkdocs-material -mkdocs-htmlproofer-plugin==0.8.0 +mkdocs-htmlproofer-plugin==0.9.0 # via -r dev-requirements.in -mkdocs-material==8.5.3 +mkdocs-material==8.5.6 # via -r dev-requirements.in mkdocs-material-extensions==1.0.3 # via mkdocs-material -mypy==0.971 +mypy==0.982 # via -r dev-requirements.in mypy-extensions==0.4.3 # via @@ -138,11 +136,11 @@ pyflakes==2.5.0 # via flake8 pygments==2.13.0 # via mkdocs-material -pymdown-extensions==9.5 +pymdown-extensions==9.6 # via mkdocs-material pymongo==4.2.0 # via -r dev-requirements.in -pyopenssl==22.0.0 +pyopenssl==22.1.0 # via -r dev-requirements.in pyparsing==3.0.9 # via packaging @@ -187,17 +185,17 @@ tox==3.26.0 # via -r dev-requirements.in types-cryptography==3.3.23 # via types-pyopenssl -types-pyopenssl==22.0.10 +types-pyopenssl==22.1.0.0 # via -r dev-requirements.in types-python-dateutil==2.8.19 # via -r dev-requirements.in -types-pyyaml==6.0.11 +types-pyyaml==6.0.12 # via -r dev-requirements.in -types-requests==2.28.11 +types-requests==2.28.11.2 # via -r dev-requirements.in -types-urllib3==1.26.24 +types-urllib3==1.26.25 # via types-requests -typing-extensions==4.3.0 +typing-extensions==4.4.0 # via mypy urllib3==1.26.12 # via @@ -209,8 +207,6 @@ watchdog==2.1.9 # via mkdocs websocket-client==1.4.1 # via kubernetes -zipp==3.8.1 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/maintenance/scripts/combine_backup.py b/maintenance/scripts/combine_backup.py index 609208e82d..9841e2715b 100755 --- a/maintenance/scripts/combine_backup.py +++ b/maintenance/scripts/combine_backup.py @@ -12,7 +12,7 @@ from aws_backup import AwsBackup from combine_app import CombineApp -from maint_utils import wait_for_dependents +from maint_utils import check_env_vars, wait_for_dependents from script_step import ScriptStep @@ -36,7 +36,10 @@ def main() -> None: else: logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.WARNING) combine = CombineApp() - aws = AwsBackup(bucket=os.environ["aws_bucket"]) + aws_bucket, combine_host, db_files_subdir, backend_files_subdir = check_env_vars( + ["aws_bucket", "combine_host", "db_files_subdir", "backend_files_subdir"] + ) + aws = AwsBackup(bucket=aws_bucket) step = ScriptStep() step.print("Make sure backend and database are available") @@ -52,7 +55,7 @@ def main() -> None: with tempfile.TemporaryDirectory() as backup_dir: backup_file = Path("combine-backup.tar.gz") date_str = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - aws_file = f"{os.environ['combine_host']}-{date_str}.tar.gz" + aws_file = f"{combine_host}-{date_str}.tar.gz" step.print("Dump the database.") db_pod = combine.get_pod_id(CombineApp.Component.Database) @@ -72,19 +75,18 @@ def main() -> None: db_pod, [ "ls", - os.environ["db_files_subdir"], + db_files_subdir, ], check_results=False, ) if check_backup_results.returncode != 0: print("No database backup file - most likely empty database.", file=sys.stderr) sys.exit(0) - db_subdir = os.environ["db_files_subdir"] combine.kubectl( [ "cp", - f"{db_pod}:/{db_subdir}", - str(Path(backup_dir) / db_subdir), + f"{db_pod}:/{db_files_subdir}", + str(Path(backup_dir) / db_files_subdir), ] ) @@ -93,12 +95,11 @@ def main() -> None: if not backend_pod: print("Cannot find the backend container.", file=sys.stderr) sys.exit(1) - backend_subdir = os.environ["backend_files_subdir"] combine.kubectl( [ "cp", - f"{backend_pod}:/home/app/{backend_subdir}/", - str(Path(backup_dir) / backend_subdir), + f"{backend_pod}:/home/app/{backend_files_subdir}/", + str(Path(backup_dir) / backend_files_subdir), ] ) @@ -107,7 +108,7 @@ def main() -> None: os.chdir(backup_dir) with tarfile.open(backup_file, "x:gz") as tar: - for name in (os.environ["backend_files_subdir"], os.environ["db_files_subdir"]): + for name in (backend_files_subdir, db_files_subdir): tar.add(name) step.print("Push backup to AWS S3 storage.") diff --git a/maintenance/scripts/combine_restore.py b/maintenance/scripts/combine_restore.py index 3f17bb6ed4..17a1983f0f 100755 --- a/maintenance/scripts/combine_restore.py +++ b/maintenance/scripts/combine_restore.py @@ -31,6 +31,7 @@ from aws_backup import AwsBackup from combine_app import CombineApp import humanfriendly +from maint_utils import check_env_vars from script_step import ScriptStep @@ -65,8 +66,12 @@ def main() -> None: logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) else: logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.WARNING) + # Look up the required environment variables + aws_bucket, db_files_subdir, backend_files_subdir = check_env_vars( + ["aws_bucket", "db_files_subdir", "backend_files_subdir"] + ) combine = CombineApp() - aws = AwsBackup(bucket=os.environ["aws_bucket"]) + aws = AwsBackup(bucket=aws_bucket) step = ScriptStep() step.print("Prepare for the restore.") @@ -80,7 +85,7 @@ def main() -> None: backup_list_output = aws.list().stdout.strip().split("\n") if len(backup_list_output) == 0: - print(f"No backups available from {os.environ['aws_bucket']}") + print(f"No backups available from {aws_bucket}") sys.exit(0) # Convert the list of backups to a more useful structure @@ -125,7 +130,6 @@ def main() -> None: if not db_pod: print("Cannot find the database container.", file=sys.stderr) sys.exit(1) - db_files_subdir = os.environ["db_files_subdir"] combine.kubectl( [ "cp", @@ -167,13 +171,11 @@ def main() -> None: [ "/bin/bash", "-c", - f"rm -rf /home/app/{os.environ['backend_files_subdir']}/*", + f"rm -rf /home/app/{backend_files_subdir}/*", ], ) - combine.kubectl( - ["cp", os.environ["backend_files_subdir"], f"{backend_pod}:/home/app", "--no-preserve"] - ) + combine.kubectl(["cp", backend_files_subdir, f"{backend_pod}:/home/app", "--no-preserve"]) if __name__ == "__main__": diff --git a/maintenance/scripts/db_update_sem_dom_in_senses.py b/maintenance/scripts/db_update_sem_dom_in_senses.py index 0a30225bd7..076e4ced70 100755 --- a/maintenance/scripts/db_update_sem_dom_in_senses.py +++ b/maintenance/scripts/db_update_sem_dom_in_senses.py @@ -64,7 +64,7 @@ def get_guid(id: str, name: str) -> UUID: if id in domain_info: if name in domain_info[id].names: return domain_info[id].guid - logging.warning(f"Using blank GUID for {id}") + logging.warning(f"Using blank GUID for {id} {name}") return blank_guid @@ -88,7 +88,7 @@ def main() -> None: logging_level = logging.INFO if args.verbose else logging.WARNING logging.basicConfig(format="%(levelname)s:%(message)s", level=logging_level) - client: MongoClient[Dict[str, Any]] = MongoClient(f"mongodb://{args.host}/") + client: MongoClient[Dict[str, Any]] = MongoClient(args.host, args.port) db = client.CombineDatabase codec_opts: CodecOptions[Dict[str, Any]] = CodecOptions( uuid_representation=UuidRepresentation.PYTHON_LEGACY @@ -119,7 +119,7 @@ def main() -> None: if found_updates: updates[ObjectId(word["_id"])] = word # apply the updates - logging.info(f"Updating {len(updates)}/{num_docs} documents in {collection_name}.") + logging.info(f"Updating {len(updates)}/{total_docs} documents in {collection_name}.") for obj_id, update in updates.items(): curr_collection.update_one({"_id": obj_id}, {"$set": update}) diff --git a/maintenance/scripts/maint_utils.py b/maintenance/scripts/maint_utils.py index c1e8834385..83d7313ae6 100644 --- a/maintenance/scripts/maint_utils.py +++ b/maintenance/scripts/maint_utils.py @@ -2,11 +2,30 @@ from __future__ import annotations +import logging +import os import subprocess import sys from typing import List +def check_env_vars(var_names: List[str]) -> tuple[str, ...]: + """Look up a list of environment variables and validate them.""" + value_list: List[str] = [] + missing_values: List[str] = [] + for var in var_names: + value = os.getenv(var) + if value is None or not value: + missing_values.append(var) + value_list.append("") + else: + value_list.append(value) + if len(missing_values) > 0: + logging.critical(f"Missing or empty environment variables: {missing_values}") + sys.exit(1) + return tuple(value_list) + + def run_cmd(cmd: List[str], *, check_results: bool = True) -> subprocess.CompletedProcess[str]: """Run a command with subprocess and catch any CalledProcessErrors.""" try: