diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc60cdd..fbd41985 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,22 @@ Here we write upgrading notes for RAD Lab repo. It's a team effort to make them ### Fixed +## [5.1.1](https://github.com/GoogleCloudPlatform/rad-lab/releases/tag/v5.0.0) (2022-03-18) + +[Full Changelog](https://github.com/GoogleCloudPlatform/rad-lab/compare/v5.0.0...v5.1.1) + +### Added +- _RAD Lab Module:_ Option to create container image Vertex AI notebooks in Data Science Module. [(#48)](https://github.com/GoogleCloudPlatform/rad-lab/pull/48) +- _RAD Lab Module:_ Capability for users to upload their own Jupyter notebooks to Vertex AI notebooks on Data Science Module deployment. [(#48)](https://github.com/GoogleCloudPlatform/rad-lab/pull/48) +- _RAD Lab Launcher:_ Enhancements to accept Command Line arguments. [(#41)](https://github.com/GoogleCloudPlatform/rad-lab/issues/41)[(#44)](https://github.com/GoogleCloudPlatform/rad-lab/pull/44) + +### Changed +- _RAD Lab Launcher:_ Added a CLI argument / flag to disable RADLab Permission pre-checks. [(#46)](https://github.com/GoogleCloudPlatform/rad-lab/pull/46) + +### Fixed +- _RAD Lab Launcher:_ Fixing links in the Error Messages. [(1e64c2d)](https://github.com/GoogleCloudPlatform/rad-lab/commit/1e64c2d2074c5688a589651ebaf2e1845370897a) +- _RAD Lab Launcher:_ Fixing a bug to also check for Owner role (i.e. roles/owner) which already contains all required permission for pre-checks for RAD Lab management project. [(#46)](https://github.com/GoogleCloudPlatform/rad-lab/pull/46) + ## [5.0.0](https://github.com/GoogleCloudPlatform/rad-lab/releases/tag/v5.0.0) (2022-02-23) [Full Changelog](https://github.com/GoogleCloudPlatform/rad-lab/compare/v4.2.1...v5.0.0) diff --git a/modules/data_science/README.md b/modules/data_science/README.md index 9b3b1b91..d00956ab 100644 --- a/modules/data_science/README.md +++ b/modules/data_science/README.md @@ -2,7 +2,7 @@ ## GCP Products/Services -* AI Platform Notebooks +* Vertex AI Workbench Notebooks (a.k.a. AI Platform Notebooks) * BigQuery * Cloud Storage * Virtual Private Cloud (VPC) @@ -13,6 +13,8 @@ Below Architechture Diagram is the base representation of what will be created a ![](../../docs/images/V1_DataScience.png) +We provide sample Jupyter Notebooks as part of data science module deployment. If you would like to include your own Jupyter Notebooks add them into [scripts/build/notebooks](./scripts/build/notebooks) folder and then run the deployment. You will be able to access your Jupyter notebooks from the Vertex AI Workbench Notebook created as part of the deployment. + ## IAM Permissions Prerequisites Ensure that the identity executing this module has the following IAM permissions, **when creating the project** (`create_project` = true): @@ -98,10 +100,16 @@ module "existing_project_and_network" { | billing_account_id | Billing Account associated to the GCP Resources | string | ✓ | | | *boot_disk_size_gb* | The size of the boot disk in GB attached to this instance | number | | 100 | | *boot_disk_type* | Disk types for notebook instances | string | | PD_SSD | +| *container_image_repository* | Container Image Repo, only set if creating container image notebook instance by setting `create_container_image` variable to true | string | | | +| *container_image_tag* | Container Image Tag, only set if creating container image notebook instance by setting `create_container_image` variable to true | string | | latest | +| *create_container_image* | If the notebook needs to have image type as Container set this variable to true, set it to false when using dafault image type i.e. VM. | bool | | false | | *create_network* | If the module has to be deployed in an existing network, set this variable to false. | bool | | true | | *create_project* | Set to true if the module has to create a project. If you want to deploy in an existing project, set this variable to false. | bool | | true | +| *enable_gpu_driver* | Install GPU driver on the instance | bool | | false | | *enable_services* | Enable the necessary APIs on the project. When using an existing project, this can be set to false. | bool | | true | | *folder_id* | Folder ID where the project should be created. It can be skipped if already setting organization_id. Leave blank if the project should be created directly underneath the Organization node. | string | | | +| *gpu_accelerator_core_count* | Number of of GPU core count | number | | null | +| *gpu_accelerator_type* | Type of GPU you would like to spin up | string | | | | *image_family* | Image of the AI notebook. | string | | tf-latest-cpu | | *image_project* | Google Cloud project where the image is hosted. | string | | deeplearning-platform-release | | *ip_cidr_range* | Unique IP CIDR Range for AI Notebooks subnet | string | | 10.142.190.0/24 | diff --git a/modules/data_science/main.tf b/modules/data_science/main.tf index acf5466a..d6d9e126 100644 --- a/modules/data_science/main.tf +++ b/modules/data_science/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,8 @@ locals { "roles/compute.instanceAdmin", "roles/notebooks.admin", "roles/bigquery.user", - "roles/storage.objectViewer" + "roles/storage.objectViewer", + "roles/iam.serviceAccountUser" ] project_services = var.enable_services ? [ @@ -182,14 +183,34 @@ resource "google_notebooks_instance" "ai_notebook" { location = var.zone machine_type = var.machine_type - vm_image { - project = var.image_project - image_family = var.image_family + dynamic "vm_image" { + for_each = var.create_container_image ? [] : [1] + content { + project = var.image_project + image_family = var.image_family + } + } + + dynamic "container_image" { + for_each = var.create_container_image ? [1] : [] + content { + repository = var.container_image_repository + tag = var.container_image_tag + } + } + + install_gpu_driver = var.enable_gpu_driver + + dynamic "accelerator_config"{ + for_each = var.enable_gpu_driver ? [1] : [] + content { + type = var.gpu_accelerator_type + core_count = var.gpu_accelerator_core_count + } } service_account = google_service_account.sa_p_notebook.email - install_gpu_driver = false boot_disk_type = var.boot_disk_type boot_disk_size_gb = var.boot_disk_size_gb @@ -209,7 +230,10 @@ resource "google_notebooks_instance" "ai_notebook" { terraform = "true" proxy-mode = "mail" } - depends_on = [time_sleep.wait_120_seconds] + depends_on = [ + time_sleep.wait_120_seconds, + google_storage_bucket_object.notebooks + ] } resource "google_storage_bucket" "user_scripts_bucket" { diff --git a/modules/data_science/orgpolicy.tf b/modules/data_science/orgpolicy.tf index e93ba9e3..8e7f8512 100644 --- a/modules/data_science/orgpolicy.tf +++ b/modules/data_science/orgpolicy.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/data_science/outputs.tf b/modules/data_science/outputs.tf index 98f6dcaa..11d81369 100644 --- a/modules/data_science/outputs.tf +++ b/modules/data_science/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/modules/data_science/samplenotebooks.tf b/modules/data_science/samplenotebooks.tf index 64436fcb..39c91f04 100644 --- a/modules/data_science/samplenotebooks.tf +++ b/modules/data_science/samplenotebooks.tf @@ -15,21 +15,10 @@ */ - resource "google_storage_bucket_object" "notebook1" { - name = "notebooks/BigQuery_tutorial.ipynb" - source = "${path.module}/scripts/build/BigQuery_tutorial.ipynb" - bucket = google_storage_bucket.user_scripts_bucket.name -} - -resource "google_storage_bucket_object" "notebook2" { - name = "notebooks/Exploring_gnomad_on_BigQuery.ipynb" - source = "${path.module}/scripts/build/Exploring_gnomad_on_BigQuery.ipynb" - bucket = google_storage_bucket.user_scripts_bucket.name -} - -resource "google_storage_bucket_object" "notebook3" { - name = "notebooks/Quantum_Simulation_qsimcirq.ipynb" - source = "${path.module}/scripts/build/Quantum_Simulation_qsimcirq.ipynb" +resource "google_storage_bucket_object" "notebooks" { + for_each = fileset("${path.module}/scripts/build/", "notebooks/*.ipynb") + name = each.value + source = join("/", ["${path.module}/scripts/build", each.value]) bucket = google_storage_bucket.user_scripts_bucket.name } diff --git a/modules/data_science/scripts/build/README.md b/modules/data_science/scripts/build/README.md index 866df474..2a9aa847 100644 --- a/modules/data_science/scripts/build/README.md +++ b/modules/data_science/scripts/build/README.md @@ -7,7 +7,7 @@ * Not all warning messages are bad! Often you will run a cell that will return warnings in the output (sometimes these warnings will even be alarmingly color-coded in red or pink). This does not mean that the cell was not successful, and these warnings can often be ignored. -**What's in sample/bigquery-public-data**. +**What's in sample/notebooks**. Example AI notebooks (jupyter notebook) in a GCS location that gets added to your AI notebooks when you install Radlab Datascience module, you can update this script to pull or use your own examples from a GCS location based on what is needed for the end users. @@ -49,8 +49,3 @@ This code is shared as is for quickly deploying development environments with re For all GCP support queries reach out to Google Cloud support. For Radlab specific queries reach out to your Google account team who provisioned Radlab access. - -### License - -### Workspace Change Log - diff --git a/modules/data_science/scripts/build/BigQuery_tutorial.ipynb b/modules/data_science/scripts/build/notebooks/BigQuery_tutorial.ipynb similarity index 100% rename from modules/data_science/scripts/build/BigQuery_tutorial.ipynb rename to modules/data_science/scripts/build/notebooks/BigQuery_tutorial.ipynb diff --git a/modules/data_science/scripts/build/Exploring_gnomad_on_BigQuery.ipynb b/modules/data_science/scripts/build/notebooks/Exploring_gnomad_on_BigQuery.ipynb similarity index 100% rename from modules/data_science/scripts/build/Exploring_gnomad_on_BigQuery.ipynb rename to modules/data_science/scripts/build/notebooks/Exploring_gnomad_on_BigQuery.ipynb diff --git a/modules/data_science/scripts/build/Quantum_Simulation_qsimcirq.ipynb b/modules/data_science/scripts/build/notebooks/Quantum_Simulation_qsimcirq.ipynb similarity index 100% rename from modules/data_science/scripts/build/Quantum_Simulation_qsimcirq.ipynb rename to modules/data_science/scripts/build/notebooks/Quantum_Simulation_qsimcirq.ipynb diff --git a/modules/data_science/scripts/build/startup_script.sh b/modules/data_science/scripts/build/startup_script.sh index 067845ff..13426818 100644 --- a/modules/data_science/scripts/build/startup_script.sh +++ b/modules/data_science/scripts/build/startup_script.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2021 Google LLC +# Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ # Create dir for sample notebooks echo "Creating directory to store samples." -mkdir -p sample/bigquery-public-data +mkdir -p /home/jupyter/sample/notebooks # Setting environment variabled for Project ID echo "Setting Project ID variable." @@ -24,9 +24,7 @@ export PROJECT_ID=$(gcloud config get-value project) # Copy samples to the notebook echo "Copying sample notebooks to the instance." -gsutil cp gs://user-scripts-${PROJECT_ID}/notebooks/BigQuery_tutorial.ipynb /home/jupyter/sample/bigquery-public-data/BigQuery_tutorial.ipynb -gsutil cp gs://user-scripts-${PROJECT_ID}/notebooks/Exploring_gnomad_on_BigQuery.ipynb /home/jupyter/sample/bigquery-public-data/Exploring_gnomad_on_BigQuery.ipynb -gsutil cp gs://user-scripts-${PROJECT_ID}/notebooks/Quantum_Simulation_qsimcirq.ipynb /home/jupyter/sample/bigquery-public-data/Quantum_Simulation_qsimcirq.ipynb +gsutil -m cp -r gs://user-scripts-${PROJECT_ID}/notebooks/*.ipynb /home/jupyter/sample/notebooks echo "Startup script finished." diff --git a/modules/data_science/terraform.tfvars.example b/modules/data_science/terraform.tfvars.example new file mode 100644 index 00000000..86d432b2 --- /dev/null +++ b/modules/data_science/terraform.tfvars.example @@ -0,0 +1,101 @@ +# Organization ID where GCP Resources need to get spin up for RAD Lab. It can be skipped if already setting folder_id +organization_id = "123456789" + +# Folder ID where the project should be created. It can be skipped if already setting organization_id. +# Leave blank if the project should be created directly underneath the Organization node. +folder_id = "987654321" + +# Billing Account associated to the GCP Resources +billing_account_id = "ABCDE-12345-FGHIJ" + +# When using an existing project, this can be set to false if the required APIs are already enabled. +# Default is true to enable necessary APIs on the newly created GCP project for RAD Lab +enable_services = false + +# When setting below variable to false to deploy module in existing GCP project. +# Default is true i.e. to create a new project. +create_project = false + +# Prefix for the project ID, if creating new GCP project in RAD Lab deployment using above variable. +# GCP Project ID of existing project, if using existing GCP project in RAD Lab deployment using above variable. +project_name = "radlab-project" + +# Random ID a.k.a. Deployment ID for the RAD Lab Deployment. +# Only set it when either Updating or Deleting the existing deployment. +# This will be added as suffix of 4 random characters to the `project_name` when creating the deployment. +random_id = "abc1" + +# When you have already se the below Orgpolicies set it below variables to false. +# Default is true to set necessary Org Policies on the newly created GCP project for RAD Lab +set_external_ip_policy = false +set_shielded_vm_policy = false +set_trustedimage_project_policy = false + +# GCP Zone where all the GCP resources will be spun up. +zone = "us-east4-c" + +# When setting below variable to false to deploy module in existing VPC network. Default is true +create_network = false + +# Name of the network to be created or if using the existing one. +network_name = "network-name" + +# Name of the sub-network to be created or if using the existing one. +subnet_name = "sub-network-name" + +# Unique IP CIDR Range for the subnet. Not required if using existing network. +ip_cidr_range = "10.1.1.0/24" + +# Flag to determine what type of Image you would like to deploy for AI Notebook Workbench. +# Default is false i.e. VM Image type and to create Container Image type set the variable to true. +create_container_image = false + +# Number of notebooks you would like to spin up as part of RAD Lab Data Science environment. +# Default is 1 +notebook_count = "5" + +# List of Users you would like to give access to the created Notebooks & GCS Bucket apart from the identity created the notebook. +trusted_users = ["user:notebookuser1@domain.com", "user:notebookuser2@domain.com"] + +# Below 2 variables are specific only to VM type image for Notebooks. +## Specific Deep Learning VM Images for the AI Notebook. More Info: https://cloud.google.com/deep-learning-vm/docs/images +image_family = "tf-latest-cpu" + +## Google Cloud project where the image is hosted. +image_project = "deeplearning-platform-release" + +# Below 2 variables are specific only to Container type image for Notebooks. + +## The path to the container image repository. For example: gcr.io/{project_id}/{imageName} +## More Info: https://cloud.google.com/container-registry/docs/pushing-and-pulling#add-registry +## The link to the list of Google Container Registry public images is: https://console.cloud.google.com/gcr/images/google-containers/GLOBAL. +container_image_repository = "" + + +## The tag of the container image. +## If not specified, this defaults to the latest tag. +container_image_tag = "latest" + +# Set Machine type which defines VM kind +# More Info: https://cloud.google.com/compute/docs/general-purpose-machines +machine_type = "n1-standard-1" + +# The size of the boot disk in GB attached to this instance, up to a maximum of 64000 GB (64 TB). +# The minimum recommended value is 100 GB. If not specified, this defaults to 100. +boot_disk_size_gb = 200 + +# Possible disk types for notebook instances. +# Possible values are DISK_TYPE_UNSPECIFIED, PD_STANDARD, PD_SSD, and PD_BALANCED. +boot_disk_type = "PD_SSD" + +# Flag to Install GPU Driver. +# Default is false i.e. do not install GPU Driver +enable_gpu_driver = true + +# Type of this accelerator. +# Possible values are ACCELERATOR_TYPE_UNSPECIFIED, NVIDIA_TESLA_K80, NVIDIA_TESLA_P100, NVIDIA_TESLA_V100, NVIDIA_TESLA_P4, NVIDIA_TESLA_T4, NVIDIA_TESLA_T4_VWS, NVIDIA_TESLA_P100_VWS, NVIDIA_TESLA_P4_VWS, NVIDIA_TESLA_A100, TPU_V2, and TPU_V3. +# Default none is set. +gpu_accelerator_type = "NVIDIA_TESLA_V100" + +# Count of cores of the above accelerator. +gpu_accelerator_core_count = 1 diff --git a/modules/data_science/variables.tf b/modules/data_science/variables.tf index bda67d82..acb4b4c3 100644 --- a/modules/data_science/variables.tf +++ b/modules/data_science/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,12 @@ variable "boot_disk_type" { default = "PD_SSD" } +variable "create_container_image" { + description = "If the notebook needs to have image type as Container set this variable to true, set it to false when using dafault image type i.e. VM." + type = bool + default = false +} + variable "create_network" { description = "If the module has to be deployed in an existing network, set this variable to false." type = bool @@ -43,6 +49,24 @@ variable "create_project" { default = true } +variable "container_image_repository" { + description = "Container Image Repo, only set if creating container image notebook instance by setting `create_container_image` variable to true" + type = string + default = "" +} + +variable "container_image_tag" { + description = "Container Image Tag, only set if creating container image notebook instance by setting `create_container_image` variable to true" + type = string + default = "latest" +} + +variable "enable_gpu_driver" { + description = "Install GPU driver on the instance" + type = bool + default = false +} + variable "enable_services" { description = "Enable the necessary APIs on the project. When using an existing project, this can be set to false." type = bool @@ -55,6 +79,18 @@ variable "folder_id" { default = "" } +variable "gpu_accelerator_type" { + description = "Type of GPU you would like to spin up" + type = string + default = "" +} + +variable "gpu_accelerator_core_count" { + description = "Number of of GPU core count" + type = number + default = null +} + variable "image_family" { description = "Image of the AI notebook." type = string diff --git a/radlab-launcher/README.md b/radlab-launcher/README.md index a70977d1..12fd470a 100644 --- a/radlab-launcher/README.md +++ b/radlab-launcher/README.md @@ -79,24 +79,39 @@ You can use the Google Cloud Console to [view](https://cloud.google.com/iam/docs python3 radlab.py ``` -3. To set any module specific variables, use `--varfile` argument while running **radlab.py** and pass a file with variables content: +NOTE: Save the **deployment_id** from the output for future reference. It is required to supply the deployment id for updating or deleting the RAD Lab module deployment. -``` -% python3 radlab.py -h -usage: radlab.py [-h] [--varfile FILE] +### Using Command Line Arguments + +RAD Lab launcher accepts following command line arguments: + +* `--rad-project` or `-p` : To set GCP Project ID for RAD Lab management. +* `--rad-bucket` or `-b` : To set GCS Bucket ID under RAD Lab management GCP Project where Terraform states & configs for the modules will be stored. +* `--module` or `-m` : To select the RAD Lab Module you would like to deploy. +* `--action` or `-a` : To select the action you would like to perform on the selected RAD Lab module. +* `--varfile` or `-f` : To pass a file with the key-value pairs of module variables and their default overriden values. +* `--disable-perm-check` or `-dc` : To disable RAD Lab permissions pre-check . NOTE: This doesn't means one will not need the required permissions. This will just disable the permission pre-checks which RAD Lab Launcher do for the module deployments. Thus deployment may still fail eventually if required permissions are not set for the identity spinning up the modules. -optional arguments: - -h, --help show this help message and exit - --varfile FILE Input file (with complete path) for terraform.tfvars contents -% +_Usage:_ + +``` +python3 radlab.py --module --action --rad-project --rad-bucket --varfile ``` -NOTE: When the above parameter is not passed then the modules are deployed with module's default variable values. +OR +``` +python3 radlab.py -m -a -p -b -f +``` + +### Overriding default variables of a RAD Lab Module -Example : +To set any module specific variables, use `--varfile` argument while running **radlab.py** and pass a file with variables content. Variables like **organization_id**, **folder_id**, **billing_account_id**, **random_id** (a.k.a. **deployment id**), which are requested as part of guided setup, can be set via --varfile argument by passing them in a file. There is a `terraform.tfvars.example` file under each [module](../modules/) as an example on how can you set/override the default variables. + +_Usage :_ ``` python3 radlab.py --varfile // ``` -NOTE: Save the **deployment_id** from the output for future reference. It is used to make updates or delete the RAD Lab module deployment. + +NOTE: When the above argument is not passed then the modules are deployed with module's default variable values in `variables.tf` file. ## Example Launch of Data Science Module diff --git a/radlab-launcher/installer_prereq.py b/radlab-launcher/installer_prereq.py index c010c4ae..3005f57d 100644 --- a/radlab-launcher/installer_prereq.py +++ b/radlab-launcher/installer_prereq.py @@ -16,12 +16,11 @@ import os import subprocess -from colorama import Fore, Back, Style def main(): # Install python dependencies. - print(Fore.BLUE + "\nInstalling Libraries..." + Style.RESET_ALL) + print("\nInstalling Libraries...") os.system("pip3 install --no-cache-dir -r requirements.txt") # Set up Terraform binaries @@ -29,10 +28,10 @@ def main(): ## Check if Terrafrom binaries are already installed if "command not found" in tfOutput: - print(Fore.YELLOW + "\nTerraform binaries not installed. Starting installation...\n" + Style.RESET_ALL) + print("\nTerraform binaries not installed. Starting installation...\n") os.system("python3 terraform_installer.py") else: - print(Fore.YELLOW + "\nTerraform binaries already installed. Skipping installation...\n" + Style.RESET_ALL) + print("\nTerraform binaries already installed. Skipping installation...\n") # Printing Terraform Version os.system("terraform -version") @@ -40,7 +39,7 @@ def main(): # Set up Cloud sdk & Kubectl libraries os.system("python3 cloudsdk_kubectl_installer.py") - print(Fore.BLUE + "\nPRE-REQ INSTALLTION COMPLETED\n" + Style.RESET_ALL) + print("\nPRE-REQ INSTALLTION COMPLETED\n") if __name__ == "__main__": main() \ No newline at end of file diff --git a/radlab-launcher/radlab.py b/radlab-launcher/radlab.py index d992d5a2..7f6b1f3a 100644 --- a/radlab-launcher/radlab.py +++ b/radlab-launcher/radlab.py @@ -28,6 +28,7 @@ import argparse import platform import subprocess +from art import * from os import path from pprint import pprint from google.cloud import storage @@ -36,19 +37,17 @@ from python_terraform import Terraform from oauth2client.client import GoogleCredentials -STATE_CREATE_DEPLOYMENT = "1" -STATE_UPDATE_DEPLOYMENT = "2" -STATE_DELETE_DEPLOYMENT = "3" -STATE_LIST_DEPLOYMENT = "4" +ACTION_CREATE_DEPLOYMENT = "1" +ACTION_UPDATE_DEPLOYMENT = "2" +ACTION_DELETE_DEPLOYMENT = "3" +ACTION_LIST_DEPLOYMENT = "4" + +def main(varcontents={}, module_name=None , action=None, projid=None, tfbucket=None, check=None): -def main(varcontents={}): - - projid = "" orgid = "" folderid = "" billing_acc = "" currentusr = "" - state = "" setup_path = os.getcwd() @@ -59,22 +58,29 @@ def main(varcontents={}): projid = set_proj(projid) # Checking for User Permissions - launcherperm(projid,currentusr) + if check == True: + launcherperm(projid,currentusr) # Listing / Selecting from available RAD Lab modules - module_name = list_modules() + if module_name is None: + module_name = list_modules() # Checking Module specific permissions - moduleperm(projid,module_name,currentusr) + if check == True: + moduleperm(projid,module_name,currentusr) # Validating user input Terraform variables against selected module validate_tfvars(varcontents, module_name) + # Select Action to perform + if action is None or action == "": + action = select_action().strip() + # Setting up required attributes for any RAD Lab module deployment - state,env_path,tfbucket,orgid,billing_acc,folderid,randomid = module_deploy_common_settings(module_name,setup_path,varcontents,projid) + env_path,tfbucket,orgid,billing_acc,folderid,randomid = module_deploy_common_settings(action,module_name,setup_path,varcontents,projid,tfbucket) # Utilizing Terraform Wrapper for init / apply / destroy - env(state, orgid, billing_acc, folderid, env_path, randomid, tfbucket, projid) + env(action, orgid, billing_acc, folderid, env_path, randomid, tfbucket, projid) print("\nGCS Bucket storing Terrafrom Configs: "+ tfbucket +"\n") print("\nTERRAFORM DEPLOYMENT COMPLETED!!!\n") @@ -114,16 +120,19 @@ def radlabauth(currentusr): return currentusr def set_proj(projid): - projid = os.popen("gcloud config list --format 'value(core.project)' 2>/dev/null").read().strip() - if(projid != ""): - select_proj = input("\nWhich Project would you like to use for RAD Lab management (Example - Creating/Utilizing GCS bucket where Terraform states will be stored) ? :" + "\n[1] Currently set project - " + Fore.GREEN + projid + Style.RESET_ALL + "\n[2] Enter a different Project ID" +Fore.YELLOW + Style.BRIGHT + "\nChoose a number for the RAD Lab management Project" + Style.RESET_ALL + ': ').strip() - if(select_proj == '2'): - projid = input(Fore.YELLOW + Style.BRIGHT + "Enter the Project ID" + Style.RESET_ALL + ': ').strip() + if projid is None: + projid = os.popen("gcloud config list --format 'value(core.project)' 2>/dev/null").read().strip() + if(projid != ""): + select_proj = input("\nWhich Project would you like to use for RAD Lab management (Example - Creating/Utilizing GCS bucket where Terraform states will be stored) ? :" + "\n[1] Currently set project - " + Fore.GREEN + projid + Style.RESET_ALL + "\n[2] Enter a different Project ID" +Fore.YELLOW + Style.BRIGHT + "\nChoose a number for the RAD Lab management Project" + Style.RESET_ALL + ': ').strip() + if(select_proj == '2'): + projid = input(Fore.YELLOW + Style.BRIGHT + "Enter the Project ID" + Style.RESET_ALL + ': ').strip() + os.system("gcloud config set project " + projid) + elif(select_proj != '1' and select_proj != '2'): + sys.exit(Fore.RED + "\nError Occured - INVALID choice.\n") + else: + projid = input(Fore.YELLOW + Style.BRIGHT + "\nEnter the Project ID for RAD Lab management" + Style.RESET_ALL + ': ').strip() os.system("gcloud config set project " + projid) - elif(select_proj != '1' and select_proj != '2'): - sys.exit(Fore.RED + "\nError Occured - INVALID choice.\n") else: - projid = input(Fore.YELLOW + Style.BRIGHT + "\nEnter the Project ID for RAD Lab management" + Style.RESET_ALL + ': ').strip() os.system("gcloud config set project " + projid) print("\nProject ID (Selected) : " + Fore.GREEN + Style.BRIGHT + projid + Style.RESET_ALL) return projid @@ -143,24 +152,36 @@ def launcherperm(projid,currentusr): projiam = True for role in launcherprojroles: rolefound = False + ownerrole = False for y in range(len(response0['bindings'])): # print("ROLE --->") # print(response0['bindings'][y]['role']) # print("MEMBERS --->") # print(response0['bindings'][y]['members']) - if(role == response0['bindings'][y]['role']): + + # Check for Owner role on RAD Lab Management Project + if(response0['bindings'][y]['role'] == 'roles/owner' and 'user:'+currentusr in response0['bindings'][y]['members']): + rolefound = True + ownerrole = True + print("\n" + currentusr + " has roles/owner role for RAD Lab Management Project: " + projid) + break + # Check for Required roles on RAD Lab Management Project + elif(response0['bindings'][y]['role'] == role): rolefound = True if('user:'+currentusr not in response0['bindings'][y]['members']): projiam = False - sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/radlab-launcher/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) else: pass if rolefound == False: - sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n((Review https://github.com/GoogleCloudPlatform/rad-lab/radlab-launcher/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) - + sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + + if(ownerrole == True): + break + if projiam == True: - print(Fore.GREEN + '\nRADLAB MODULE - Project Permission check passed\n' + Style.RESET_ALL) + print(Fore.GREEN + '\nRADLAB LAUNCHER - Project Permission check passed' + Style.RESET_ALL) service1 = discovery.build('cloudresourcemanager', 'v3', credentials=credentials) request1 = service1.projects().get(name='projects/'+projid) @@ -168,33 +189,45 @@ def launcherperm(projid,currentusr): if 'parent' in response1.keys(): service2 = discovery.build('cloudresourcemanager', 'v3', credentials=credentials) - request2 = service2.organizations().getIamPolicy(resource=response1['parent']) + org = findorg(response1['parent']) + request2 = service2.organizations().getIamPolicy(resource=org) response2 = request2.execute() orgiam = True for role in launcherorgroles: rolefound = False for x in range(len(response2['bindings'])): - # print("ROLE --->") - # print(response2['bindings'][x]['role']) - # print("MEMBERS --->") - # print(response2['bindings'][x]['members']) + # print("ROLE --->") + # print(response2['bindings'][x]['role']) + # print("MEMBERS --->") + # print(response2['bindings'][x]['members']) if(role == response2['bindings'][x]['role']): rolefound = True if('user:'+currentusr not in response2['bindings'][x]['members']): orgiam = False - sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/radlab-launcher/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) else: pass if rolefound == False: - sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/radlab-launcher/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB LAUNCHER PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/radlab-launcher#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) if orgiam == True: - print(Fore.GREEN + '\nRADLAB MODULE - Organization Permission check passed\n' + Style.RESET_ALL) + print(Fore.GREEN + '\nRADLAB LAUNCHER - Organization Permission check passed' + Style.RESET_ALL) else: print(Fore.YELLOW + '\nRADLAB LAUNCHER - Skipping Organization Permission check. No Organization associated with the project: ' + projid + Style.RESET_ALL) +def findorg(parent): + + if 'folders' in parent: + credentials = GoogleCredentials.get_application_default() + s = discovery.build('cloudresourcemanager', 'v3', credentials=credentials) + req = s.folders().get(name=parent) + res = req.execute() + return findorg(res['parent']) + else: + # print(Fore.GREEN + "Org identified: " + Style.BRIGHT + parent + Style.RESET_ALL) + return parent def moduleperm(projid,module_name,currentusr): @@ -291,7 +324,7 @@ def moduleperm(projid,module_name,currentusr): except Exception as e: print(e) - print("SET ORG POLICY: " + str(setorgpolicy)) + print("\nSET ORG POLICY: " + str(setorgpolicy)) print("CREATE PROJECT: " + str(create_project)) # Scrape out Module specific permissions for the module @@ -351,16 +384,16 @@ def moduleperm(projid,module_name,currentusr): rolefound = True if('user:'+currentusr not in response1['bindings'][y]['members']): projiam = False - sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/modules/"+module_name+"/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/modules/"+module_name+"#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) else: pass if rolefound == False: - sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/modules/"+module_name+"/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/modules/"+module_name+"#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) if projiam == True: - print(Fore.GREEN + '\nRADLAB MODULE - Project Permission check passed\n' + Style.RESET_ALL) + print(Fore.GREEN + '\nRADLAB MODULE ('+module_name+')- Project Permission check passed' + Style.RESET_ALL) # Check Org level permissions if len(orgroles) != 0: @@ -371,7 +404,8 @@ def moduleperm(projid,module_name,currentusr): if 'parent' in response.keys(): # print("/*************** ORG IAM POLICY *************/") - request2 = service.organizations().getIamPolicy(resource=response['parent']) + org = findorg(response['parent']) + request2 = service.organizations().getIamPolicy(resource=org) response2 = request2.execute() # pprint(response2) orgiam = True @@ -387,25 +421,25 @@ def moduleperm(projid,module_name,currentusr): rolefound = True if('user:'+currentusr not in response2['bindings'][x]['members']): orgiam = False - sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/modules/"+module_name+"/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE ("+ module_name + ") PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/modules/"+module_name+"#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) else: pass if rolefound == False: - sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/modules/"+module_name+"/README.md#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) + sys.exit(Fore.RED + "\nError Occured - RADLAB MODULE ("+ module_name + ") PERMISSION ISSUE | " + role + " permission missing...\n(Review https://github.com/GoogleCloudPlatform/rad-lab/tree/main/modules/"+module_name+"#iam-permissions-prerequisites for more details)\n" +Style.RESET_ALL ) if orgiam == True: - print(Fore.GREEN + '\nRADLAB MODULE - Organization Permission check passed\n' + Style.RESET_ALL) + print(Fore.GREEN + '\nRADLAB MODULE ('+ module_name + ') - Organization Permission check passed' + Style.RESET_ALL) else: print(Fore.YELLOW + '\nRADLAB LAUNCHER - Skipping Organization Permission check. No Organization associated with the project: ' + projid + Style.RESET_ALL) -def env(state, orgid, billing_acc, folderid, env_path, randomid, tfbucket, projid): +def env(action, orgid, billing_acc, folderid, env_path, randomid, tfbucket, projid): tr = Terraform(working_dir=env_path) return_code, stdout, stderr = tr.init_cmd(capture_output=False) - if(state == STATE_CREATE_DEPLOYMENT or state == STATE_UPDATE_DEPLOYMENT): + if(action == ACTION_CREATE_DEPLOYMENT or action == ACTION_UPDATE_DEPLOYMENT): return_code, stdout, stderr = tr.apply_cmd(capture_output=False,auto_approve=True,var={'organization_id':orgid, 'billing_account_id':billing_acc, 'folder_id':folderid, 'random_id':randomid}) - elif(state == STATE_DELETE_DEPLOYMENT): + elif(action == ACTION_DELETE_DEPLOYMENT): return_code, stdout, stderr = tr.destroy_cmd(capture_output=False,auto_approve=True,var={'organization_id':orgid, 'billing_account_id':billing_acc, 'folder_id':folderid,'random_id':randomid}) # return_code - 0 Success & 1 Error @@ -415,7 +449,7 @@ def env(state, orgid, billing_acc, folderid, env_path, randomid, tfbucket, proji else: target_path = 'radlab/'+ env_path.split('/')[len(env_path.split('/'))-1] +'/deployments' - if(state == STATE_CREATE_DEPLOYMENT or state == STATE_UPDATE_DEPLOYMENT): + if(action == ACTION_CREATE_DEPLOYMENT or action == ACTION_UPDATE_DEPLOYMENT): if glob.glob(env_path + '/*.tf'): upload_from_directory(projid, env_path, '/*.tf', tfbucket, target_path) @@ -428,7 +462,7 @@ def env(state, orgid, billing_acc, folderid, env_path, randomid, tfbucket, proji if glob.glob(env_path + '/templates'): upload_from_directory(projid, env_path, '/templates/**', tfbucket, target_path) - elif(state == STATE_DELETE_DEPLOYMENT): + elif(action == ACTION_DELETE_DEPLOYMENT): deltfgcs(tfbucket, 'radlab/'+ env_path.split('/')[len(env_path.split('/'))-1], projid) # Deleting Local deployment config @@ -446,10 +480,10 @@ def upload_from_directory(projid, directory_path: str, content: str, dest_bucket blob = bucket.blob(remote_path) blob.upload_from_filename(local_file) -def select_state(): - state = input("\nAction to perform for RAD Lab Deployment ?\n[1] Create New\n[2] Update\n[3] Delete\n[4] List\n" + Fore.YELLOW + Style.BRIGHT + "Choose a number for the RAD Lab Module Deployment Action"+ Style.RESET_ALL + ': ').strip() - if(state == STATE_CREATE_DEPLOYMENT or state == STATE_UPDATE_DEPLOYMENT or state == STATE_DELETE_DEPLOYMENT or state == STATE_LIST_DEPLOYMENT): - return state +def select_action(): + action = input("\nAction to perform for RAD Lab Deployment ?\n[1] Create New\n[2] Update\n[3] Delete\n[4] List\n" + Fore.YELLOW + Style.BRIGHT + "Choose a number for the RAD Lab Module Deployment Action"+ Style.RESET_ALL + ': ').strip() + if(action == ACTION_CREATE_DEPLOYMENT or action == ACTION_UPDATE_DEPLOYMENT or action == ACTION_DELETE_DEPLOYMENT or action == ACTION_LIST_DEPLOYMENT): + return action else: sys.exit(Fore.RED + "\nError Occured - INVALID choice.\n") @@ -661,15 +695,15 @@ def delifexist(env_path): if(os.path.isdir(env_path)): shutil.rmtree(env_path) -def getbucket(state,projid): +def getbucket(action,projid): """Lists all buckets.""" storage_client = storage.Client(project=projid) bucketoption = '' - if(state == STATE_CREATE_DEPLOYMENT): + if(action == ACTION_CREATE_DEPLOYMENT): bucketoption = input("\nWant to use existing GCS Bucket for Terraform configs or Create Bucket ?:\n[1] Use Existing Bucket\n[2] Create New Bucket\n"+ Fore.YELLOW + Style.BRIGHT + "Choose a number for your choice"+ Style.RESET_ALL + ': ').strip() - if(bucketoption == '1' or state == STATE_UPDATE_DEPLOYMENT or state == STATE_DELETE_DEPLOYMENT or state == STATE_LIST_DEPLOYMENT): + if(bucketoption == '1' or action == ACTION_UPDATE_DEPLOYMENT or action == ACTION_DELETE_DEPLOYMENT or action == ACTION_LIST_DEPLOYMENT): try: buckets = storage_client.list_buckets() @@ -798,16 +832,15 @@ def list_modules(): else: sys.exit(Fore.RED + "\nInvalid module") -def module_deploy_common_settings(module_name,setup_path,varcontents,projid): - # Select Action to perform - state = select_state() +def module_deploy_common_settings(action,module_name,setup_path,varcontents,projid,tfbucket): # Get Terraform Bucket Details - tfbucket = getbucket(state.strip(),projid) + if tfbucket is None: + tfbucket = getbucket(action,projid) print("\nGCS bucket for Terraform config & state (Selected) : " + Fore.GREEN + Style.BRIGHT + tfbucket + Style.RESET_ALL ) # Setting Org ID, Billing Account, Folder ID - if(state == STATE_CREATE_DEPLOYMENT): + if(action == ACTION_CREATE_DEPLOYMENT): # Check for any overides of basic inputs from terraform.tfvars file orgid, billing_acc, folderid, randomid = check_basic_inputs_tfvars(varcontents) @@ -834,9 +867,9 @@ def module_deploy_common_settings(module_name,setup_path,varcontents,projid): # Create file with billing/org/folder details create_env(env_path, orgid, billing_acc, folderid) - return state,env_path,tfbucket,orgid,billing_acc,folderid,randomid + return env_path,tfbucket,orgid,billing_acc,folderid,randomid - elif(state == STATE_UPDATE_DEPLOYMENT or state == STATE_DELETE_DEPLOYMENT): + elif(action == ACTION_UPDATE_DEPLOYMENT or action == ACTION_DELETE_DEPLOYMENT): # Get Deployment ID randomid = input(Fore.YELLOW + Style.BRIGHT + "\nEnter RAD Lab Module Deployment ID (example 'l8b3' is the id for project with id - radlab-ds-analytics-l8b3)" + Style.RESET_ALL + ': ') @@ -862,22 +895,22 @@ def module_deploy_common_settings(module_name,setup_path,varcontents,projid): settfstategcs(env_path,prefix,tfbucket,projid) # Create file with billing/org/folder details - if(state == STATE_UPDATE_DEPLOYMENT): + if(action == ACTION_UPDATE_DEPLOYMENT): if os.path.exists(env_path + '/terraform.tfvars'): os.remove(env_path + '/terraform.tfvars') create_tfvars(env_path,varcontents) - if(state == STATE_DELETE_DEPLOYMENT): + if(action == ACTION_DELETE_DEPLOYMENT): print("DELETING DEPLOYMENT...") - return state,env_path,tfbucket,orgid,billing_acc,folderid,randomid + return env_path,tfbucket,orgid,billing_acc,folderid,randomid - elif(state == STATE_LIST_DEPLOYMENT): + elif(action == ACTION_LIST_DEPLOYMENT): list_radlab_deployments(tfbucket, module_name, projid) sys.exit() else: - sys.exit(Fore.RED + "\nInvalid RAD Lab Module State selected") + sys.exit(Fore.RED + "\nInvalid RAD Lab Module Action selected") def validate_tfvars(varcontents, module_name): @@ -955,15 +988,40 @@ def fetchvariables(filecontents): sys.exit(Fore.RED + 'No variables in the input file') if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--varfile', dest="file", type=argparse.FileType('r', encoding='UTF-8'), help="Input file (with complete path) for terraform.tfvars contents", required=False) - args = parser.parse_args() - - if args.file is not None: - print("Checking input file...") - filecontents = args.file.readlines() - variables = fetchvariables(filecontents) - else: - variables = {} - main(variables) - # moduleperm("radlab-solution","app_mod_elastic","radlab-admin@guptamukul.altostrat.com") \ No newline at end of file + try: + print('\n'+text2art("RADLAB",font="larry3d")) + + parser = argparse.ArgumentParser() + parser.add_argument('-p','--rad-project', dest="projid", help="RAD Lab management GCP Project.", required=False) + parser.add_argument('-b','--rad-bucket', dest="tfbucket", help="RAD Lab management GCS Bucket where Terraform states for the modules will be stored.", required=False) + parser.add_argument('-m','--module', dest="module_name", choices=sorted([s.replace(os.path.dirname(os.getcwd()) + '/modules/', "") for s in glob.glob(os.path.dirname(os.getcwd()) + '/modules/*')]), help="RADLab Module name under ../../modules folder", required=False) + parser.add_argument('-a','--action', dest="action", choices=['create', 'update', 'delete','list'], help="Type of action you want to perform for the selected RADLab module.", required=False) + parser.add_argument('-f','--varfile', dest="file", type=argparse.FileType('r', encoding='UTF-8'), help="Input file (with complete path) for terraform.tfvars contents.", required=False) + parser.add_argument('-dc','--disable-perm-check', dest="disable_perm_check", action='store_false', help="Flag to disable RAD Lab permissions pre-check.", required=False) + + args = parser.parse_args() + + # File Argument + if args.file is not None: + print("Checking input file...") + filecontents = args.file.readlines() + variables = fetchvariables(filecontents) + else: + variables = {} + + # Action Argument + if args.action == 'create': + action = ACTION_CREATE_DEPLOYMENT + elif args.action == 'update': + action = ACTION_UPDATE_DEPLOYMENT + elif args.action == 'delete': + action = ACTION_DELETE_DEPLOYMENT + elif args.action == 'list': + action = ACTION_LIST_DEPLOYMENT + else: + action = None + + main(variables, args.module_name, action, args.projid, args.tfbucket, args.disable_perm_check) + + except Exception as e: + print(e) \ No newline at end of file diff --git a/radlab-launcher/requirements.txt b/radlab-launcher/requirements.txt index edb62669..31460078 100644 --- a/radlab-launcher/requirements.txt +++ b/radlab-launcher/requirements.txt @@ -1,3 +1,4 @@ +art beautifulsoup4>=0.01 requests>=2.26.0 google-api-python-client>=2.17.0