From 8d0aca8c831f353a8970176108f4bd2b2d52f3ed Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 21 Sep 2021 13:25:33 -0500
Subject: [PATCH 001/126] Add repo to TSC team during bootstrap

---
 utils/exchange-bootstrap.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/utils/exchange-bootstrap.sh b/utils/exchange-bootstrap.sh
index fea53a54..0c5d4dd0 100755
--- a/utils/exchange-bootstrap.sh
+++ b/utils/exchange-bootstrap.sh
@@ -39,6 +39,7 @@ then
 fi
 
 EXCHANGE_ORG="${EXCHANGE_ORG:-StackStorm-Exchange}"
+EXCHANGE_TEAM="${EXCHANGE_TEAM:-tsc}"
 EXCHANGE_PREFIX="${EXCHANGE_PREFIX:-stackstorm}"
 PACK="${1/${EXCHANGE_PREFIX}-/}"  # Ensure that PACK is just the bare pack name
 REPO_ALIAS=${PACK}
@@ -168,6 +169,13 @@ then
        "https://api.github.com/repos/${EXCHANGE_ORG}/${REPO_NAME}/hooks"
 fi
 
+echo "GitHub: Allow ${EXCHANGE_TEAM} team to 'maintain' ${EXCHANGE_ORG}/${REPO_NAME}"
+curl -sS --fail -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" -X PUT \
+     --header "Content-Type: application/json" \
+     --header "Accept: application/vnd.github.v3+json" \
+     -d '{"permission": "maintain"}' \
+     "https://api.github.com/orgs/${EXCHANGE_ORG}/teams/${EXCHANGE_TEAM}/repos/${EXCHANGE_ORG}/${REPO_NAME}"
+
 # This will open a private tab in the user's browser to the PAT page, with the
 # name field already filled in and the scope fields already checked, and direct
 # the user to click the "Generate Token" button, then copy and paste the

From ded32804789903a94fec983f722308e5f825307b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 21 Sep 2021 15:09:32 -0500
Subject: [PATCH 002/126] reenable CircleCI API call that enables CircleCI
 builds

---
 utils/exchange-bootstrap.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/utils/exchange-bootstrap.sh b/utils/exchange-bootstrap.sh
index 0c5d4dd0..21c9831c 100755
--- a/utils/exchange-bootstrap.sh
+++ b/utils/exchange-bootstrap.sh
@@ -185,12 +185,12 @@ curl -sS --fail -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" -X PUT \
 # PAT
 ${CI_REPO_ROOT}/tools/reset_github_user_token_and_update_circleci.sh --set-user $PACK
 
-# NO longer needed, this API request fails everytime we call it
-# # CircleCI: follow the project
-# echo "CircleCI: Following the project"
-# curl -v -sS --fail -X POST \
-#      --header "Circle-Token: ${CIRCLECI_TOKEN}" \
-#      "https://circleci.com/api/v1.1/project/github/${EXCHANGE_ORG}/${REPO_NAME}/follow"
+# XXX: this API request was failing, but it is working now. Not sure why.
+# CircleCI: follow the project (we need this, or CircleCI won't watch the repo)
+echo "CircleCI: Following the project (enables CircleCI builds)"
+curl -v -sS --fail -X POST \
+    --header "Circle-Token: ${CIRCLECI_TOKEN}" \
+    "https://circleci.com/api/v1.1/project/github/${EXCHANGE_ORG}/${REPO_NAME}/follow"
 
 
 # CircleCI: upload the read-write key

From 6364be3b98d816f652131d1ed5cf92459ada216f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 11 Nov 2021 22:15:19 -0600
Subject: [PATCH 003/126] stub gha composite actions

---
 gha/apt-dependencies/action.yaml      |  51 +++++++++++++
 gha/apt-dependencies/apt-packages.txt |  17 +++++
 gha/checkout/action.yaml              |  53 +++++++++++++
 gha/py-dependencies/action.yaml       | 102 ++++++++++++++++++++++++++
 gha/test/action.yaml                  |  59 +++++++++++++++
 5 files changed, 282 insertions(+)
 create mode 100644 gha/apt-dependencies/action.yaml
 create mode 100644 gha/apt-dependencies/apt-packages.txt
 create mode 100644 gha/checkout/action.yaml
 create mode 100644 gha/py-dependencies/action.yaml
 create mode 100644 gha/test/action.yaml

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
new file mode 100644
index 00000000..cdd88782
--- /dev/null
+++ b/gha/apt-dependencies/action.yaml
@@ -0,0 +1,51 @@
+---
+name: Install APT Dependencies
+description: |
+  Install debian dependencies required for StackStorm-Exchange pack tests.
+author: StackStorm
+
+inputs:
+  cache-version:
+    required: false
+    default: "v0"
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Create a directory for debian packages so we can cache it
+      # this is what we did on CircleCI. Not sure if it'll work on GHA
+      run: >
+        sudo rm -rf /var/cache/apt/archives
+        && sudo ln -s ~/apt_cache /var/cache/apt/archives
+        && mkdir -p ~/apt_cache/partial
+
+    # TODO: install and cache apt-packages defined by packs
+    #       maybe via input?
+
+    - name: Cache APT Dependencies
+      id: cache-apt-deps
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/apt_cache
+        key: ${{ runner.os }}-apt-${{ input.cache-version }}-${{ hashFiles(github.action_path + '/apt-packages.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-apt-${{ input.cache-version }}-
+
+    - name: Install APT Dependencies
+      env:
+        CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+        APT_PACKAGES_FILE_PATH: ${{github.action_path + '/apt-packages.txt'}}
+      run: |
+        if [[ "$CACHE_HIT" != 'true' ]]; then
+        sudo apt-get -y update
+        fi
+        APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)
+        sudo apt-get -y install ${APT_PACKAGES}
+
+    - name: Print versions
+      run: |
+        set -ex
+        jq --version
+        gh --version
diff --git a/gha/apt-dependencies/apt-packages.txt b/gha/apt-dependencies/apt-packages.txt
new file mode 100644
index 00000000..7800030f
--- /dev/null
+++ b/gha/apt-dependencies/apt-packages.txt
@@ -0,0 +1,17 @@
+# pre-installed on GHA
+#jq
+#gh # github cli
+#imagemagick
+
+# for st2 pip build
+libldap2-dev
+libsasl2-dev
+# st2 also installs
+#libssl-dev libyaml-dev ldap-utils
+
+# not sure if we still need this
+#python3-dev
+
+# deploy dependencies (maybe use pre-installed imagemagick instead)
+#gmic
+#optipng
diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
new file mode 100644
index 00000000..2c64bd3b
--- /dev/null
+++ b/gha/checkout/action.yaml
@@ -0,0 +1,53 @@
+---
+name: Checkout Pack Repo and CI Repos
+description: |
+  Configures git user and clones repositories
+  required for StackStorm-Exchange pack tests.
+author: StackStorm
+
+inputs:
+  git_user_name:
+    required: false
+    default: "StackStorm Exchange"
+  git_user_email:
+    required: false
+    default: "info@stackstorm.com"
+  st2_branch:
+    required: false
+    default: master
+  lint_configs_branch:
+    required: false
+    default: master
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure git user
+      env:
+        GIT_USER_NAME: "${{ inputs.git_user_name }}"
+        GIT_USER_EMAIL: "${{ inputs.git_user_email }}"
+      run: |
+        git config --global user.name "${GIT_USER_NAME}"
+        git config --global user.email "${GIT_USER_EMAIL}"
+
+    - name: Checkout pack repo
+      uses: actions/checkout@v2
+        path: pack
+        depth: 1
+
+    - name: Checkout st2 repo
+      # so other scripts can reference StackStorm Python code
+      uses: actions/checkout@v2
+      with:
+        repository: StackStorm/st2
+        ref: ${{ inputs.st2_branch }}
+        path: st2
+        depth: 1
+
+    - name: Checkout lint-configs repo
+      uses: actions/checkout@v2
+      with:
+        repository: StackStorm/lint-configs
+        ref: ${{ inputs.lint_configs_branch }}
+        path: lint-configs
+        depth: 1
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
new file mode 100644
index 00000000..73b6689d
--- /dev/null
+++ b/gha/py-dependencies/action.yaml
@@ -0,0 +1,102 @@
+---
+name: Install Python Dependencies
+description: |
+  Install python dependencies required for StackStorm-Exchange pack tests.
+  Before using this, make sure to run
+  StackStorm-Exchange/ci/gha/checkout and
+  StackStorm-Exchange/ci/gha/apt-dependencies.
+author: StackStorm
+
+inputs:
+  python-version:
+    required: true
+    description: Which python version we should install
+  cache-version:
+    required: false
+    default: "v0"
+
+outputs:
+  pip-version:
+    description: The installed pip version (pulled from st2.git)
+    value: ${{ steps.virtualenv.outputs.pip-version }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: 'Set up Python (${{ input.python-version }})'
+      uses: actions/setup-python@v2
+      with:
+        python-version: '${{ input.python-version }}'
+
+    - name: Cache Python Dependencies
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pip
+          ~/virtualenv
+        key: ${{ runner.os }}-python-${{ input.python-version }}-${{ input.cache-version }}-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-python-${{ input.python-version }}-${{ input.cache-version }}-
+
+    - name: Install virtualenv
+      # this should run in the st2 checkout
+      working-directory: st2
+      run: |
+        ./scripts/github/install-virtualenv.sh
+
+    - name: Create ~/virtualenv
+      id: virtualenv
+      # this should run in the st2 checkout
+      working-directory: st2
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+      run: |
+        ./scripts/github/install-virtualenv.sh
+        PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
+        echo "::set-output name=pip-version::${PIP_VERSION}"
+        ${VIRTUALENV_DIR}/bin/pip install -U "pip==${PIP_VERSION}" setuptools
+
+    - name: Install StackStorm requirements
+      # this should run in the st2 checkout
+      working-directory: st2
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+      run: |
+        ${VIRTUALENV_DIR}/bin/pip install -r requirements.txt
+
+    - name: Install runners
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        ACTION_PATH: ${{ github.action_path }}
+        ST2_REPO_PATH: st2
+      # TODO: This Makefile has CircleCI assumptions
+      run: |
+        cp ${ACTION_PATH}/../../.circle/Makefile .
+        make requirements-ci .install-runners
+
+    - name: Install pack requirements
+      working-directory: pack
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        PACK_REQUIREMENTS_FILE: requirements.txt
+        PACK_TESTS_REQUIREMENTS_FILE: requirements-tests.txt
+      run: |
+        if [[ -f "${PACK_REQUIREMENTS_FILE}" ]]; then
+        echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}"
+        ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_REQUIREMENTS_FILE}"
+        fi
+        if [[ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]]; then
+        echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}"
+        ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}"
+        fi
+
+    - name: Print versions
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+      run: |
+        set -ex
+        source ${VIRTUALENV_DIR}/activate
+        python3 --version
+        pip --version
+        virtualenv --version
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
new file mode 100644
index 00000000..e8772d0e
--- /dev/null
+++ b/gha/test/action.yaml
@@ -0,0 +1,59 @@
+---
+name: Run pack tests
+description: |
+  Run StackStorm-Exchange pack tests.
+  Before using this, make sure to run
+  StackStorm-Exchange/ci/gha/checkout,
+  StackStorm-Exchange/ci/gha/apt-dependencies, and
+  StackStorm-Exchange/ci/gha/py-dependencies.
+author: StackStorm
+
+inputs:
+  enable-common-libs:
+    description: |
+      When true, use an st2.conf that sets packs.enable_common_libs=true
+      see: https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html
+    default: false
+    required: false
+outputs:
+  pack-name:
+    description: The pack name pulled from pack.yaml
+    value: ${{ steps.pack-name.outputs.pack-name }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Get Pack Name
+      id: pack-name
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        ACTION_PATH: ${{ github.action_path }}
+      run: |
+        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
+        if [[ -z "${PACK_NAME}" ]]; then
+        echo "Unable to retrieve pack name."
+        exit 1
+        fi
+        echo "::set-output name=pack-name::${PACK_NAME}"
+
+    - name: 
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        FORCE_CHECK_ALL_FILES: true if on default branch
+        ROOT_DIR: pack directory
+        CI_DIR: ${{ github.action_path + "/../.." }}
+        ST2_REPO_PATH: st2 directory
+        ENABLE_COMMON_LIBS: ${{ input.enable-common-libs }}
+      # TODO: This Makefile has CircleCI assumptions
+      run: |
+        if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then
+        echo "Common libs PATH selected"
+        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf
+        else
+        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
+        fi
+        cp ${CI_DIR}/.circle/Makefile .
+        make -C "${ROOT_DIR}" all-ci
+
+

From f4e8164682a57bab6ba14dae0872ec458373e8cb Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Nov 2021 00:49:50 -0600
Subject: [PATCH 004/126] stub deploy composit gha action

---
 gha/deploy/action.yaml | 61 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 gha/deploy/action.yaml

diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
new file mode 100644
index 00000000..34c7f062
--- /dev/null
+++ b/gha/deploy/action.yaml
@@ -0,0 +1,61 @@
+---
+name: Deploy pack
+description: |
+  Deploy pack to StackStorm-Exchange index.
+author: StackStorm
+
+inputs: {}
+outputs:
+  pack-name:
+    description: The pack name pulled from pack.yaml
+    value: ${{ steps.pack-name.outputs.pack-name }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Get Pack Name
+      id: pack-name
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        ACTION_PATH: ${{ github.action_path }}
+      run: |
+        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
+        if [[ -z "${PACK_NAME}" ]]; then
+        echo "Unable to retrieve pack name."
+        exit 1
+        fi
+        echo "::set-output name=pack-name::${PACK_NAME}"
+
+    # checkout pack and index repos
+
+    # create version tags
+
+    # webhook that triggers a workflow in index repo to regen
+    #  - rebuild pack index directory
+    #    - convert resource metadata to json
+    #    - convert config schema to json
+    #    - add resource stats to updated pack.yaml
+    #  - rebuild index JSON
+    #  - copy and optimize icon
+
+    - name: 
+      env:
+        VIRTUALENV_DIR: "~/virtualenv"
+        FORCE_CHECK_ALL_FILES: true if on default branch
+        ROOT_DIR: pack directory
+        CI_DIR: ${{ github.action_path + "/../.." }}
+        ST2_REPO_PATH: st2 directory
+        ENABLE_COMMON_LIBS: ${{ input.enable-common-libs }}
+      # TODO: This Makefile has CircleCI assumptions
+      run: |
+        if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then
+        echo "Common libs PATH selected"
+        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf
+        else
+        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
+        fi
+        cp ${CI_DIR}/.circle/Makefile .
+        make -C "${ROOT_DIR}" all-ci
+
+

From 746b50d400765118669a8b379dff7d51c9007816 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Mon, 22 Nov 2021 10:01:38 -0700
Subject: [PATCH 005/126] gha actions: Added a shell field (opting to use the
 bash shell)

---
 gha/apt-dependencies/action.yaml | 3 +++
 gha/checkout/action.yaml         | 1 +
 gha/py-dependencies/action.yaml  | 6 ++++++
 3 files changed, 10 insertions(+)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index cdd88782..6f6ae919 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -15,6 +15,7 @@ runs:
 
     - name: Create a directory for debian packages so we can cache it
       # this is what we did on CircleCI. Not sure if it'll work on GHA
+      shell: bash
       run: >
         sudo rm -rf /var/cache/apt/archives
         && sudo ln -s ~/apt_cache /var/cache/apt/archives
@@ -34,6 +35,7 @@ runs:
           ${{ runner.os }}-apt-${{ input.cache-version }}-
 
     - name: Install APT Dependencies
+      shell: bash
       env:
         CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         APT_PACKAGES_FILE_PATH: ${{github.action_path + '/apt-packages.txt'}}
@@ -45,6 +47,7 @@ runs:
         sudo apt-get -y install ${APT_PACKAGES}
 
     - name: Print versions
+      shell: bash
       run: |
         set -ex
         jq --version
diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index 2c64bd3b..4491c20e 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -23,6 +23,7 @@ runs:
   using: "composite"
   steps:
     - name: Configure git user
+      shell: bash
       env:
         GIT_USER_NAME: "${{ inputs.git_user_name }}"
         GIT_USER_EMAIL: "${{ inputs.git_user_email }}"
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 73b6689d..7c62627d 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -40,12 +40,14 @@ runs:
           ${{ runner.os }}-python-${{ input.python-version }}-${{ input.cache-version }}-
 
     - name: Install virtualenv
+      shell: bash
       # this should run in the st2 checkout
       working-directory: st2
       run: |
         ./scripts/github/install-virtualenv.sh
 
     - name: Create ~/virtualenv
+      shell: bash
       id: virtualenv
       # this should run in the st2 checkout
       working-directory: st2
@@ -59,6 +61,7 @@ runs:
 
     - name: Install StackStorm requirements
       # this should run in the st2 checkout
+      shell: bash
       working-directory: st2
       env:
         VIRTUALENV_DIR: "~/virtualenv"
@@ -66,6 +69,7 @@ runs:
         ${VIRTUALENV_DIR}/bin/pip install -r requirements.txt
 
     - name: Install runners
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
         ACTION_PATH: ${{ github.action_path }}
@@ -76,6 +80,7 @@ runs:
         make requirements-ci .install-runners
 
     - name: Install pack requirements
+      shell: bash
       working-directory: pack
       env:
         VIRTUALENV_DIR: "~/virtualenv"
@@ -92,6 +97,7 @@ runs:
         fi
 
     - name: Print versions
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
       run: |

From 454c0ecfeee6c0a87a5b4bcae910c7fc982aa6aa Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 23 Nov 2021 16:25:44 -0500
Subject: [PATCH 006/126] gha actions: use bash shell

---
 gha/deploy/action.yaml | 2 ++
 gha/test/action.yaml   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index 34c7f062..29df457f 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -16,6 +16,7 @@ runs:
 
     - name: Get Pack Name
       id: pack-name
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
         ACTION_PATH: ${{ github.action_path }}
@@ -40,6 +41,7 @@ runs:
     #  - copy and optimize icon
 
     - name: 
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
         FORCE_CHECK_ALL_FILES: true if on default branch
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index e8772d0e..60c94532 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -26,6 +26,7 @@ runs:
 
     - name: Get Pack Name
       id: pack-name
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
         ACTION_PATH: ${{ github.action_path }}
@@ -38,6 +39,7 @@ runs:
         echo "::set-output name=pack-name::${PACK_NAME}"
 
     - name: 
+      shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"
         FORCE_CHECK_ALL_FILES: true if on default branch

From 8347b9a58d14694ff944a1fd0327b3dfa0fef94f Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Mon, 22 Nov 2021 10:34:56 -0700
Subject: [PATCH 007/126] gha actions: Changed 'input' to 'inputs'

---
 gha/apt-dependencies/action.yaml | 4 ++--
 gha/deploy/action.yaml           | 2 +-
 gha/py-dependencies/action.yaml  | 8 ++++----
 gha/test/action.yaml             | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index 6f6ae919..cda4e5d3 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -30,9 +30,9 @@ runs:
       with:
         path: |
           ~/apt_cache
-        key: ${{ runner.os }}-apt-${{ input.cache-version }}-${{ hashFiles(github.action_path + '/apt-packages.txt') }}
+        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles(github.action_path + '/apt-packages.txt') }}
         restore-keys: |
-          ${{ runner.os }}-apt-${{ input.cache-version }}-
+          ${{ runner.os }}-apt-${{ inputs.cache-version }}-
 
     - name: Install APT Dependencies
       shell: bash
diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index 29df457f..34dbdc13 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -48,7 +48,7 @@ runs:
         ROOT_DIR: pack directory
         CI_DIR: ${{ github.action_path + "/../.." }}
         ST2_REPO_PATH: st2 directory
-        ENABLE_COMMON_LIBS: ${{ input.enable-common-libs }}
+        ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
       # TODO: This Makefile has CircleCI assumptions
       run: |
         if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 7c62627d..ec0d351b 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -24,10 +24,10 @@ runs:
   using: "composite"
   steps:
 
-    - name: 'Set up Python (${{ input.python-version }})'
+    - name: 'Set up Python (${{ inputs.python-version }})'
       uses: actions/setup-python@v2
       with:
-        python-version: '${{ input.python-version }}'
+        python-version: '${{ inputs.python-version }}'
 
     - name: Cache Python Dependencies
       uses: actions/cache@v2
@@ -35,9 +35,9 @@ runs:
         path: |
           ~/.cache/pip
           ~/virtualenv
-        key: ${{ runner.os }}-python-${{ input.python-version }}-${{ input.cache-version }}-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-${{ hashFiles('requirements.txt') }}
         restore-keys: |
-          ${{ runner.os }}-python-${{ input.python-version }}-${{ input.cache-version }}-
+          ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-
 
     - name: Install virtualenv
       shell: bash
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 60c94532..f97c8e60 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -46,7 +46,7 @@ runs:
         ROOT_DIR: pack directory
         CI_DIR: ${{ github.action_path + "/../.." }}
         ST2_REPO_PATH: st2 directory
-        ENABLE_COMMON_LIBS: ${{ input.enable-common-libs }}
+        ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
       # TODO: This Makefile has CircleCI assumptions
       run: |
         if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then

From ca9c592ee6c7d2844ae12eeadacc4d28f1b564f6 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Mon, 22 Nov 2021 10:44:09 -0700
Subject: [PATCH 008/126] gha actions: Fixing a string concatenation error by
 using format()

---
 gha/apt-dependencies/action.yaml | 4 ++--
 gha/deploy/action.yaml           | 2 +-
 gha/test/action.yaml             | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index cda4e5d3..e8adbd1d 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -30,7 +30,7 @@ runs:
       with:
         path: |
           ~/apt_cache
-        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles(github.action_path + '/apt-packages.txt') }}
+        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles(format('{0}{1}', github.action_path, '/apt-packages.txt')) }}
         restore-keys: |
           ${{ runner.os }}-apt-${{ inputs.cache-version }}-
 
@@ -38,7 +38,7 @@ runs:
       shell: bash
       env:
         CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        APT_PACKAGES_FILE_PATH: ${{github.action_path + '/apt-packages.txt'}}
+        APT_PACKAGES_FILE_PATH:  ${{ format('{0}{1}', github.action_path, '/apt-packages.txt') }}
       run: |
         if [[ "$CACHE_HIT" != 'true' ]]; then
         sudo apt-get -y update
diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index 34dbdc13..26fcb347 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -46,7 +46,7 @@ runs:
         VIRTUALENV_DIR: "~/virtualenv"
         FORCE_CHECK_ALL_FILES: true if on default branch
         ROOT_DIR: pack directory
-        CI_DIR: ${{ github.action_path + "/../.." }}
+        CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}
         ST2_REPO_PATH: st2 directory
         ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
       # TODO: This Makefile has CircleCI assumptions
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index f97c8e60..07671c4f 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -44,7 +44,7 @@ runs:
         VIRTUALENV_DIR: "~/virtualenv"
         FORCE_CHECK_ALL_FILES: true if on default branch
         ROOT_DIR: pack directory
-        CI_DIR: ${{ github.action_path + "/../.." }}
+        CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}
         ST2_REPO_PATH: st2 directory
         ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
       # TODO: This Makefile has CircleCI assumptions

From 0a40f7e4abc03876735d363db7b520d492ffe695 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Tue, 23 Nov 2021 10:23:40 -0700
Subject: [PATCH 009/126] Update action.yaml

---
 gha/checkout/action.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index 4491c20e..f6bea45a 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -34,7 +34,7 @@ runs:
     - name: Checkout pack repo
       uses: actions/checkout@v2
         path: pack
-        depth: 1
+        fetch-depth: 1
 
     - name: Checkout st2 repo
       # so other scripts can reference StackStorm Python code
@@ -43,7 +43,7 @@ runs:
         repository: StackStorm/st2
         ref: ${{ inputs.st2_branch }}
         path: st2
-        depth: 1
+        fetch-depth: 1
 
     - name: Checkout lint-configs repo
       uses: actions/checkout@v2
@@ -51,4 +51,4 @@ runs:
         repository: StackStorm/lint-configs
         ref: ${{ inputs.lint_configs_branch }}
         path: lint-configs
-        depth: 1
+        fetch-depth: 1

From 995b8910d3bc4ec05e5bb41d1e03f20be02cd8dd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 23 Nov 2021 17:12:33 -0500
Subject: [PATCH 010/126] typo fix

---
 gha/checkout/action.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index f6bea45a..3587ab74 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -33,6 +33,7 @@ runs:
 
     - name: Checkout pack repo
       uses: actions/checkout@v2
+      with:
         path: pack
         fetch-depth: 1
 

From af4afdbcbcf35cca295b6bb4c54417e6344e3bcf Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 23 Nov 2021 17:12:42 -0500
Subject: [PATCH 011/126] name tests task

---
 gha/test/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 07671c4f..e7f785bd 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -38,7 +38,7 @@ runs:
         fi
         echo "::set-output name=pack-name::${PACK_NAME}"
 
-    - name: 
+    - name: Run tests
       shell: bash
       env:
         VIRTUALENV_DIR: "~/virtualenv"

From f72b97cd93e37fe31dd84bb7e776c3baab8d7c27 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Tue, 23 Nov 2021 09:19:15 -0700
Subject: [PATCH 012/126] apt-dependencies action: env var syntax

---
 gha/apt-dependencies/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index e8adbd1d..0846f0e3 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -38,9 +38,9 @@ runs:
       shell: bash
       env:
         CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        APT_PACKAGES_FILE_PATH:  ${{ format('{0}{1}', github.action_path, '/apt-packages.txt') }}
+        APT_PACKAGES_FILE_PATH: ${{ format('{0}{1}', github.action_path, '/apt-packages.txt') }}
       run: |
-        if [[ "$CACHE_HIT" != 'true' ]]; then
+        if [[ "${CACHE_HIT}" != 'true' ]]; then
         sudo apt-get -y update
         fi
         APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)

From a6c9f2ef994d8effe841b254ea36b177c4585ade Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 23 Nov 2021 17:32:13 -0500
Subject: [PATCH 013/126] gha actions: do not rely on tilde expansion for home
 var

---
 gha/deploy/action.yaml          |  8 ++++++--
 gha/py-dependencies/action.yaml | 14 +++++++++-----
 gha/test/action.yaml            |  8 ++++++--
 3 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index 26fcb347..a6cbe3d7 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -14,11 +14,15 @@ runs:
   using: "composite"
   steps:
 
+    - name: add HOME to env context
+      shell: bash
+      run: echo ::set-env name=HOME::$HOME
+
     - name: Get Pack Name
       id: pack-name
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         ACTION_PATH: ${{ github.action_path }}
       run: |
         export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
@@ -43,7 +47,7 @@ runs:
     - name: 
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         FORCE_CHECK_ALL_FILES: true if on default branch
         ROOT_DIR: pack directory
         CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index ec0d351b..9346f536 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -24,6 +24,10 @@ runs:
   using: "composite"
   steps:
 
+    - name: add HOME to env context
+      shell: bash
+      run: echo ::set-env name=HOME::$HOME
+
     - name: 'Set up Python (${{ inputs.python-version }})'
       uses: actions/setup-python@v2
       with:
@@ -52,7 +56,7 @@ runs:
       # this should run in the st2 checkout
       working-directory: st2
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         ./scripts/github/install-virtualenv.sh
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
@@ -64,14 +68,14 @@ runs:
       shell: bash
       working-directory: st2
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         ${VIRTUALENV_DIR}/bin/pip install -r requirements.txt
 
     - name: Install runners
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         ACTION_PATH: ${{ github.action_path }}
         ST2_REPO_PATH: st2
       # TODO: This Makefile has CircleCI assumptions
@@ -83,7 +87,7 @@ runs:
       shell: bash
       working-directory: pack
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         PACK_REQUIREMENTS_FILE: requirements.txt
         PACK_TESTS_REQUIREMENTS_FILE: requirements-tests.txt
       run: |
@@ -99,7 +103,7 @@ runs:
     - name: Print versions
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         set -ex
         source ${VIRTUALENV_DIR}/activate
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index e7f785bd..8d833e9b 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -24,11 +24,15 @@ runs:
   using: "composite"
   steps:
 
+    - name: add HOME to env context
+      shell: bash
+      run: echo ::set-env name=HOME::$HOME
+
     - name: Get Pack Name
       id: pack-name
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         ACTION_PATH: ${{ github.action_path }}
       run: |
         export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
@@ -41,7 +45,7 @@ runs:
     - name: Run tests
       shell: bash
       env:
-        VIRTUALENV_DIR: "~/virtualenv"
+        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         FORCE_CHECK_ALL_FILES: true if on default branch
         ROOT_DIR: pack directory
         CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}

From b6dbe8c2d2cfbfd21d9ba53ab461275126782ff1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 23 Nov 2021 17:34:37 -0500
Subject: [PATCH 014/126] gha actions: simplifiy GITHUB_ACTION_PATH usage

---
 gha/deploy/action.yaml          | 3 +--
 gha/py-dependencies/action.yaml | 3 +--
 gha/test/action.yaml            | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index a6cbe3d7..a85e9a2c 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -23,9 +23,8 @@ runs:
       shell: bash
       env:
         VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        ACTION_PATH: ${{ github.action_path }}
       run: |
-        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
+        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${GITHUB_ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
         if [[ -z "${PACK_NAME}" ]]; then
         echo "Unable to retrieve pack name."
         exit 1
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 9346f536..713ffddd 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -76,11 +76,10 @@ runs:
       shell: bash
       env:
         VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        ACTION_PATH: ${{ github.action_path }}
         ST2_REPO_PATH: st2
       # TODO: This Makefile has CircleCI assumptions
       run: |
-        cp ${ACTION_PATH}/../../.circle/Makefile .
+        cp ${GITHUB_ACTION_PATH}/../../.circle/Makefile .
         make requirements-ci .install-runners
 
     - name: Install pack requirements
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 8d833e9b..d5ccade6 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -33,9 +33,8 @@ runs:
       shell: bash
       env:
         VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        ACTION_PATH: ${{ github.action_path }}
       run: |
-        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
+        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${GITHUB_ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
         if [[ -z "${PACK_NAME}" ]]; then
         echo "Unable to retrieve pack name."
         exit 1

From df88fc7be6b1df2ac771145e300bf16ffbe5ae9a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 16:57:17 -0500
Subject: [PATCH 015/126] use gha env file instead of set-env command

---
 gha/deploy/action.yaml          | 2 +-
 gha/py-dependencies/action.yaml | 2 +-
 gha/test/action.yaml            | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/gha/deploy/action.yaml b/gha/deploy/action.yaml
index a85e9a2c..122fef17 100644
--- a/gha/deploy/action.yaml
+++ b/gha/deploy/action.yaml
@@ -16,7 +16,7 @@ runs:
 
     - name: add HOME to env context
       shell: bash
-      run: echo ::set-env name=HOME::$HOME
+      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
 
     - name: Get Pack Name
       id: pack-name
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 713ffddd..f42df391 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -26,7 +26,7 @@ runs:
 
     - name: add HOME to env context
       shell: bash
-      run: echo ::set-env name=HOME::$HOME
+      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
 
     - name: 'Set up Python (${{ inputs.python-version }})'
       uses: actions/setup-python@v2
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index d5ccade6..36da8c45 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -26,7 +26,7 @@ runs:
 
     - name: add HOME to env context
       shell: bash
-      run: echo ::set-env name=HOME::$HOME
+      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
 
     - name: Get Pack Name
       id: pack-name

From 69d8526f98c9fd42203eacdeac550885b65965c6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 17:06:34 -0500
Subject: [PATCH 016/126] gha: fix apt-dependencies cache hashFiles

---
 gha/apt-dependencies/action.yaml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index 0846f0e3..35c3e89b 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -21,8 +21,12 @@ runs:
         && sudo ln -s ~/apt_cache /var/cache/apt/archives
         && mkdir -p ~/apt_cache/partial
 
-    # TODO: install and cache apt-packages defined by packs
-    #       maybe via input?
+    # hashFiles only reads files relative to GITHUB_WORKSPACE
+    - name: Copy apt-packages.txt for caching
+      shell: bash
+      run: |
+        cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
+      # TODO: append pack-defined .github/apt-packages.txt to ${GITHUB_WORKSPACE}/apt-packages.txt
 
     - name: Cache APT Dependencies
       id: cache-apt-deps
@@ -30,7 +34,7 @@ runs:
       with:
         path: |
           ~/apt_cache
-        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles(format('{0}{1}', github.action_path, '/apt-packages.txt')) }}
+        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles('apt-packages.txt') }}
         restore-keys: |
           ${{ runner.os }}-apt-${{ inputs.cache-version }}-
 
@@ -38,7 +42,7 @@ runs:
       shell: bash
       env:
         CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
-        APT_PACKAGES_FILE_PATH: ${{ format('{0}{1}', github.action_path, '/apt-packages.txt') }}
+        APT_PACKAGES_FILE_PATH: apt-packages.txt
       run: |
         if [[ "${CACHE_HIT}" != 'true' ]]; then
         sudo apt-get -y update

From acf6496143540b64180f1dab2c6585627a1fbbc0 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 18:11:05 -0500
Subject: [PATCH 017/126] gha: narrow apt cache contents and quiet apt output

---
 gha/apt-dependencies/action.yaml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index 35c3e89b..ad7d3ab0 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -33,22 +33,23 @@ runs:
       uses: actions/cache@v2
       with:
         path: |
-          ~/apt_cache
-        key: ${{ runner.os }}-apt-${{ inputs.cache-version }}-${{ hashFiles('apt-packages.txt') }}
+          ~/apt_cache/*.deb
+        key: ${{ inputs.cache-version }}-apt-${{ hashFiles('apt-packages.txt') }}
         restore-keys: |
-          ${{ runner.os }}-apt-${{ inputs.cache-version }}-
+          ${{ inputs.cache-version }}-apt-
 
     - name: Install APT Dependencies
       shell: bash
       env:
         CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
         APT_PACKAGES_FILE_PATH: apt-packages.txt
+        DEBIAN_FRONTEND: noninteractive
       run: |
         if [[ "${CACHE_HIT}" != 'true' ]]; then
-        sudo apt-get -y update
+        sudo apt-get -o=Dpkg::Use-Pty=0 -yq update
         fi
         APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)
-        sudo apt-get -y install ${APT_PACKAGES}
+        sudo apt-get -o=Dpkg::Use-Pty=0 -yq install ${APT_PACKAGES}
 
     - name: Print versions
       shell: bash

From 964f9c8ed1178a5aa71d84c875f1eff621c05688 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 20:24:16 -0500
Subject: [PATCH 018/126] gha: fix py cache hashFiles

---
 gha/py-dependencies/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index f42df391..15bef2e9 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -39,7 +39,7 @@ runs:
         path: |
           ~/.cache/pip
           ~/virtualenv
-        key: ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-${{ hashFiles('st2/requirements.txt', 'pack/requirements*.txt') }}
         restore-keys: |
           ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-
 

From 8bd1ce328cb3a2c1d04bc02d910eae136504b21b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 20:36:55 -0500
Subject: [PATCH 019/126] create virtualenv correctly

---
 gha/py-dependencies/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 15bef2e9..eb0c977f 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -58,9 +58,9 @@ runs:
       env:
         VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
-        ./scripts/github/install-virtualenv.sh
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
         echo "::set-output name=pip-version::${PIP_VERSION}"
+        [[ -x "${VIRTUALENV_DIR}/bin/python" ]] || virtualenv --pip "${PIP_VERSION}" ~/virtualenv
         ${VIRTUALENV_DIR}/bin/pip install -U "pip==${PIP_VERSION}" setuptools
 
     - name: Install StackStorm requirements

From 0ea192d4e803558eeea4608b6bbf9e7503b38677 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 21:19:20 -0500
Subject: [PATCH 020/126] gha: use env file to add path vars

---
 gha/checkout/action.yaml        |  8 ++++++++
 gha/py-dependencies/action.yaml | 15 +++------------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index 3587ab74..695f8d43 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -22,6 +22,14 @@ inputs:
 runs:
   using: "composite"
   steps:
+
+    - name: Add checkout paths to env context
+      shell: bash
+      run: |
+        echo "ST2_REPO_PATH=${{ github.workspace }}/st2" >> ${GITHUB_ENV}
+        echo "ROOT_DIR=${{ github.workspace }}/pack" >> ${GITHUB_ENV}
+        echo "CI_DIR=$(realpath ${{ github.action_path }}/../..)" >> ${GITHUB_ENV}
+
     - name: Configure git user
       shell: bash
       env:
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index eb0c977f..8c0b7640 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -24,9 +24,10 @@ runs:
   using: "composite"
   steps:
 
-    - name: add HOME to env context
+    - name: Add VIRTUALENV_DIR to env context
       shell: bash
-      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
+      run: |
+        echo "VIRTUALENV_DIR=${HOME}/virtualenv" >> ${GITHUB_ENV}
 
     - name: 'Set up Python (${{ inputs.python-version }})'
       uses: actions/setup-python@v2
@@ -55,8 +56,6 @@ runs:
       id: virtualenv
       # this should run in the st2 checkout
       working-directory: st2
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
         echo "::set-output name=pip-version::${PIP_VERSION}"
@@ -67,16 +66,11 @@ runs:
       # this should run in the st2 checkout
       shell: bash
       working-directory: st2
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         ${VIRTUALENV_DIR}/bin/pip install -r requirements.txt
 
     - name: Install runners
       shell: bash
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        ST2_REPO_PATH: st2
       # TODO: This Makefile has CircleCI assumptions
       run: |
         cp ${GITHUB_ACTION_PATH}/../../.circle/Makefile .
@@ -86,7 +80,6 @@ runs:
       shell: bash
       working-directory: pack
       env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
         PACK_REQUIREMENTS_FILE: requirements.txt
         PACK_TESTS_REQUIREMENTS_FILE: requirements-tests.txt
       run: |
@@ -101,8 +94,6 @@ runs:
 
     - name: Print versions
       shell: bash
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
       run: |
         set -ex
         source ${VIRTUALENV_DIR}/activate

From f2017412768f0906e870711a4141a46a84445138 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 21:32:37 -0500
Subject: [PATCH 021/126] gha: move requirements-ci from Makefile to gha
 workflow

---
 gha/py-dependencies/action.yaml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 8c0b7640..46787ed0 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -57,10 +57,14 @@ runs:
       # this should run in the st2 checkout
       working-directory: st2
       run: |
+        echo "==================== virtualenv ===================="
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
         echo "::set-output name=pip-version::${PIP_VERSION}"
         [[ -x "${VIRTUALENV_DIR}/bin/python" ]] || virtualenv --pip "${PIP_VERSION}" ~/virtualenv
-        ${VIRTUALENV_DIR}/bin/pip install -U "pip==${PIP_VERSION}" setuptools
+        ${VIRTUALENV_DIR}/bin/pip install -q -U "pip==${PIP_VERSION}" setuptools
+        echo "==================== requirements-ci ===================="
+        ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-dev.txt
+        ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-pack-tests.txt
 
     - name: Install StackStorm requirements
       # this should run in the st2 checkout
@@ -73,8 +77,8 @@ runs:
       shell: bash
       # TODO: This Makefile has CircleCI assumptions
       run: |
-        cp ${GITHUB_ACTION_PATH}/../../.circle/Makefile .
-        make requirements-ci .install-runners
+        cp ${CI_DIR}/.circle/Makefile .
+        make .install-runners
 
     - name: Install pack requirements
       shell: bash

From 4f6277e962eb430f9ec4aca85fe0e83ba8c1a2a8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 21:53:57 -0500
Subject: [PATCH 022/126] gha: inline relevant Makefile section

---
 gha/py-dependencies/action.yaml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 46787ed0..47ec5730 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -57,12 +57,12 @@ runs:
       # this should run in the st2 checkout
       working-directory: st2
       run: |
-        echo "==================== virtualenv ===================="
+        printf "\n==================== virtualenv ====================\n\n"
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
         echo "::set-output name=pip-version::${PIP_VERSION}"
         [[ -x "${VIRTUALENV_DIR}/bin/python" ]] || virtualenv --pip "${PIP_VERSION}" ~/virtualenv
         ${VIRTUALENV_DIR}/bin/pip install -q -U "pip==${PIP_VERSION}" setuptools
-        echo "==================== requirements-ci ===================="
+        printf "\n==================== requirements-ci ====================\n\n"
         ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-dev.txt
         ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-pack-tests.txt
 
@@ -75,10 +75,17 @@ runs:
 
     - name: Install runners
       shell: bash
-      # TODO: This Makefile has CircleCI assumptions
       run: |
-        cp ${CI_DIR}/.circle/Makefile .
-        make .install-runners
+        printf "\n================== install runners ====================\n\n"
+        for runner in ${ST2_REPO_PATH}/contrib/runners/*; do
+        echo "==========================================================="
+        echo "Installing runner:" ${runner##*/}
+        echo "==========================================================="
+        (. ${VIRTUALENV_DIR}/bin/activate; cd $runner; python setup.py develop)
+        done
+        printf "\n================== register metrics drivers ======================\n\n"
+        # Install st2common to register metrics drivers
+        (. ${VIRTUALENV_DIR}/bin/activate; cd ${ST2_REPO_PATH}/st2common; python setup.py develop)
 
     - name: Install pack requirements
       shell: bash

From b234e9c9b5bf3f061142e7c344b20b38b13d0849 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 22:06:57 -0500
Subject: [PATCH 023/126] gha: fix py deps print versions step

---
 gha/py-dependencies/action.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 47ec5730..5b17698e 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -106,8 +106,7 @@ runs:
     - name: Print versions
       shell: bash
       run: |
-        set -ex
-        source ${VIRTUALENV_DIR}/activate
+        source ${VIRTUALENV_DIR}/bin/activate
         python3 --version
         pip --version
         virtualenv --version

From 5ac9d522ab9792f3d28d4d5604774587e23a4460 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 22:08:51 -0500
Subject: [PATCH 024/126] gha: less verbose pip install

---
 gha/py-dependencies/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 5b17698e..112be074 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -71,7 +71,7 @@ runs:
       shell: bash
       working-directory: st2
       run: |
-        ${VIRTUALENV_DIR}/bin/pip install -r requirements.txt
+        ${VIRTUALENV_DIR}/bin/pip install -q -r requirements.txt
 
     - name: Install runners
       shell: bash

From 93fe56d7846e884b58ac2439ddba9818ed5a68e4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 22:23:26 -0500
Subject: [PATCH 025/126] gha: use native gha output grouping/folding

---
 gha/apt-dependencies/action.yaml |  4 ++++
 gha/py-dependencies/action.yaml  | 41 ++++++++++++++++++++++----------
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/gha/apt-dependencies/action.yaml b/gha/apt-dependencies/action.yaml
index ad7d3ab0..d2ecd687 100644
--- a/gha/apt-dependencies/action.yaml
+++ b/gha/apt-dependencies/action.yaml
@@ -45,15 +45,19 @@ runs:
         APT_PACKAGES_FILE_PATH: apt-packages.txt
         DEBIAN_FRONTEND: noninteractive
       run: |
+        echo "::group::Install APT Dependencies"
         if [[ "${CACHE_HIT}" != 'true' ]]; then
         sudo apt-get -o=Dpkg::Use-Pty=0 -yq update
         fi
         APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)
         sudo apt-get -o=Dpkg::Use-Pty=0 -yq install ${APT_PACKAGES}
+        echo "::endgroup::"
 
     - name: Print versions
       shell: bash
       run: |
+        echo "::group::Print Versions"
         set -ex
         jq --version
         gh --version
+        echo "::endgroup::"
diff --git a/gha/py-dependencies/action.yaml b/gha/py-dependencies/action.yaml
index 112be074..6e743b2c 100644
--- a/gha/py-dependencies/action.yaml
+++ b/gha/py-dependencies/action.yaml
@@ -44,12 +44,14 @@ runs:
         restore-keys: |
           ${{ runner.os }}-python-${{ inputs.python-version }}-${{ inputs.cache-version }}-
 
-    - name: Install virtualenv
+    - name: Install virtualenv command
       shell: bash
       # this should run in the st2 checkout
       working-directory: st2
       run: |
+        echo "::group::Install virtualenv command"
         ./scripts/github/install-virtualenv.sh
+        echo "::endgroup::"
 
     - name: Create ~/virtualenv
       shell: bash
@@ -57,43 +59,55 @@ runs:
       # this should run in the st2 checkout
       working-directory: st2
       run: |
-        printf "\n==================== virtualenv ====================\n\n"
+        echo "::group::Create ~/virtualenv"
         PIP_VERSION=$(grep '^PIP_VERSION' Makefile | awk '{print $3}')
         echo "::set-output name=pip-version::${PIP_VERSION}"
         [[ -x "${VIRTUALENV_DIR}/bin/python" ]] || virtualenv --pip "${PIP_VERSION}" ~/virtualenv
         ${VIRTUALENV_DIR}/bin/pip install -q -U "pip==${PIP_VERSION}" setuptools
-        printf "\n==================== requirements-ci ====================\n\n"
+        echo "::endgroup::"
+
+    - name: Install CI Requirements
+      shell: bash
+      run: |
+        echo "::group::Install CI Requirements"
         ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-dev.txt
         ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-pack-tests.txt
+        echo "::endgroup::"
 
-    - name: Install StackStorm requirements
+    - name: Install StackStorm Requirements
       # this should run in the st2 checkout
       shell: bash
       working-directory: st2
       run: |
+        echo "::group::Install StackStorm Requirements"
         ${VIRTUALENV_DIR}/bin/pip install -q -r requirements.txt
+        echo "::endgroup::"
 
-    - name: Install runners
+    - name: Install Runners
       shell: bash
       run: |
-        printf "\n================== install runners ====================\n\n"
         for runner in ${ST2_REPO_PATH}/contrib/runners/*; do
-        echo "==========================================================="
-        echo "Installing runner:" ${runner##*/}
-        echo "==========================================================="
+        echo "::group::Install Runner: ${runner##*/}"
         (. ${VIRTUALENV_DIR}/bin/activate; cd $runner; python setup.py develop)
+        echo "::endgroup::"
         done
-        printf "\n================== register metrics drivers ======================\n\n"
+
+    - name: Register Metrics Drivers
+      shell: bash
+      run: |
+        echo "::group::Register Metrics Drivers"
         # Install st2common to register metrics drivers
         (. ${VIRTUALENV_DIR}/bin/activate; cd ${ST2_REPO_PATH}/st2common; python setup.py develop)
+        echo "::endgroup::"
 
-    - name: Install pack requirements
+    - name: Install Pack Requirements
       shell: bash
       working-directory: pack
       env:
         PACK_REQUIREMENTS_FILE: requirements.txt
         PACK_TESTS_REQUIREMENTS_FILE: requirements-tests.txt
       run: |
+        echo "::group::Install Pack Requirements"
         if [[ -f "${PACK_REQUIREMENTS_FILE}" ]]; then
         echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}"
         ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_REQUIREMENTS_FILE}"
@@ -102,11 +116,14 @@ runs:
         echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}"
         ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}"
         fi
+        echo "::endgroup::"
 
-    - name: Print versions
+    - name: Print Versions
       shell: bash
       run: |
+        echo "::group::Print Versions"
         source ${VIRTUALENV_DIR}/bin/activate
         python3 --version
         pip --version
         virtualenv --version
+        echo "::endgroup::"

From 481e0db9f602ecd88a4601b6b6a34846a1960730 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 22:31:59 -0500
Subject: [PATCH 026/126] gha: update env var usage in tests action

---
 gha/test/action.yaml | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 36da8c45..c75da4b4 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -24,31 +24,28 @@ runs:
   using: "composite"
   steps:
 
-    - name: add HOME to env context
-      shell: bash
-      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
-
     - name: Get Pack Name
       id: pack-name
       shell: bash
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
+      working-directory: pack
       run: |
-        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${GITHUB_ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
+        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${CI_DIR}/.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
         if [[ -z "${PACK_NAME}" ]]; then
         echo "Unable to retrieve pack name."
         exit 1
         fi
         echo "::set-output name=pack-name::${PACK_NAME}"
 
+    - name: Add CI vars to env context
+      shell: bash
+      run: |
+        # TODO: FORCE_CHECK_ALL_FILES should only be true if on default branch
+        echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
+        echo "PACK_NAME=${{ steps.pack-name.outputs.pack-name }}" >> ${GITHUB_ENV}
+
     - name: Run tests
       shell: bash
       env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        FORCE_CHECK_ALL_FILES: true if on default branch
-        ROOT_DIR: pack directory
-        CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}
-        ST2_REPO_PATH: st2 directory
         ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
       # TODO: This Makefile has CircleCI assumptions
       run: |

From a1f7f9d2e4998bdb13e784d3521fa37888e9e676 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 23:41:57 -0500
Subject: [PATCH 027/126] gha: tell make where to find Makefile

---
 gha/test/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index c75da4b4..ee7083d0 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -56,6 +56,6 @@ runs:
         export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
         fi
         cp ${CI_DIR}/.circle/Makefile .
-        make -C "${ROOT_DIR}" all-ci
+        make -C "${ROOT_DIR}" -f "${GITHUB_WORKSPACE}/Makefile" all-ci
 
 

From 9477db36b8b3de591156ce8b5550d8d0ef1cc123 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 25 Nov 2021 23:59:07 -0500
Subject: [PATCH 028/126] gha: lookup default branch

---
 gha/test/action.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index ee7083d0..10865378 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -42,6 +42,9 @@ runs:
         # TODO: FORCE_CHECK_ALL_FILES should only be true if on default branch
         echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
         echo "PACK_NAME=${{ steps.pack-name.outputs.pack-name }}" >> ${GITHUB_ENV}
+        source ${CI_DIR}/tools/functions.sh
+        cd pack
+        echo "BASE_BRANCH=origin/$(_gh_default_branch)" >> ${GITHUB_ENV}
 
     - name: Run tests
       shell: bash

From c34524e4d63fd69ae2d5faa9e840797bc755732e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 00:18:05 -0500
Subject: [PATCH 029/126] gha: add GH_TOKEN env var

---
 gha/test/action.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 10865378..c0c5afd4 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -38,6 +38,8 @@ runs:
 
     - name: Add CI vars to env context
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
       run: |
         # TODO: FORCE_CHECK_ALL_FILES should only be true if on default branch
         echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}

From b640c6e2e09ead8d73823cdcbed424854a50ab0c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 00:36:49 -0500
Subject: [PATCH 030/126] gha: set FORCE_CHECK_ALL_FILES correctly

---
 gha/test/action.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index c0c5afd4..df85962e 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -41,12 +41,14 @@ runs:
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        # TODO: FORCE_CHECK_ALL_FILES should only be true if on default branch
-        echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
         echo "PACK_NAME=${{ steps.pack-name.outputs.pack-name }}" >> ${GITHUB_ENV}
         source ${CI_DIR}/tools/functions.sh
         cd pack
-        echo "BASE_BRANCH=origin/$(_gh_default_branch)" >> ${GITHUB_ENV}
+        DEFAULT_BRANCH=$(_gh_default_branch)
+        echo "BASE_BRANCH=origin/${DEFAULT_BRANCH}" >> ${GITHUB_ENV}
+        if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
+        echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
+        fi
 
     - name: Run tests
       shell: bash

From f141e8e0f75364f540ba53b8c56236c6aee0ecea Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 00:41:12 -0500
Subject: [PATCH 031/126] gha: consolidate env var definition

---
 gha/test/action.yaml | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index df85962e..9a12e92a 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -49,19 +49,17 @@ runs:
         if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
         echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
         fi
-
-    - name: Run tests
-      shell: bash
-      env:
-        ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
-      # TODO: This Makefile has CircleCI assumptions
-      run: |
-        if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then
+        if [[ "true" == "${{ inputs.enable-common-libs }}" ]]; then
         echo "Common libs PATH selected"
         export ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf
         else
         export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
         fi
+
+    - name: Run tests
+      shell: bash
+      # TODO: This Makefile has CircleCI assumptions
+      run: |
         cp ${CI_DIR}/.circle/Makefile .
         make -C "${ROOT_DIR}" -f "${GITHUB_WORKSPACE}/Makefile" all-ci
 

From 6f200689c3f6f448db089c8a9e310a10016ae132 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 10:43:39 -0500
Subject: [PATCH 032/126] gha: save env vars correctly

---
 gha/test/action.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 9a12e92a..7231ba96 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -40,10 +40,10 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+      working-directory: pack
       run: |
         echo "PACK_NAME=${{ steps.pack-name.outputs.pack-name }}" >> ${GITHUB_ENV}
         source ${CI_DIR}/tools/functions.sh
-        cd pack
         DEFAULT_BRANCH=$(_gh_default_branch)
         echo "BASE_BRANCH=origin/${DEFAULT_BRANCH}" >> ${GITHUB_ENV}
         if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
@@ -51,9 +51,9 @@ runs:
         fi
         if [[ "true" == "${{ inputs.enable-common-libs }}" ]]; then
         echo "Common libs PATH selected"
-        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf
+        echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf" >> ${GITHUB_ENV}
         else
-        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
+        echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf" >> ${GITHUB_ENV}
         fi
 
     - name: Run tests

From 0f4ca88d93ff29573182de1428871abd128fd93a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 00:46:54 -0500
Subject: [PATCH 033/126] gha: pack clone should include all branches and tags

---
 gha/checkout/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index 695f8d43..5b93c5dd 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -43,7 +43,7 @@ runs:
       uses: actions/checkout@v2
       with:
         path: pack
-        fetch-depth: 1
+        fetch-depth: 0
 
     - name: Checkout st2 repo
       # so other scripts can reference StackStorm Python code

From 27133b6b68cb7c0018f83c64cb0557aeacc2bde3 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 11:52:08 -0500
Subject: [PATCH 034/126] gha: use virtualenv for everything in test

---
 gha/test/action.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 7231ba96..2c7db2ee 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -61,6 +61,7 @@ runs:
       # TODO: This Makefile has CircleCI assumptions
       run: |
         cp ${CI_DIR}/.circle/Makefile .
+        source ${VIRTUALENV_DIR}/bin/activate
         make -C "${ROOT_DIR}" -f "${GITHUB_WORKSPACE}/Makefile" all-ci
 
 

From 5f476c0630f3103946a5e70912b9453367e4a38d Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 12:03:42 -0500
Subject: [PATCH 035/126] adjust Makefile to not hardcode ST2_REPO_PATH

---
 .circle/Makefile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.circle/Makefile b/.circle/Makefile
index 315c7d49..c191df63 100644
--- a/.circle/Makefile
+++ b/.circle/Makefile
@@ -13,8 +13,8 @@ FORCE_CHECK_PACK ?= false
 export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK
 
 # All components are prefixed by st2
-COMPONENTS := $(wildcard /tmp/st2/st2*)
-COMPONENTS_RUNNERS := $(wildcard /tmp/st2/contrib/runners/*)
+COMPONENTS := $(wildcard $(ST2_REPO_PATH)/st2*)
+COMPONENTS_RUNNERS := $(wildcard $(ST2_REPO_PATH)/contrib/runners/*)
 
 .PHONY: all
 all: requirements lint packs-resource-register packs-tests
@@ -252,13 +252,12 @@ compile:
 	fi
 
 .PHONY: .clone_st2_repo
-.clone_st2_repo: /tmp/st2
-/tmp/st2:
+.clone_st2_repo:
 	@echo
 	@echo "==================== cloning st2 repo ===================="
 	@echo
-	@rm -rf /tmp/st2
-	@git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) /tmp/st2
+	@rm -rf $(ST2_REPO_PATH)
+	@git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) $(ST2_REPO_PATH)
 
 .PHONY: .install-runners
 .install-runners:

From fd8f0756f3d611e4c22e02dee12dbac337679c6e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 12:07:45 -0500
Subject: [PATCH 036/126] adjust Makefile to not hardcode /home/circleci

---
 .circle/Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circle/Makefile b/.circle/Makefile
index c191df63..eef842d0 100644
--- a/.circle/Makefile
+++ b/.circle/Makefile
@@ -102,7 +102,7 @@ compile:
 	. $(VIRTUALENV_DIR)/bin/activate; \
 	if [ "${COMMON_LIBS}" = "true" ]; then \
 		echo "Common libs PATH selected"; \
-		export PYTHONPATH=/home/circleci/repo/lib:${PYTHONPATH}; \
+		export PYTHONPATH=$(ROOT_DIR)/lib:${PYTHONPATH}; \
 	fi; \
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \
 		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
@@ -237,8 +237,8 @@ compile:
 			echo "Missing LICENSE file in $(ROOT_DIR)"; \
 			exit 2;\
 		fi;\
-		if [ "$$(python -c 'import yaml; f = open("/home/circleci/repo/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "napalm_logs" ] || \
-		   [ "$$(python -c 'import yaml; f = open("/home/circleci/repo/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "netbox" ]; then \
+		if [ "$$(python -c 'import yaml; f = open("$(ROOT_DIR)/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "napalm_logs" ] || \
+		   [ "$$(python -c 'import yaml; f = open("$(ROOT_DIR)/pack.yaml"); print(yaml.safe_load(f.read())["name"]); f.close();')" = "netbox" ]; then \
 			cat $(ROOT_DIR)/LICENSE | grep -q "MIT License"  || (echo "LICENSE file doesn't contain MIT license text" ; exit 2); \
 			cat $(ROOT_DIR)/LICENSE | grep -q "Copyright (c) 2017 John Anderson"  || (echo "LICENSE file doesn't include John Anderson's name" ; exit 2); \
 			cat $(ROOT_DIR)/LICENSE | grep -q "Permission is hereby granted, free of charge, to any person obtaining a copy"  || (echo "LICENSE file doesn't contain MIT license text" ; exit 2); \

From 8fce6ef6c63786abfe897bedfa40494b2909c40b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 12:11:04 -0500
Subject: [PATCH 037/126] gha: temporarily enable FORCE_CHECK_ALL_FILES for all
 branches

---
 gha/test/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 2c7db2ee..67e8d4c2 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -46,9 +46,9 @@ runs:
         source ${CI_DIR}/tools/functions.sh
         DEFAULT_BRANCH=$(_gh_default_branch)
         echo "BASE_BRANCH=origin/${DEFAULT_BRANCH}" >> ${GITHUB_ENV}
-        if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
+        #if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
         echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
-        fi
+        #fi
         if [[ "true" == "${{ inputs.enable-common-libs }}" ]]; then
         echo "Common libs PATH selected"
         echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf" >> ${GITHUB_ENV}

From 1b08e2d0fdc43f7cb36627e84e85a9c811ed9ce1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 12:24:10 -0500
Subject: [PATCH 038/126] gha: define LINT_CONFIGS_PATH

---
 .circle/Makefile         | 7 ++++---
 gha/checkout/action.yaml | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.circle/Makefile b/.circle/Makefile
index eef842d0..3f0365a4 100644
--- a/.circle/Makefile
+++ b/.circle/Makefile
@@ -9,6 +9,7 @@ ST2_REPO_PATH ?= /tmp/st2
 ST2_REPO_BRANCH ?= master
 FORCE_CHECK_ALL_FILES ?= false
 FORCE_CHECK_PACK ?= false
+LINT_CONFIGS_PATH ?= $(CI_DIR)/lint-configs
 
 export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK
 
@@ -79,14 +80,14 @@ compile:
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
 		echo "Force flake8 checks on all files"; \
 		find $(ROOT_DIR)/* -name "*.py" | while read py_file; do \
-			flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
+			flake8 --config=$(LINT_CONFIGS_PATH)/python/.flake8 $$py_file || exit 1; \
 		done; \
 	elif [ ${NUM_CHANGED_PY} -gt 0 ]; then \
 		echo "Checking ${NUM_CHANGED_PY} Python files"; \
 		$(CI_DIR)/utils/git-changes py >~/.git-changes-py; \
 		while read -r file; do \
 			if [ -n "$$file" ]; then \
-				flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
+				flake8 --config=$(LINT_CONFIGS_PATH)/python/.flake8 $$file || exit 1; \
 			fi; \
 		done < ~/.git-changes-py; \
 		if [ -e ~/.git-changes-py ]; then rm ~/.git-changes-py; fi; \
@@ -106,7 +107,7 @@ compile:
 	fi; \
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \
 		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
-		CONFIG_DIR=$(CI_DIR)/lint-configs/ \
+		CONFIG_DIR=$(LINT_CONFIGS_PATH)/ \
 		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
 	else \
 		echo "No files have changed, skipping run..."; \
diff --git a/gha/checkout/action.yaml b/gha/checkout/action.yaml
index 5b93c5dd..85a3ef7c 100644
--- a/gha/checkout/action.yaml
+++ b/gha/checkout/action.yaml
@@ -29,6 +29,7 @@ runs:
         echo "ST2_REPO_PATH=${{ github.workspace }}/st2" >> ${GITHUB_ENV}
         echo "ROOT_DIR=${{ github.workspace }}/pack" >> ${GITHUB_ENV}
         echo "CI_DIR=$(realpath ${{ github.action_path }}/../..)" >> ${GITHUB_ENV}
+        echo "LINT_CONFIGS_PATH=${{ github.workspace }}/lint-configs" >> ${GITHUB_ENV}
 
     - name: Configure git user
       shell: bash

From 9611b29e8c9c50384e52a667e3d773a57b843d8a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 12:42:15 -0500
Subject: [PATCH 039/126] gha: simplify Makefile usage

---
 gha/test/action.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 67e8d4c2..261fb7fd 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -60,8 +60,7 @@ runs:
       shell: bash
       # TODO: This Makefile has CircleCI assumptions
       run: |
-        cp ${CI_DIR}/.circle/Makefile .
         source ${VIRTUALENV_DIR}/bin/activate
-        make -C "${ROOT_DIR}" -f "${GITHUB_WORKSPACE}/Makefile" all-ci
+        make -C "${ROOT_DIR}" -f "${CI_DIR}/.circle/Makefile" all-ci
 
 

From a5e0bb7c69a93f909b9695668001deb48b840e8a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 13:00:54 -0500
Subject: [PATCH 040/126] gha: add services for tests

---
 gha/test/action.yaml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 261fb7fd..5d2be91b 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -58,9 +58,22 @@ runs:
 
     - name: Run tests
       shell: bash
+      # NB: some tests require mongo and/or rabbitmq (see services: below).
       # TODO: This Makefile has CircleCI assumptions
       run: |
         source ${VIRTUALENV_DIR}/bin/activate
         make -C "${ROOT_DIR}" -f "${CI_DIR}/.circle/Makefile" all-ci
 
-
+  services:
+    mongo:
+      image: mongo:3.4
+      ports:
+        - 27017:27017
+    rabbitmq:
+      image: rabbitmq:3
+      ports:
+        - 5672:5672
+    redis:
+      image: redis
+      ports:
+        - 6379:6379

From 0b5695cd93a04c67f7bca01e8f73db8293d32d5f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 15:37:31 -0500
Subject: [PATCH 041/126] gha: add reusable pack-test workflow

---
 gha/pack-build_and_test.yaml | 58 ++++++++++++++++++++++++++++++++++++
 gha/test/action.yaml         | 16 +---------
 2 files changed, 59 insertions(+), 15 deletions(-)
 create mode 100644 gha/pack-build_and_test.yaml

diff --git a/gha/pack-build_and_test.yaml b/gha/pack-build_and_test.yaml
new file mode 100644
index 00000000..6b17792c
--- /dev/null
+++ b/gha/pack-build_and_test.yaml
@@ -0,0 +1,58 @@
+name: CI - Build and Test
+
+on:
+  workflow_call:
+    inputs:
+      apt-cache-version:
+        required: false
+        type: string
+        default: "v0"
+      py-cache-version:
+        required: false
+        type: string
+        default: "v0"
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    name: 'Build and Test - Python ${{ matrix.python-version-short }}'
+    strategy:
+      matrix:
+        include:
+          - python-version-short: 3.6
+            python-version: 3.6.13
+    steps:
+      # eventually replace @gha with @master
+      - name: Checkout Pack Repo and CI Repos
+        uses: StackStorm-Exchange/ci/gha/checkout@gha
+        
+      - name: Install APT Dependencies
+        uses: StackStorm-Exchange/ci/gha/apt-dependencies@gha
+        with:
+          cache-version: ${{ inputs.apt-cache-version }}
+
+      - name: Install Python Dependencies
+        uses: StackStorm-Exchange/ci/gha/py-dependencies@gha
+        with:
+          cache-version: ${{ inputs.py-cache-version }}
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run pack tests
+        uses: StackStorm-Exchange/ci/gha/test@gha
+        with:
+          # This makes the tests use an alternate config that enables shared libs
+          enable-common-libs: true
+
+    services:
+      mongo:
+        image: mongo:3.4
+        ports:
+          - 27017:27017
+      rabbitmq:
+        image: rabbitmq:3
+        ports:
+          - 5672:5672
+      #redis:
+      #  image: redis
+      #  ports:
+      #    - 6379:6379
diff --git a/gha/test/action.yaml b/gha/test/action.yaml
index 5d2be91b..aab4c830 100644
--- a/gha/test/action.yaml
+++ b/gha/test/action.yaml
@@ -58,22 +58,8 @@ runs:
 
     - name: Run tests
       shell: bash
-      # NB: some tests require mongo and/or rabbitmq (see services: below).
       # TODO: This Makefile has CircleCI assumptions
       run: |
         source ${VIRTUALENV_DIR}/bin/activate
+        # NB: tests require services (mongo, rabbitmq) which must be defined by the calling workflow
         make -C "${ROOT_DIR}" -f "${CI_DIR}/.circle/Makefile" all-ci
-
-  services:
-    mongo:
-      image: mongo:3.4
-      ports:
-        - 27017:27017
-    rabbitmq:
-      image: rabbitmq:3
-      ports:
-        - 5672:5672
-    redis:
-      image: redis
-      ports:
-        - 6379:6379

From 741f0ef5d178706729236aa0bdc533b5c5d24e36 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 15:50:48 -0500
Subject: [PATCH 042/126] gha: move reusable workflow to .github/workflows

---
 {gha => .github/workflows}/pack-build_and_test.yaml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename {gha => .github/workflows}/pack-build_and_test.yaml (100%)

diff --git a/gha/pack-build_and_test.yaml b/.github/workflows/pack-build_and_test.yaml
similarity index 100%
rename from gha/pack-build_and_test.yaml
rename to .github/workflows/pack-build_and_test.yaml

From 82639e5ca0ab71e8732174d4d979f02183f61c41 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 15:54:24 -0500
Subject: [PATCH 043/126] cleanup reusable workflow name

---
 .github/workflows/pack-build_and_test.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pack-build_and_test.yaml b/.github/workflows/pack-build_and_test.yaml
index 6b17792c..28856186 100644
--- a/.github/workflows/pack-build_and_test.yaml
+++ b/.github/workflows/pack-build_and_test.yaml
@@ -15,7 +15,9 @@ on:
 jobs:
   build_and_test:
     runs-on: ubuntu-latest
-    name: 'Build and Test - Python ${{ matrix.python-version-short }}'
+    # When parent workflow is named "Build and Test" this shows up as:
+    #   "Build and Test / Python 3.6"
+    name: 'Python ${{ matrix.python-version-short }}'
     strategy:
       matrix:
         include:

From ab4e0404f848c9b8c6748580bc11253d5e9fdf2f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 15:57:59 -0500
Subject: [PATCH 044/126] move gha/ to .github/actions

---
 {gha => .github/actions}/apt-dependencies/action.yaml     | 0
 .../actions}/apt-dependencies/apt-packages.txt            | 0
 {gha => .github/actions}/checkout/action.yaml             | 2 +-
 {gha => .github/actions}/deploy/action.yaml               | 0
 {gha => .github/actions}/py-dependencies/action.yaml      | 0
 {gha => .github/actions}/test/action.yaml                 | 0
 .github/workflows/pack-build_and_test.yaml                | 8 ++++----
 7 files changed, 5 insertions(+), 5 deletions(-)
 rename {gha => .github/actions}/apt-dependencies/action.yaml (100%)
 rename {gha => .github/actions}/apt-dependencies/apt-packages.txt (100%)
 rename {gha => .github/actions}/checkout/action.yaml (95%)
 rename {gha => .github/actions}/deploy/action.yaml (100%)
 rename {gha => .github/actions}/py-dependencies/action.yaml (100%)
 rename {gha => .github/actions}/test/action.yaml (100%)

diff --git a/gha/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
similarity index 100%
rename from gha/apt-dependencies/action.yaml
rename to .github/actions/apt-dependencies/action.yaml
diff --git a/gha/apt-dependencies/apt-packages.txt b/.github/actions/apt-dependencies/apt-packages.txt
similarity index 100%
rename from gha/apt-dependencies/apt-packages.txt
rename to .github/actions/apt-dependencies/apt-packages.txt
diff --git a/gha/checkout/action.yaml b/.github/actions/checkout/action.yaml
similarity index 95%
rename from gha/checkout/action.yaml
rename to .github/actions/checkout/action.yaml
index 85a3ef7c..2eee7fb6 100644
--- a/gha/checkout/action.yaml
+++ b/.github/actions/checkout/action.yaml
@@ -28,7 +28,7 @@ runs:
       run: |
         echo "ST2_REPO_PATH=${{ github.workspace }}/st2" >> ${GITHUB_ENV}
         echo "ROOT_DIR=${{ github.workspace }}/pack" >> ${GITHUB_ENV}
-        echo "CI_DIR=$(realpath ${{ github.action_path }}/../..)" >> ${GITHUB_ENV}
+        echo "CI_DIR=$(realpath ${{ github.action_path }}/../../..)" >> ${GITHUB_ENV}
         echo "LINT_CONFIGS_PATH=${{ github.workspace }}/lint-configs" >> ${GITHUB_ENV}
 
     - name: Configure git user
diff --git a/gha/deploy/action.yaml b/.github/actions/deploy/action.yaml
similarity index 100%
rename from gha/deploy/action.yaml
rename to .github/actions/deploy/action.yaml
diff --git a/gha/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
similarity index 100%
rename from gha/py-dependencies/action.yaml
rename to .github/actions/py-dependencies/action.yaml
diff --git a/gha/test/action.yaml b/.github/actions/test/action.yaml
similarity index 100%
rename from gha/test/action.yaml
rename to .github/actions/test/action.yaml
diff --git a/.github/workflows/pack-build_and_test.yaml b/.github/workflows/pack-build_and_test.yaml
index 28856186..2543eedf 100644
--- a/.github/workflows/pack-build_and_test.yaml
+++ b/.github/workflows/pack-build_and_test.yaml
@@ -26,21 +26,21 @@ jobs:
     steps:
       # eventually replace @gha with @master
       - name: Checkout Pack Repo and CI Repos
-        uses: StackStorm-Exchange/ci/gha/checkout@gha
+        uses: StackStorm-Exchange/ci/.github/actions/checkout@gha
         
       - name: Install APT Dependencies
-        uses: StackStorm-Exchange/ci/gha/apt-dependencies@gha
+        uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@gha
         with:
           cache-version: ${{ inputs.apt-cache-version }}
 
       - name: Install Python Dependencies
-        uses: StackStorm-Exchange/ci/gha/py-dependencies@gha
+        uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
         with:
           cache-version: ${{ inputs.py-cache-version }}
           python-version: ${{ matrix.python-version }}
 
       - name: Run pack tests
-        uses: StackStorm-Exchange/ci/gha/test@gha
+        uses: StackStorm-Exchange/ci/.github/actions/test@gha
         with:
           # This makes the tests use an alternate config that enables shared libs
           enable-common-libs: true

From 7da82a9716d1731af8564abddeea20440c95f0f0 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:02:03 -0500
Subject: [PATCH 045/126] gha: drop debug comment

---
 .github/actions/test/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
index aab4c830..db080f2b 100644
--- a/.github/actions/test/action.yaml
+++ b/.github/actions/test/action.yaml
@@ -46,9 +46,9 @@ runs:
         source ${CI_DIR}/tools/functions.sh
         DEFAULT_BRANCH=$(_gh_default_branch)
         echo "BASE_BRANCH=origin/${DEFAULT_BRANCH}" >> ${GITHUB_ENV}
-        #if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
+        if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
         echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
-        #fi
+        fi
         if [[ "true" == "${{ inputs.enable-common-libs }}" ]]; then
         echo "Common libs PATH selected"
         echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf" >> ${GITHUB_ENV}

From 3db92f761e8a32440e4849d8ba6a2faf76a7a996 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:04:13 -0500
Subject: [PATCH 046/126] gha: allow packs to add system packages via
 .github/apt-packages.txt

---
 .github/actions/apt-dependencies/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index d2ecd687..d86ff232 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -26,7 +26,7 @@ runs:
       shell: bash
       run: |
         cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
-      # TODO: append pack-defined .github/apt-packages.txt to ${GITHUB_WORKSPACE}/apt-packages.txt
+        [[ -f pack/.github/apt-packages.txt ]] && cat pack/.github/apt-packages.txt >> ${GITHUB_WORKSPACE}/apt-packages.txt
 
     - name: Cache APT Dependencies
       id: cache-apt-deps

From c89b37df697274396a76a16c29e1712020ace40a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:14:00 -0500
Subject: [PATCH 047/126] gha: move py REQUIREMENTS_DIR

---
 .circle/Makefile                                      | 11 ++++++-----
 .github/actions/py-dependencies/action.yaml           |  5 +++--
 .../actions/py-dependencies/requirements-ci-ci.txt    |  4 ++++
 .github/actions/py-dependencies/requirements-dev.txt  |  9 +++++++++
 .../py-dependencies/requirements-pack-tests.txt       |  5 +++++
 .github/actions/py-dependencies/requirements.txt      |  3 +++
 6 files changed, 30 insertions(+), 7 deletions(-)
 create mode 100644 .github/actions/py-dependencies/requirements-ci-ci.txt
 create mode 100644 .github/actions/py-dependencies/requirements-dev.txt
 create mode 100644 .github/actions/py-dependencies/requirements-pack-tests.txt
 create mode 100644 .github/actions/py-dependencies/requirements.txt

diff --git a/.circle/Makefile b/.circle/Makefile
index 3f0365a4..b0a10cbc 100644
--- a/.circle/Makefile
+++ b/.circle/Makefile
@@ -10,6 +10,7 @@ ST2_REPO_BRANCH ?= master
 FORCE_CHECK_ALL_FILES ?= false
 FORCE_CHECK_PACK ?= false
 LINT_CONFIGS_PATH ?= $(CI_DIR)/lint-configs
+REQUIREMENTS_DIR ?= $(CI_DIR)/.circle
 
 export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK
 
@@ -106,7 +107,7 @@ compile:
 		export PYTHONPATH=$(ROOT_DIR)/lib:${PYTHONPATH}; \
 	fi; \
 	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \
-		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
+		REQUIREMENTS_DIR=$(REQUIREMENTS_DIR)/ \
 		CONFIG_DIR=$(LINT_CONFIGS_PATH)/ \
 		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
 	else \
@@ -284,8 +285,8 @@ requirements: virtualenv .clone_st2_repo .install-runners
 	@echo "==================== requirements ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==20.3.3"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-dev.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-pack-tests.txt
 
 .PHONY: requirements-ci
 requirements-ci:
@@ -293,8 +294,8 @@ requirements-ci:
 	@echo "==================== requirements-ci ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==20.3.3"
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt
-	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-dev.txt
+	. $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(REQUIREMENTS_DIR)/requirements-pack-tests.txt
 
 .PHONY: virtualenv
 virtualenv: $(VIRTUALENV_DIR)/bin/activate
diff --git a/.github/actions/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
index 6e743b2c..c0315187 100644
--- a/.github/actions/py-dependencies/action.yaml
+++ b/.github/actions/py-dependencies/action.yaml
@@ -28,6 +28,7 @@ runs:
       shell: bash
       run: |
         echo "VIRTUALENV_DIR=${HOME}/virtualenv" >> ${GITHUB_ENV}
+        echo "REQUIREMENTS_DIR=${GITHUB_ACTION_PATH}" >> ${GITHUB_ENV}
 
     - name: 'Set up Python (${{ inputs.python-version }})'
       uses: actions/setup-python@v2
@@ -70,8 +71,8 @@ runs:
       shell: bash
       run: |
         echo "::group::Install CI Requirements"
-        ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-dev.txt
-        ${VIRTUALENV_DIR}/bin/pip install -q -r ${CI_DIR}/.circle/requirements-pack-tests.txt
+        ${VIRTUALENV_DIR}/bin/pip install -q -r ${REQUIREMENTS_DIR}/requirements-dev.txt
+        ${VIRTUALENV_DIR}/bin/pip install -q -r ${REQUIREMENTS_DIR}/requirements-pack-tests.txt
         echo "::endgroup::"
 
     - name: Install StackStorm Requirements
diff --git a/.github/actions/py-dependencies/requirements-ci-ci.txt b/.github/actions/py-dependencies/requirements-ci-ci.txt
new file mode 100644
index 00000000..d6fdb7ee
--- /dev/null
+++ b/.github/actions/py-dependencies/requirements-ci-ci.txt
@@ -0,0 +1,4 @@
+flake8
+pylint
+pyyaml
+requests
diff --git a/.github/actions/py-dependencies/requirements-dev.txt b/.github/actions/py-dependencies/requirements-dev.txt
new file mode 100644
index 00000000..294650d7
--- /dev/null
+++ b/.github/actions/py-dependencies/requirements-dev.txt
@@ -0,0 +1,9 @@
+# Needed for chek and lint scripts
+-e git+https://github.com/StackStorm/st2sdk.git@master#egg=st2sdk
+pyyaml
+pep8>=1.6.0,<1.7
+flake8==3.7.7
+astroid==1.6.5
+pylint==1.9.4
+# fix isort dependency of pylint 1.9.4 by keeping it below its 5.x version
+isort>=4.2.5,<5
diff --git a/.github/actions/py-dependencies/requirements-pack-tests.txt b/.github/actions/py-dependencies/requirements-pack-tests.txt
new file mode 100644
index 00000000..84bf40d6
--- /dev/null
+++ b/.github/actions/py-dependencies/requirements-pack-tests.txt
@@ -0,0 +1,5 @@
+mock>=1.3.0,<2.1
+unittest2>=1.1.0,<2.0
+nose>=1.3.7
+# temporary workaround for travis issue
+eventlet==0.19.0
diff --git a/.github/actions/py-dependencies/requirements.txt b/.github/actions/py-dependencies/requirements.txt
new file mode 100644
index 00000000..2c438f9d
--- /dev/null
+++ b/.github/actions/py-dependencies/requirements.txt
@@ -0,0 +1,3 @@
+jsonschema==2.5.1
+pyyaml==4.2b1
+requests[security]==2.21.0

From 728ef154165abd52d723426bb0c1dd758c22b5ca Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:16:16 -0500
Subject: [PATCH 048/126] gha: Makefile is no longer hard-coded to require
 circleci

---
 .github/actions/test/action.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
index db080f2b..436416f0 100644
--- a/.github/actions/test/action.yaml
+++ b/.github/actions/test/action.yaml
@@ -58,7 +58,8 @@ runs:
 
     - name: Run tests
       shell: bash
-      # TODO: This Makefile has CircleCI assumptions
+      # NB: This Makefile has CircleCI-based defaults which we override with env vars
+      # defined in the various composite actions.
       run: |
         source ${VIRTUALENV_DIR}/bin/activate
         # NB: tests require services (mongo, rabbitmq) which must be defined by the calling workflow

From b1f18f90574cda23a269780ed5d5994a158332f6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:18:44 -0500
Subject: [PATCH 049/126] gha: fix pack/.github/apt-packages.txt conditional

---
 .github/actions/apt-dependencies/action.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index d86ff232..3fdf40f9 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -26,7 +26,9 @@ runs:
       shell: bash
       run: |
         cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
-        [[ -f pack/.github/apt-packages.txt ]] && cat pack/.github/apt-packages.txt >> ${GITHUB_WORKSPACE}/apt-packages.txt
+        if [[ -f pack/.github/apt-packages.txt ]]; then
+        cat pack/.github/apt-packages.txt >> ${GITHUB_WORKSPACE}/apt-packages.txt
+        fi
 
     - name: Cache APT Dependencies
       id: cache-apt-deps

From c9095939a8fdf444900ff3753089913e6ea7f16e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 16:51:53 -0500
Subject: [PATCH 050/126] gha: expose enable-common-libs input in reusable
 workflow

---
 .github/workflows/pack-build_and_test.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pack-build_and_test.yaml b/.github/workflows/pack-build_and_test.yaml
index 2543eedf..2855558b 100644
--- a/.github/workflows/pack-build_and_test.yaml
+++ b/.github/workflows/pack-build_and_test.yaml
@@ -11,6 +11,13 @@ on:
         required: false
         type: string
         default: "v0"
+      enable-common-libs:
+        description: |
+          When true, use an st2.conf that sets packs.enable_common_libs=true
+          see: https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   build_and_test:
@@ -42,8 +49,7 @@ jobs:
       - name: Run pack tests
         uses: StackStorm-Exchange/ci/.github/actions/test@gha
         with:
-          # This makes the tests use an alternate config that enables shared libs
-          enable-common-libs: true
+          enable-common-libs: ${{ inputs.enable-common-libs }}
 
     services:
       mongo:

From 8fa382da4ef167cd4f28b885f638ee6b010e76fe Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 17:12:41 -0500
Subject: [PATCH 051/126] gha: correct doc strings about action deps

---
 .github/actions/apt-dependencies/action.yaml | 2 ++
 .github/actions/py-dependencies/action.yaml  | 4 ++--
 .github/actions/test/action.yaml             | 6 +++---
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index 3fdf40f9..73b74d43 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -2,6 +2,8 @@
 name: Install APT Dependencies
 description: |
   Install debian dependencies required for StackStorm-Exchange pack tests.
+  Before using this, make sure to run
+  StackStorm-Exchange/ci/.github/actions/checkout.
 author: StackStorm
 
 inputs:
diff --git a/.github/actions/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
index c0315187..b444d15c 100644
--- a/.github/actions/py-dependencies/action.yaml
+++ b/.github/actions/py-dependencies/action.yaml
@@ -3,8 +3,8 @@ name: Install Python Dependencies
 description: |
   Install python dependencies required for StackStorm-Exchange pack tests.
   Before using this, make sure to run
-  StackStorm-Exchange/ci/gha/checkout and
-  StackStorm-Exchange/ci/gha/apt-dependencies.
+  StackStorm-Exchange/ci/.github/actions/checkout and
+  StackStorm-Exchange/ci/.github/actions/apt-dependencies.
 author: StackStorm
 
 inputs:
diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
index 436416f0..df42fbce 100644
--- a/.github/actions/test/action.yaml
+++ b/.github/actions/test/action.yaml
@@ -3,9 +3,9 @@ name: Run pack tests
 description: |
   Run StackStorm-Exchange pack tests.
   Before using this, make sure to run
-  StackStorm-Exchange/ci/gha/checkout,
-  StackStorm-Exchange/ci/gha/apt-dependencies, and
-  StackStorm-Exchange/ci/gha/py-dependencies.
+  StackStorm-Exchange/ci/.github/actions/checkout,
+  StackStorm-Exchange/ci/.github/actions/apt-dependencies, and
+  StackStorm-Exchange/ci/.github/actions/py-dependencies.
 author: StackStorm
 
 inputs:

From b8c6bc5b6c93c7df324ca74b22052e9001e4a3c1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 17:13:35 -0500
Subject: [PATCH 052/126] gha: update stub deploy workflow

---
 .github/actions/deploy/action.yaml | 43 +++++++-----------------------
 1 file changed, 10 insertions(+), 33 deletions(-)

diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml
index 122fef17..5d690974 100644
--- a/.github/actions/deploy/action.yaml
+++ b/.github/actions/deploy/action.yaml
@@ -2,6 +2,11 @@
 name: Deploy pack
 description: |
   Deploy pack to StackStorm-Exchange index.
+  Before using this, make sure to run
+  StackStorm-Exchange/ci/.github/actions/checkout,
+  StackStorm-Exchange/ci/.github/actions/apt-dependencies,
+  StackStorm-Exchange/ci/.github/actions/py-dependencies, and
+  StackStorm-Exchange/ci/.github/actions/test.
 author: StackStorm
 
 inputs: {}
@@ -14,22 +19,7 @@ runs:
   using: "composite"
   steps:
 
-    - name: add HOME to env context
-      shell: bash
-      run: echo "HOME=${HOME}" >> ${GITHUB_ENV}
-
-    - name: Get Pack Name
-      id: pack-name
-      shell: bash
-      env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-      run: |
-        export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${GITHUB_ACTION_PATH}/../../.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
-        if [[ -z "${PACK_NAME}" ]]; then
-        echo "Unable to retrieve pack name."
-        exit 1
-        fi
-        echo "::set-output name=pack-name::${PACK_NAME}"
+    # The .github/actions/test action retrieves the PACK_NAME, so it is already present in the env at this point.
 
     # checkout pack and index repos
 
@@ -43,24 +33,11 @@ runs:
     #  - rebuild index JSON
     #  - copy and optimize icon
 
-    - name: 
+    - name: Update exchange.stackstorm.org
       shell: bash
       env:
-        VIRTUALENV_DIR: ${{ env.HOME }}/virtualenv
-        FORCE_CHECK_ALL_FILES: true if on default branch
-        ROOT_DIR: pack directory
-        CI_DIR: ${{ format('{0}{1}', github.action_path, '/../..') }}
-        ST2_REPO_PATH: st2 directory
-        ENABLE_COMMON_LIBS: ${{ inputs.enable-common-libs }}
+        GH_TOKEN: ${{ github.token }}
       # TODO: This Makefile has CircleCI assumptions
       run: |
-        if [[ "true" == "${ENABLE_COMMON_LIBS}" ]]; then
-        echo "Common libs PATH selected"
-        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf
-        else
-        export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
-        fi
-        cp ${CI_DIR}/.circle/Makefile .
-        make -C "${ROOT_DIR}" all-ci
-
-
+        source ${VIRTUALENV_DIR}/bin/activate
+        # ???

From 590b851635e9156e7d24bd0b1b4a846b60bf3c93 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 21:33:11 -0500
Subject: [PATCH 053/126] gha: copy var from circle workflow to GHA test action

---
 .github/actions/test/action.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
index df42fbce..7b66dc3c 100644
--- a/.github/actions/test/action.yaml
+++ b/.github/actions/test/action.yaml
@@ -55,6 +55,9 @@ runs:
         else
         echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf" >> ${GITHUB_ENV}
         fi
+        # Don't install various StackStorm dependencies which are already
+        # installed by CI again in the various check scripts
+        echo "ST2_INSTALL_DEPS=0" >> ${GITHUB_ENV}
 
     - name: Run tests
       shell: bash

From 6d5f8c7d377e7f9eae0ebe6d1c5c48837acacf45 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 21:34:09 -0500
Subject: [PATCH 054/126] gha: add reusable index update workflow

---
 .github/workflows/index-update.yaml | 100 ++++++++++++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 .github/workflows/index-update.yaml

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
new file mode 100644
index 00000000..43ae10d6
--- /dev/null
+++ b/.github/workflows/index-update.yaml
@@ -0,0 +1,100 @@
+name: Index Update
+
+on:
+  workflow_call:
+    inputs:
+      git_user_name:
+        required: false
+        default: "StackStorm Exchange"
+      git_user_email:
+        required: false
+        default: "info@stackstorm.com"
+      ci_branch:
+        required: false
+        # TODO: switch to master
+        default: gha
+      exchange_tools_branch:
+        required: false
+        # TODO: switch to master
+        default: add-more-tools
+      packs_org:
+        required: false
+        default: StackStorm-Exchange
+      pack_repo_prefix:
+        required: false
+        default: stackstorm
+
+jobs:
+  regenerate_index:
+    runs-on: ubuntu-latest
+    # When parent workflow is named "Update Index" this shows up as:
+    #   "Update Index / Regenerate"
+    name: Regenerate
+    environment:
+      GH_TOKEN: ${{ github.token }}
+
+    steps:
+
+    - name: Add checkout paths to env context
+      shell: bash
+      run: |
+        echo "CI_DIR=${{ github.workspace }}/ci" >> ${GITHUB_ENV}
+        echo "TOOLS_DIR=${{ github.workspace }}/exchange-tools" >> ${GITHUB_ENV}
+        echo "ST2_REPO_PATH=${{ github.workspace }}/st2" >> ${GITHUB_ENV}
+        echo "INDEX_DIR=${{ github.workspace }}/index" >> ${GITHUB_ENV}
+        echo "PACKS_PATH=${{ github.workspace }}/packs" >> ${GITHUB_ENV}
+
+    - name: Configure git user
+      shell: bash
+      env:
+        GIT_USER_NAME: "${{ github.events.inputs.git_user_name }}"
+        GIT_USER_EMAIL: "${{ github.events.inputs.git_user_email }}"
+      run: |
+        git config --global user.name "${GIT_USER_NAME}"
+        git config --global user.email "${GIT_USER_EMAIL}"
+
+    - name: Checkout index repo
+      uses: actions/checkout@v2
+      with:
+        path: index
+        fetch-depth: 1
+
+    - name: Checkout ci repo
+      uses: actions/checkout@v2
+      with:
+        repository: StackStorm-Exchange/ci
+        ref: ${{ github.events.inputs.ci_branch }}
+        path: ci
+        fetch-depth: 1
+
+    - name: Checkout exchange-tools repo
+      uses: actions/checkout@v2
+      with:
+        # TODO: switch to StackStorm-Exchange once these are merged:
+        # https://github.com/StackStorm-Exchange/exchange-tools/pull/3
+        # https://github.com/StackStorm-Exchange/exchange-tools/pull/2
+        #repository: StackStorm-Exchange/exchange-tools
+        repository: cognifloyd/exchange-tools
+        ref: ${{ github.events.inputs.exchange_tools_branch }}
+        path: exchange-tools
+        fetch-depth: 1
+
+    - name: Checkout st2 repo
+      # so other scripts can reference StackStorm Python code
+      uses: actions/checkout@v2
+      with:
+        repository: StackStorm/st2
+        ref: ${{ inputs.st2_branch }}
+        path: st2
+        fetch-depth: 1
+
+    - name: Checkout Pack Repos
+      shell: bash
+      run: |
+        ${TOOLS_DIR}/clone-all-repos.sh ${PACKS_PATH} ${{ github.event.inputs.packs_org }} ${{ github.event.inputs.pack_repo_prefix }}
+
+    - name: For each pack...
+      shell: bash
+      # TODO: add pack_repo_prefix arg to pack-all.sh
+      run: |
+        ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose 'pwd'

From 60f0dab8fde6973604e65ce0818d8d9400a9f0be Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 21:47:19 -0500
Subject: [PATCH 055/126] gha: fix env usage in index-update workflow

---
 .github/workflows/index-update.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 43ae10d6..2fa79573 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -30,8 +30,6 @@ jobs:
     # When parent workflow is named "Update Index" this shows up as:
     #   "Update Index / Regenerate"
     name: Regenerate
-    environment:
-      GH_TOKEN: ${{ github.token }}
 
     steps:
 
@@ -90,11 +88,15 @@ jobs:
 
     - name: Checkout Pack Repos
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
       run: |
         ${TOOLS_DIR}/clone-all-repos.sh ${PACKS_PATH} ${{ github.event.inputs.packs_org }} ${{ github.event.inputs.pack_repo_prefix }}
 
     - name: For each pack...
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
       # TODO: add pack_repo_prefix arg to pack-all.sh
       run: |
         ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose 'pwd'

From 88678474c504424ccb7472c696356bb6c0a825fa Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 26 Nov 2021 21:56:02 -0500
Subject: [PATCH 056/126] gha: add inputs types

---
 .github/workflows/index-update.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 2fa79573..558d8ec1 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -5,23 +5,29 @@ on:
     inputs:
       git_user_name:
         required: false
+        type: string
         default: "StackStorm Exchange"
       git_user_email:
         required: false
+        type: string
         default: "info@stackstorm.com"
       ci_branch:
         required: false
+        type: string
         # TODO: switch to master
         default: gha
       exchange_tools_branch:
         required: false
+        type: string
         # TODO: switch to master
         default: add-more-tools
       packs_org:
         required: false
+        type: string
         default: StackStorm-Exchange
       pack_repo_prefix:
         required: false
+        type: string
         default: stackstorm
 
 jobs:

From 22f057eacf045e65ee7584e6c4ad0ae875e9a28e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 09:09:37 -0500
Subject: [PATCH 057/126] gha: use https to clone packs

---
 .github/workflows/index-update.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 558d8ec1..74d8d1da 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -96,8 +96,17 @@ jobs:
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+        PACKS_ORG: ${{ github.event.inputs.packs_org }}
+        PACKS_PREFIX: ${{ github.event.inputs.pack_repo_prefix }}
       run: |
-        ${TOOLS_DIR}/clone-all-repos.sh ${PACKS_PATH} ${{ github.event.inputs.packs_org }} ${{ github.event.inputs.pack_repo_prefix }}
+        mkdir -p ${PACKS_PATH}
+        cd ${PACKS_PATH}
+        source ${TOOLS_DIR}/functions.sh
+        for repo_name in $(_gh_list_repo_names ${PACKS_ORG} ${PACKS_PREFIX}); do
+        echo "::group::Clone ${PACKS_ORG}/${repo_name}"
+        gh repo clone "${PACKS_ORG}/${repo_name}"
+        echo "::endgroup::"
+        done
 
     - name: For each pack...
       shell: bash

From 48dc99c8e21d5f0ede62dc8c91c74752825ca775 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 09:19:13 -0500
Subject: [PATCH 058/126] gha: fix inputs reference

---
 .github/workflows/index-update.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 74d8d1da..19264ed6 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -51,8 +51,8 @@ jobs:
     - name: Configure git user
       shell: bash
       env:
-        GIT_USER_NAME: "${{ github.events.inputs.git_user_name }}"
-        GIT_USER_EMAIL: "${{ github.events.inputs.git_user_email }}"
+        GIT_USER_NAME: "${{ github.event.inputs.git_user_name }}"
+        GIT_USER_EMAIL: "${{ github.event.inputs.git_user_email }}"
       run: |
         git config --global user.name "${GIT_USER_NAME}"
         git config --global user.email "${GIT_USER_EMAIL}"
@@ -67,7 +67,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         repository: StackStorm-Exchange/ci
-        ref: ${{ github.events.inputs.ci_branch }}
+        ref: ${{ github.event.inputs.ci_branch }}
         path: ci
         fetch-depth: 1
 
@@ -79,7 +79,7 @@ jobs:
         # https://github.com/StackStorm-Exchange/exchange-tools/pull/2
         #repository: StackStorm-Exchange/exchange-tools
         repository: cognifloyd/exchange-tools
-        ref: ${{ github.events.inputs.exchange_tools_branch }}
+        ref: ${{ github.event.inputs.exchange_tools_branch }}
         path: exchange-tools
         fetch-depth: 1
 

From 0f73ac074253989a38cb7709808eb4c89dc89307 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 18:46:17 -0500
Subject: [PATCH 059/126] gha: workflow_call uses inputs not
 github.events.inputs

---
 .github/workflows/index-update.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 19264ed6..a64eaa3c 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -51,8 +51,8 @@ jobs:
     - name: Configure git user
       shell: bash
       env:
-        GIT_USER_NAME: "${{ github.event.inputs.git_user_name }}"
-        GIT_USER_EMAIL: "${{ github.event.inputs.git_user_email }}"
+        GIT_USER_NAME: ${{ inputs.git_user_name }}
+        GIT_USER_EMAIL: ${{ inputs.git_user_email }}
       run: |
         git config --global user.name "${GIT_USER_NAME}"
         git config --global user.email "${GIT_USER_EMAIL}"
@@ -67,7 +67,7 @@ jobs:
       uses: actions/checkout@v2
       with:
         repository: StackStorm-Exchange/ci
-        ref: ${{ github.event.inputs.ci_branch }}
+        ref: ${{ inputs.ci_branch }}
         path: ci
         fetch-depth: 1
 
@@ -79,7 +79,7 @@ jobs:
         # https://github.com/StackStorm-Exchange/exchange-tools/pull/2
         #repository: StackStorm-Exchange/exchange-tools
         repository: cognifloyd/exchange-tools
-        ref: ${{ github.event.inputs.exchange_tools_branch }}
+        ref: ${{ inputs.exchange_tools_branch }}
         path: exchange-tools
         fetch-depth: 1
 
@@ -96,8 +96,8 @@ jobs:
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
-        PACKS_ORG: ${{ github.event.inputs.packs_org }}
-        PACKS_PREFIX: ${{ github.event.inputs.pack_repo_prefix }}
+        PACKS_ORG: ${{ inputs.packs_org }}
+        PACKS_PREFIX: ${{ inputs.pack_repo_prefix }}
       run: |
         mkdir -p ${PACKS_PATH}
         cd ${PACKS_PATH}

From b1e62583a80d2bf7dd3d2720247bb295b2b93767 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 19:20:12 -0500
Subject: [PATCH 060/126] gha: add working directory to all packs task

---
 .github/workflows/index-update.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index a64eaa3c..a98aec44 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -112,6 +112,7 @@ jobs:
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+      working-directory: packs
       # TODO: add pack_repo_prefix arg to pack-all.sh
       run: |
         ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose 'pwd'

From b8bfe631773c6541ecce66531141c3073df13f80 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 21:45:36 -0500
Subject: [PATCH 061/126] gha: better each pack test

---
 .github/workflows/index-update.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index a98aec44..fa2fdd97 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -113,6 +113,8 @@ jobs:
       env:
         GH_TOKEN: ${{ github.token }}
       working-directory: packs
-      # TODO: add pack_repo_prefix arg to pack-all.sh
       run: |
-        ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose 'pwd'
+        function do_it() {
+        ls pack.yaml
+        }
+        ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose do_it

From 87de8a017a7fdf50934d4b064b7a587780e89a42 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 22:09:52 -0500
Subject: [PATCH 062/126] gha: export -f do_it

---
 .github/workflows/index-update.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index fa2fdd97..1208abe9 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -115,6 +115,9 @@ jobs:
       working-directory: packs
       run: |
         function do_it() {
+        echo "::group::pack - ${pack}"
         ls pack.yaml
+        echo "::endgroup::"
         }
+        export -f do_it
         ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose do_it

From 8ffd88361a9eb655eed0084e8080077b81a2e280 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 22:21:19 -0500
Subject: [PATCH 063/126] gha: drop verbose

---
 .github/workflows/index-update.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 1208abe9..eb86ac0b 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -120,4 +120,5 @@ jobs:
         echo "::endgroup::"
         }
         export -f do_it
-        ${TOOLS_DIR}/pack-all.sh --continue-on-error --verbose do_it
+        ${TOOLS_DIR}/pack-all.sh do_it
+        #${TOOLS_DIR}/pack-all.sh --continue-on-error do_it

From 7ab5e62cdc178dbb6f7f0d048ee1a858ce791c8d Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 22:46:27 -0500
Subject: [PATCH 064/126] gha: debug failure

---
 .github/workflows/index-update.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index eb86ac0b..0b259e4f 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -120,5 +120,6 @@ jobs:
         echo "::endgroup::"
         }
         export -f do_it
-        ${TOOLS_DIR}/pack-all.sh do_it
+        ${TOOLS_DIR}/pack-all.sh do_it || echo $?
         #${TOOLS_DIR}/pack-all.sh --continue-on-error do_it
+        echo $?

From 2da4cdf570d2b90ad89ccc0c4fadfb6b4d3e95a0 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 27 Nov 2021 23:47:04 -0500
Subject: [PATCH 065/126] gha: simplify pack loop

---
 .github/workflows/index-update.yaml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 0b259e4f..9c5f8cde 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -114,12 +114,10 @@ jobs:
         GH_TOKEN: ${{ github.token }}
       working-directory: packs
       run: |
-        function do_it() {
+        for pack in *; do
         echo "::group::pack - ${pack}"
+        pushd ${pack}
         ls pack.yaml
+        popd
         echo "::endgroup::"
-        }
-        export -f do_it
-        ${TOOLS_DIR}/pack-all.sh do_it || echo $?
-        #${TOOLS_DIR}/pack-all.sh --continue-on-error do_it
-        echo $?
+        done

From 50d8201cf670bf549d42c519f5e95d6b6ea3c7bf Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 00:09:25 -0500
Subject: [PATCH 066/126] gha: skip incomplete packs

---
 .github/workflows/index-update.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 9c5f8cde..2d9fc3e5 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -115,6 +115,7 @@ jobs:
       working-directory: packs
       run: |
         for pack in *; do
+        if ! [ -f "${pack}/pack.yaml" ] ; then continue; fi
         echo "::group::pack - ${pack}"
         pushd ${pack}
         ls pack.yaml

From 13bd387de1f0c03c8d67e43ec57b41f10094ec4d Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 00:27:49 -0500
Subject: [PATCH 067/126] gha: clean up pack loop output

---
 .github/workflows/index-update.yaml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 2d9fc3e5..794beede 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -115,10 +115,13 @@ jobs:
       working-directory: packs
       run: |
         for pack in *; do
-        if ! [ -f "${pack}/pack.yaml" ] ; then continue; fi
-        echo "::group::pack - ${pack}"
-        pushd ${pack}
+        if ! [ -f "${pack}/pack.yaml" ] ; then
+        echo "pack repo - ${pack}: SKIPPED - missing pack.yaml"
+        continue
+        fi
+        echo "::group::pack repo - ${pack}"
+        pushd ${pack} >/dev/null
         ls pack.yaml
-        popd
+        popd >/dev/null
         echo "::endgroup::"
         done

From 0bc58716b2c5729ddd3300615fc5e54ac65eb294 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 00:50:33 -0500
Subject: [PATCH 068/126] gha: delete clone of bad pack repos

---
 .github/workflows/index-update.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 794beede..086df703 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -106,6 +106,11 @@ jobs:
         echo "::group::Clone ${PACKS_ORG}/${repo_name}"
         gh repo clone "${PACKS_ORG}/${repo_name}"
         echo "::endgroup::"
+        if ! [ -f "${repo_name}/pack.yaml" ] ; then
+        rm -rf ${repo_name}
+        echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing pack.yaml"
+        echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
+        fi
         done
 
     - name: For each pack...
@@ -115,10 +120,6 @@ jobs:
       working-directory: packs
       run: |
         for pack in *; do
-        if ! [ -f "${pack}/pack.yaml" ] ; then
-        echo "pack repo - ${pack}: SKIPPED - missing pack.yaml"
-        continue
-        fi
         echo "::group::pack repo - ${pack}"
         pushd ${pack} >/dev/null
         ls pack.yaml

From 7a3ca8717dfa074e127d60b30c7cb8a4e7b40658 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 01:07:59 -0500
Subject: [PATCH 069/126] gha: improve grouping/folding of per pack output

---
 .github/workflows/index-update.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 086df703..c3d30ecb 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -117,11 +117,13 @@ jobs:
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+        PACKS_PREFIX: ${{ inputs.pack_repo_prefix }}
       working-directory: packs
       run: |
-        for pack in *; do
-        echo "::group::pack repo - ${pack}"
-        pushd ${pack} >/dev/null
+        for pack_dir in *; do
+        pack=${pack_dir#"$PACKS_PREFIX"-}
+        echo "::group::pack - ${pack}"
+        pushd ${pack_dir} >/dev/null
         ls pack.yaml
         popd >/dev/null
         echo "::endgroup::"

From 76fd0552b9dc236020a2630b49d0a70a2689cb96 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 16:04:11 -0500
Subject: [PATCH 070/126] gha: add per pack deployment steps to index regen
 workflow

---
 .github/workflows/index-update.yaml | 33 ++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index c3d30ecb..cac8921c 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -113,10 +113,11 @@ jobs:
         fi
         done
 
-    - name: For each pack...
+    # TODO: Build virtualenv
+
+    - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
       env:
-        GH_TOKEN: ${{ github.token }}
         PACKS_PREFIX: ${{ inputs.pack_repo_prefix }}
       working-directory: packs
       run: |
@@ -124,7 +125,33 @@ jobs:
         pack=${pack_dir#"$PACKS_PREFIX"-}
         echo "::group::pack - ${pack}"
         pushd ${pack_dir} >/dev/null
-        ls pack.yaml
+        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${CI_DIR}/.circle/validate.py "${pack_dir}"look pack.yaml)
+        # Rebuild pack index directory
+${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
+        # Check if an icon has been added or changed
+        ICON_TARGET="${INDEX_DIR}/v1/icons/${PACK_NAME}.png"
+        if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; }; then
+        echo "Copying and optimizing the pack icon..."
+        mkdir -p ${INDEX_DIR}/v1/icons/
+        cp icon.png ${ICON_TARGET}
+        gmic ${ICON_TARGET} -resize 64,64 -output ${ICON_TARGET}
+        optipng -o5 ${ICON_TARGET}
+        echo "Icon copied and optimized."
+        fi
         popd >/dev/null
         echo "::endgroup::"
         done
+
+    - name: Rebuild index/v1/index.json
+      shell: bash
+      run: |
+        if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
+        if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
+        echo "Forcing index rebuild..."
+        fi
+        ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/index.py" --glob "${INDEX_DIR}/v1/packs/*/pack.yaml" --output "${INDEX_DIR}/v1/"
+        else
+        echo "No changes in metadata, skipping the index rebuild."
+        fi
+
+    # TODO: update index git repo if needed

From 6c7295628f79ff22f45834ae4bacae4fb640258e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 18:09:00 -0500
Subject: [PATCH 071/126] gha: install py-deps for index

---
 .github/actions/py-dependencies/action.yaml | 11 ++++++++++-
 .github/workflows/index-update.yaml         | 21 +++++++++++++++++++--
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/.github/actions/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
index b444d15c..0ba0311a 100644
--- a/.github/actions/py-dependencies/action.yaml
+++ b/.github/actions/py-dependencies/action.yaml
@@ -1,7 +1,7 @@
 ---
 name: Install Python Dependencies
 description: |
-  Install python dependencies required for StackStorm-Exchange pack tests.
+  Install python dependencies required for StackStorm-Exchange pack tests or index updates.
   Before using this, make sure to run
   StackStorm-Exchange/ci/.github/actions/checkout and
   StackStorm-Exchange/ci/.github/actions/apt-dependencies.
@@ -14,6 +14,12 @@ inputs:
   cache-version:
     required: false
     default: "v0"
+  mode:
+    required: false
+    default: pack
+    options:
+      - index
+      - pack
 
 outputs:
   pip-version:
@@ -68,6 +74,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install CI Requirements
+      if: inputs.mode == "pack"
       shell: bash
       run: |
         echo "::group::Install CI Requirements"
@@ -85,6 +92,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install Runners
+      if: inputs.mode == "pack"
       shell: bash
       run: |
         for runner in ${ST2_REPO_PATH}/contrib/runners/*; do
@@ -102,6 +110,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install Pack Requirements
+      if: inputs.mode == "pack"
       shell: bash
       working-directory: pack
       env:
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index cac8921c..a159d86b 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -29,6 +29,14 @@ on:
         required: false
         type: string
         default: stackstorm
+      py-cache-version:
+        required: false
+        type: string
+        default: "v0"
+      python-version:
+        required: false
+        type: string
+        default: "3.6"
 
 jobs:
   regenerate_index:
@@ -113,7 +121,12 @@ jobs:
         fi
         done
 
-    # TODO: Build virtualenv
+    - name: Install Python Dependencies
+      uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
+      with:
+        mode: index # ie: skip pack-specific deps
+        cache-version: ${{ inputs.py-cache-version }}
+        python-version: ${{ inputs.python-version }}
 
     - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
@@ -125,7 +138,8 @@ jobs:
         pack=${pack_dir#"$PACKS_PREFIX"-}
         echo "::group::pack - ${pack}"
         pushd ${pack_dir} >/dev/null
-        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${CI_DIR}/.circle/validate.py "${pack_dir}"look pack.yaml)
+        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}"look pack.yaml)
+        # TODO: handle invalid pack metadata
         # Rebuild pack index directory
 ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
         # Check if an icon has been added or changed
@@ -142,8 +156,11 @@ ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --outpu
         echo "::endgroup::"
         done
 
+    # index.py - circleci assumptions, st2common, gh graphql api usage
     - name: Rebuild index/v1/index.json
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
       run: |
         if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
         if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then

From d2d3a420da0cd8cf5ef30b63dbc121a1cc21daee Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 18:24:54 -0500
Subject: [PATCH 072/126] fix indentation

---
 .github/workflows/index-update.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index a159d86b..96ce0531 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -141,7 +141,7 @@ jobs:
         PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}"look pack.yaml)
         # TODO: handle invalid pack metadata
         # Rebuild pack index directory
-${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
+        ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
         # Check if an icon has been added or changed
         ICON_TARGET="${INDEX_DIR}/v1/icons/${PACK_NAME}.png"
         if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; }; then

From dd5b96c957cac83188dd444e72b09edd7e6e5c22 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 18:30:16 -0500
Subject: [PATCH 073/126] fix GHA expression quotes

---
 .github/actions/py-dependencies/action.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
index 0ba0311a..0a0d30d7 100644
--- a/.github/actions/py-dependencies/action.yaml
+++ b/.github/actions/py-dependencies/action.yaml
@@ -74,7 +74,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install CI Requirements
-      if: inputs.mode == "pack"
+      if: inputs.mode == 'pack'
       shell: bash
       run: |
         echo "::group::Install CI Requirements"
@@ -92,7 +92,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install Runners
-      if: inputs.mode == "pack"
+      if: inputs.mode == 'pack'
       shell: bash
       run: |
         for runner in ${ST2_REPO_PATH}/contrib/runners/*; do
@@ -110,7 +110,7 @@ runs:
         echo "::endgroup::"
 
     - name: Install Pack Requirements
-      if: inputs.mode == "pack"
+      if: inputs.mode == 'pack'
       shell: bash
       working-directory: pack
       env:

From efdbe46ffa48426f482ffe577e9babe7b9203305 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 18:53:13 -0500
Subject: [PATCH 074/126] gha: install apt deps for index update

---
 .github/workflows/index-update.yaml | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 96ce0531..fca3b4ad 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -29,6 +29,10 @@ on:
         required: false
         type: string
         default: stackstorm
+      apt-cache-version:
+        required: false
+        type: string
+        default: "v0"
       py-cache-version:
         required: false
         type: string
@@ -100,6 +104,18 @@ jobs:
         path: st2
         fetch-depth: 1
 
+    - name: Install APT Dependencies
+      uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@gha
+      with:
+        cache-version: ${{ inputs.apt-cache-version }}
+
+    - name: Install Python Dependencies
+      uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
+      with:
+        mode: index # ie: skip pack-specific deps
+        cache-version: ${{ inputs.py-cache-version }}
+        python-version: ${{ inputs.python-version }}
+
     - name: Checkout Pack Repos
       shell: bash
       env:
@@ -121,13 +137,6 @@ jobs:
         fi
         done
 
-    - name: Install Python Dependencies
-      uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
-      with:
-        mode: index # ie: skip pack-specific deps
-        cache-version: ${{ inputs.py-cache-version }}
-        python-version: ${{ inputs.python-version }}
-
     - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
       env:

From 468282b977812d2997a30ac75e67e9c01ef6b224 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 19:16:11 -0500
Subject: [PATCH 075/126] gha: install gmic and options for icon optimization

---
 .github/actions/apt-dependencies/action.yaml            | 7 +++++--
 .github/actions/apt-dependencies/index-apt-packages.txt | 5 +++++
 .github/workflows/index-update.yaml                     | 2 ++
 3 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 .github/actions/apt-dependencies/index-apt-packages.txt

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index 73b74d43..3fd7f47b 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -10,6 +10,9 @@ inputs:
   cache-version:
     required: false
     default: "v0"
+  extra-apt-packages-file:
+    required: false
+    default: pack/.github/apt-packages.txt
 
 runs:
   using: "composite"
@@ -28,8 +31,8 @@ runs:
       shell: bash
       run: |
         cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
-        if [[ -f pack/.github/apt-packages.txt ]]; then
-        cat pack/.github/apt-packages.txt >> ${GITHUB_WORKSPACE}/apt-packages.txt
+        if [[ -f ${{ inputs.extra-apt-packages-file }} ]]; then
+        cat ${{ inputs.extra-apt-packages-file }} >> ${GITHUB_WORKSPACE}/apt-packages.txt
         fi
 
     - name: Cache APT Dependencies
diff --git a/.github/actions/apt-dependencies/index-apt-packages.txt b/.github/actions/apt-dependencies/index-apt-packages.txt
new file mode 100644
index 00000000..1d292afd
--- /dev/null
+++ b/.github/actions/apt-dependencies/index-apt-packages.txt
@@ -0,0 +1,5 @@
+# In CircleCI, we couldn't directly install imagemagick, so
+# we used a workaround from https://discuss.circleci.com/t/error-installing-imagemagick/2963
+# TODO: determine if we should go back to using imagemagick instead of gmic and optipng
+gmic
+optipng
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index fca3b4ad..a5c28e2d 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -108,6 +108,8 @@ jobs:
       uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@gha
       with:
         cache-version: ${{ inputs.apt-cache-version }}
+        # this has dependencies for icon optimization
+        extra-apt-packages-file: ci/.github/actions/apt-dependencies/index-apt-packages.txt
 
     - name: Install Python Dependencies
       uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha

From 37f9177f6f02c81cb4576d3ab4f4185a11e55df5 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 21:54:58 -0500
Subject: [PATCH 076/126] gha: add commit/push index step and annotate bad
 assumptions in some scripts

---
 .github/workflows/index-update.yaml | 33 ++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index a5c28e2d..3d4d58d0 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -3,6 +3,10 @@ name: Index Update
 on:
   workflow_call:
     inputs:
+      force-rebuild-index:
+        required: false
+        type: boolean
+        default: false
       git_user_name:
         required: false
         type: string
@@ -139,6 +143,9 @@ jobs:
         fi
         done
 
+    # validate.py hard codes the pack repo prefix
+    # pack_content.py makes no CI assumptions
+    # TODO: maybe use imagemagick instead of gmic+optipng
     - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
       env:
@@ -149,7 +156,7 @@ jobs:
         pack=${pack_dir#"$PACKS_PREFIX"-}
         echo "::group::pack - ${pack}"
         pushd ${pack_dir} >/dev/null
-        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}"look pack.yaml)
+        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml)
         # TODO: handle invalid pack metadata
         # Rebuild pack index directory
         ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
@@ -167,14 +174,24 @@ jobs:
         echo "::endgroup::"
         done
 
-    # index.py - circleci assumptions, st2common, gh graphql api usage
+    # index.py has some bad assumptions:
+    # - error messages explain old CircleCI + GitHub PAT process.
+    # - error message assumes we're only updating one pack and
+    #   uses PACK_NAME env var to enhance the error message.
+    # - uses MACHINE_USER/MACHINE_PASSWORD env vars to determine
+    #   github api credentials. These were CircleCI secrets.
+    #   However, the GITHUB_TOKEN generated for GitHub actions
+    #   requires Bearer auth instead of basic auth.
+    #   Note: gh graphql credentials already do the right thing.
+    # - EXCHANGE ORG and pack repo prefix are hard coded.
     - name: Rebuild index/v1/index.json
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+        FORCE_REBUILD_INDEX: ${{ inputs.force-rebuild-index }}
       run: |
-        if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
-        if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
+        if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then
+        if [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then
         echo "Forcing index rebuild..."
         fi
         ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/index.py" --glob "${INDEX_DIR}/v1/packs/*/pack.yaml" --output "${INDEX_DIR}/v1/"
@@ -182,4 +199,10 @@ jobs:
         echo "No changes in metadata, skipping the index rebuild."
         fi
 
-    # TODO: update index git repo if needed
+    - name: Git Auto Commit and Push Index
+      id: git-commit
+      uses: stefanzweifel/git-auto-commit-action@v4.12.0
+      with:
+        repository: index
+        # TODO: list changed packs in commit message
+        commit_message: Update Index (GHA)

From 2bcc2d40945716849742fc6b30cbbfc380b53db8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 21:59:10 -0500
Subject: [PATCH 077/126] gha: display git index changes

---
 .github/workflows/index-update.yaml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 3d4d58d0..a16b02a8 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -174,6 +174,13 @@ jobs:
         echo "::endgroup::"
         done
 
+    - name: Display Index Changes (git)
+      shell: bash
+      working-directory: index
+      run: |
+        git status
+        git diff
+
     # index.py has some bad assumptions:
     # - error messages explain old CircleCI + GitHub PAT process.
     # - error message assumes we're only updating one pack and
@@ -199,6 +206,13 @@ jobs:
         echo "No changes in metadata, skipping the index rebuild."
         fi
 
+    - name: Display Index Changes (git)
+      shell: bash
+      working-directory: index
+      run: |
+        git status
+        git diff
+
     - name: Git Auto Commit and Push Index
       id: git-commit
       uses: stefanzweifel/git-auto-commit-action@v4.12.0
@@ -206,3 +220,10 @@ jobs:
         repository: index
         # TODO: list changed packs in commit message
         commit_message: Update Index (GHA)
+
+    - name: Display Index Changes (git)
+      shell: bash
+      working-directory: index
+      run: |
+        git status
+        git diff

From b6d508b1354b2cc9517f967841e66938cb0ba03e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 28 Nov 2021 22:27:24 -0500
Subject: [PATCH 078/126] mark TODOs important

---
 .github/workflows/index-update.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index a16b02a8..5d2a6437 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -135,7 +135,11 @@ jobs:
         for repo_name in $(_gh_list_repo_names ${PACKS_ORG} ${PACKS_PREFIX}); do
         echo "::group::Clone ${PACKS_ORG}/${repo_name}"
         gh repo clone "${PACKS_ORG}/${repo_name}"
+        # !TODO: checkout the latest tag.
         echo "::endgroup::"
+        # !TODO: exclude packs without tags
+        # !TODO: add index exclusion list to exclude utility packs like
+        #        test, test_content_version
         if ! [ -f "${repo_name}/pack.yaml" ] ; then
         rm -rf ${repo_name}
         echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing pack.yaml"
@@ -157,7 +161,7 @@ jobs:
         echo "::group::pack - ${pack}"
         pushd ${pack_dir} >/dev/null
         PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml)
-        # TODO: handle invalid pack metadata
+        # !TODO: handle invalid pack metadata
         # Rebuild pack index directory
         ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
         # Check if an icon has been added or changed

From 1aa417cf736e6cd91df6106cde4c3fb03001caf5 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Mon, 6 Dec 2021 11:58:41 -0700
Subject: [PATCH 079/126] Changes to GHA Update-Index Workflow (#118)

* Update index-update.yaml workflow to improve packs clone step
  - checks out the latest tag, so that the index includes actions details about the latest release
  - excludes packs without tags, because that means there have been no releases yet. Packs might be created from an incubator PR during any number of index-update runs.
  - use index_clone/v1/exclude_packs.txt to skip utility packs like test and test-content-version
---
 .github/workflows/index-update.yaml | 37 +++++++++++++++++++----------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 5d2a6437..dc5efaba 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -36,7 +36,7 @@ on:
       apt-cache-version:
         required: false
         type: string
-        default: "v0"
+        default: "v42"
       py-cache-version:
         required: false
         type: string
@@ -132,19 +132,30 @@ jobs:
         mkdir -p ${PACKS_PATH}
         cd ${PACKS_PATH}
         source ${TOOLS_DIR}/functions.sh
+        EXCLUDE_PACKS=$(grep -v '^#' "${INDEX_DIR}/v1/exclude_packs.txt" | xargs echo -n)
         for repo_name in $(_gh_list_repo_names ${PACKS_ORG} ${PACKS_PREFIX}); do
-        echo "::group::Clone ${PACKS_ORG}/${repo_name}"
-        gh repo clone "${PACKS_ORG}/${repo_name}"
-        # !TODO: checkout the latest tag.
-        echo "::endgroup::"
-        # !TODO: exclude packs without tags
-        # !TODO: add index exclusion list to exclude utility packs like
-        #        test, test_content_version
-        if ! [ -f "${repo_name}/pack.yaml" ] ; then
-        rm -rf ${repo_name}
-        echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing pack.yaml"
-        echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
-        fi
+          echo "::group::Clone ${PACKS_ORG}/${repo_name}"
+          gh repo clone "${PACKS_ORG}/${repo_name}"
+          if latestTag=$(git -C ${repo_name} describe --tags `git -C ${repo_name} rev-list --tags --max-count=1`); then
+            echo latestTag = $latestTag
+            git -C ${repo_name} checkout $latestTag -b latestTagBranch
+          fi
+          if [[ -z "$latestTag" ]]; then
+            rm -rf ${repo_name}
+            echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing git tags"
+            echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
+          elif [[ ! -f "./${repo_name}/pack.yaml" ]]; then
+            rm -rf ${repo_name}
+            echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing pack.yaml"
+            echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
+          elif [[ " ${EXCLUDE_PACKS} " =~ " ${repo_name#stackstorm-} " ]]; then
+            rm -rf ${repo_name}
+            echo "DELETED clone of ${PACKS_ORG}/${repo_name}: pack has been manually excluded"
+            echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
+          else
+            echo ${repo_name} is a valid pack.
+          fi
+          echo "::endgroup::"
         done
 
     # validate.py hard codes the pack repo prefix

From 0017a29c5fa5d00652c4dffa015a6363bee243a1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 13:06:29 -0600
Subject: [PATCH 080/126] gha: include the apt database in apt-dependencies
 cache

---
 .github/actions/apt-dependencies/action.yaml | 11 ++++++-----
 .github/workflows/index-update.yaml          |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index 3fd7f47b..01872c2d 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -22,9 +22,10 @@ runs:
       # this is what we did on CircleCI. Not sure if it'll work on GHA
       shell: bash
       run: >
-        sudo rm -rf /var/cache/apt/archives
-        && sudo ln -s ~/apt_cache /var/cache/apt/archives
-        && mkdir -p ~/apt_cache/partial
+        sudo rm -rf /var/cache/apt/archives /var/lib/apt/lists
+        && sudo ln -s ~/apt_cache/archives /var/cache/apt/archives
+        && sudo ln -s ~/apt_cache/lists /var/lib/apt/lists
+        && mkdir -p ~/apt_cache/archives/partial ~/apt_cache/lists
 
     # hashFiles only reads files relative to GITHUB_WORKSPACE
     - name: Copy apt-packages.txt for caching
@@ -41,9 +42,9 @@ runs:
       with:
         path: |
           ~/apt_cache/*.deb
-        key: ${{ inputs.cache-version }}-apt-${{ hashFiles('apt-packages.txt') }}
+        key: ${{ inputs.cache-version }}-apt-archives-and-lists-${{ hashFiles('apt-packages.txt') }}
         restore-keys: |
-          ${{ inputs.cache-version }}-apt-
+          ${{ inputs.cache-version }}-apt-archives-and-lists-
 
     - name: Install APT Dependencies
       shell: bash
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index dc5efaba..9c6ecce8 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -36,7 +36,7 @@ on:
       apt-cache-version:
         required: false
         type: string
-        default: "v42"
+        default: "v0"
       py-cache-version:
         required: false
         type: string

From 321e3cbfda878b816d6ea4727c17c5647336d1ae Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 13:17:06 -0600
Subject: [PATCH 081/126] gha: use PACKS_PREFIX var and improve message

---
 .github/workflows/index-update.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 9c6ecce8..db12d688 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -148,9 +148,9 @@ jobs:
             rm -rf ${repo_name}
             echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing pack.yaml"
             echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
-          elif [[ " ${EXCLUDE_PACKS} " =~ " ${repo_name#stackstorm-} " ]]; then
+          elif [[ " ${EXCLUDE_PACKS} " =~ " ${repo_name#${PACKS_PREFIX}-} " ]]; then
             rm -rf ${repo_name}
-            echo "DELETED clone of ${PACKS_ORG}/${repo_name}: pack has been manually excluded"
+            echo "DELETED clone of ${PACKS_ORG}/${repo_name}: pack excluded via index.git/v1/exclude_packs.txt"
             echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
           else
             echo ${repo_name} is a valid pack.

From 16d11ca16d414c57eb0d0d807d90859b0bddd889 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 13:51:32 -0600
Subject: [PATCH 082/126] gha: limit concurrency for index-update workflow

---
 .github/workflows/index-update.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index db12d688..6e4d7e42 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -1,5 +1,12 @@
 name: Index Update
 
+# Only allow one workflow run per branch at a time.
+# When one is already running, new runs will be pending.
+# https://github.blog/changelog/2021-04-19-github-actions-limit-workflow-run-or-job-concurrency/
+conurrency:
+  group: index-update-${{ github.ref }}
+  # Do not use cancel-in-progress! Or we might create a scenario where all workflows get canceled before they complete.
+
 on:
   workflow_call:
     inputs:

From 912130ae4d795a40f40968167a18447cfc266dfb Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 13:53:17 -0600
Subject: [PATCH 083/126] Revert "gha: limit concurrency for index-update
 workflow"

This reverts commit 16d11ca16d414c57eb0d0d807d90859b0bddd889.
---
 .github/workflows/index-update.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 6e4d7e42..db12d688 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -1,12 +1,5 @@
 name: Index Update
 
-# Only allow one workflow run per branch at a time.
-# When one is already running, new runs will be pending.
-# https://github.blog/changelog/2021-04-19-github-actions-limit-workflow-run-or-job-concurrency/
-conurrency:
-  group: index-update-${{ github.ref }}
-  # Do not use cancel-in-progress! Or we might create a scenario where all workflows get canceled before they complete.
-
 on:
   workflow_call:
     inputs:

From 89378cb6afc2b9c8fa4d0fabdff8660fdc374f36 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:17:37 -0600
Subject: [PATCH 084/126] document how validate.py works

---
 .circle/validate.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/.circle/validate.py b/.circle/validate.py
index 7d30ce16..52b04051 100644
--- a/.circle/validate.py
+++ b/.circle/validate.py
@@ -34,6 +34,9 @@ def load_yaml_file(path):
 
 
 def validate_schema(instance, schema):
+    # validate() returns a cleaned instance with default values assigned.
+    # and it calls jsonschema.validate(instance, schema) so this will
+    # raise ValidationError if instance is not valid according to schema
     return util_schema.validate(instance=instance, schema=schema,
                                 cls=util_schema.CustomValidator,
                                 use_default=True,
@@ -51,12 +54,21 @@ def validate_repo_name(instance, repo_name):
 
 
 if __name__ == '__main__':
+    # If an exception is raised, python basically does sys.exit(1)
+    # Without an exception, the return code is 0.
+
     repo_name = sys.argv[1]
+    # raises if yaml is invalid
     pack_meta = load_yaml_file(sys.argv[2])
 
     # TODO: Figure out why this wasn't previously executed, and execute it
+    #       stackstorm-test-content-version repo is test_content_version pack
     # validate_repo_name(pack_meta, repo_name)
-    validate_schema(pack_meta, PACK_SCHEMA)
+
+    # raises ValidationError if pack_meta doesn't validate against PACK_SCHEMA
+    cleaned_pack_meta = validate_schema(pack_meta, PACK_SCHEMA)
+
+    # raises ValueError if pack ref not defined and pack name is not a valid ref
     pack_ref = validate_pack_contains_valid_ref_or_name(pack_meta)
 
     print(pack_ref)

From 7ca8c931d1349f8b0797954175bc1b13ab5dbab8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:25:25 -0600
Subject: [PATCH 085/126] gha: reformat bash scripts with minor indents

---
 .github/actions/apt-dependencies/action.yaml |  4 +-
 .github/actions/py-dependencies/action.yaml  | 14 +++---
 .github/actions/test/action.yaml             | 12 ++---
 .github/workflows/index-update.yaml          | 48 ++++++++++----------
 4 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index 01872c2d..7ff4e9b6 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -33,7 +33,7 @@ runs:
       run: |
         cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
         if [[ -f ${{ inputs.extra-apt-packages-file }} ]]; then
-        cat ${{ inputs.extra-apt-packages-file }} >> ${GITHUB_WORKSPACE}/apt-packages.txt
+          cat ${{ inputs.extra-apt-packages-file }} >> ${GITHUB_WORKSPACE}/apt-packages.txt
         fi
 
     - name: Cache APT Dependencies
@@ -55,7 +55,7 @@ runs:
       run: |
         echo "::group::Install APT Dependencies"
         if [[ "${CACHE_HIT}" != 'true' ]]; then
-        sudo apt-get -o=Dpkg::Use-Pty=0 -yq update
+          sudo apt-get -o=Dpkg::Use-Pty=0 -yq update
         fi
         APT_PACKAGES=$(grep -v '^#' "${APT_PACKAGES_FILE_PATH}" | xargs echo -n)
         sudo apt-get -o=Dpkg::Use-Pty=0 -yq install ${APT_PACKAGES}
diff --git a/.github/actions/py-dependencies/action.yaml b/.github/actions/py-dependencies/action.yaml
index 0a0d30d7..63e526a3 100644
--- a/.github/actions/py-dependencies/action.yaml
+++ b/.github/actions/py-dependencies/action.yaml
@@ -96,9 +96,9 @@ runs:
       shell: bash
       run: |
         for runner in ${ST2_REPO_PATH}/contrib/runners/*; do
-        echo "::group::Install Runner: ${runner##*/}"
-        (. ${VIRTUALENV_DIR}/bin/activate; cd $runner; python setup.py develop)
-        echo "::endgroup::"
+          echo "::group::Install Runner: ${runner##*/}"
+          (. ${VIRTUALENV_DIR}/bin/activate; cd $runner; python setup.py develop)
+          echo "::endgroup::"
         done
 
     - name: Register Metrics Drivers
@@ -119,12 +119,12 @@ runs:
       run: |
         echo "::group::Install Pack Requirements"
         if [[ -f "${PACK_REQUIREMENTS_FILE}" ]]; then
-        echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}"
-        ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_REQUIREMENTS_FILE}"
+          echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}"
+          ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_REQUIREMENTS_FILE}"
         fi
         if [[ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]]; then
-        echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}"
-        ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}"
+          echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}"
+          ${VIRTUALENV_DIR}/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}"
         fi
         echo "::endgroup::"
 
diff --git a/.github/actions/test/action.yaml b/.github/actions/test/action.yaml
index 7b66dc3c..74248fb0 100644
--- a/.github/actions/test/action.yaml
+++ b/.github/actions/test/action.yaml
@@ -31,8 +31,8 @@ runs:
       run: |
         export PACK_NAME=$(${VIRTUALENV_DIR}/bin/python ${CI_DIR}/.circle/validate.py "${GITHUB_REPOSITORY##*/}" pack.yaml)
         if [[ -z "${PACK_NAME}" ]]; then
-        echo "Unable to retrieve pack name."
-        exit 1
+          echo "Unable to retrieve pack name."
+          exit 1
         fi
         echo "::set-output name=pack-name::${PACK_NAME}"
 
@@ -47,13 +47,13 @@ runs:
         DEFAULT_BRANCH=$(_gh_default_branch)
         echo "BASE_BRANCH=origin/${DEFAULT_BRANCH}" >> ${GITHUB_ENV}
         if [[ "${DEFAULT_BRANCH}" == "${GITHUB_REF_NAME}" ]]; then
-        echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
+          echo "FORCE_CHECK_ALL_FILES=true" >> ${GITHUB_ENV}
         fi
         if [[ "true" == "${{ inputs.enable-common-libs }}" ]]; then
-        echo "Common libs PATH selected"
-        echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf" >> ${GITHUB_ENV}
+          echo "Common libs PATH selected"
+          echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2_common_libs.tests.conf" >> ${GITHUB_ENV}
         else
-        echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf" >> ${GITHUB_ENV}
+          echo "ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf" >> ${GITHUB_ENV}
         fi
         # Don't install various StackStorm dependencies which are already
         # installed by CI again in the various check scripts
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index db12d688..ebc2cf44 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -168,25 +168,25 @@ jobs:
       working-directory: packs
       run: |
         for pack_dir in *; do
-        pack=${pack_dir#"$PACKS_PREFIX"-}
-        echo "::group::pack - ${pack}"
-        pushd ${pack_dir} >/dev/null
-        PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml)
-        # !TODO: handle invalid pack metadata
-        # Rebuild pack index directory
-        ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
-        # Check if an icon has been added or changed
-        ICON_TARGET="${INDEX_DIR}/v1/icons/${PACK_NAME}.png"
-        if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; }; then
-        echo "Copying and optimizing the pack icon..."
-        mkdir -p ${INDEX_DIR}/v1/icons/
-        cp icon.png ${ICON_TARGET}
-        gmic ${ICON_TARGET} -resize 64,64 -output ${ICON_TARGET}
-        optipng -o5 ${ICON_TARGET}
-        echo "Icon copied and optimized."
-        fi
-        popd >/dev/null
-        echo "::endgroup::"
+          pack=${pack_dir#"$PACKS_PREFIX"-}
+          echo "::group::pack - ${pack}"
+          pushd ${pack_dir} >/dev/null
+          PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml)
+          # !TODO: handle invalid pack metadata
+          # Rebuild pack index directory
+          ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
+          # Check if an icon has been added or changed
+          ICON_TARGET="${INDEX_DIR}/v1/icons/${PACK_NAME}.png"
+          if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; }; then
+            echo "Copying and optimizing the pack icon..."
+            mkdir -p ${INDEX_DIR}/v1/icons/
+            cp icon.png ${ICON_TARGET}
+            gmic ${ICON_TARGET} -resize 64,64 -output ${ICON_TARGET}
+            optipng -o5 ${ICON_TARGET}
+            echo "Icon copied and optimized."
+          fi
+          popd >/dev/null
+          echo "::endgroup::"
         done
 
     - name: Display Index Changes (git)
@@ -213,12 +213,12 @@ jobs:
         FORCE_REBUILD_INDEX: ${{ inputs.force-rebuild-index }}
       run: |
         if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then
-        if [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then
-        echo "Forcing index rebuild..."
-        fi
-        ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/index.py" --glob "${INDEX_DIR}/v1/packs/*/pack.yaml" --output "${INDEX_DIR}/v1/"
+          if [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then
+            echo "Forcing index rebuild..."
+          fi
+          ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/index.py" --glob "${INDEX_DIR}/v1/packs/*/pack.yaml" --output "${INDEX_DIR}/v1/"
         else
-        echo "No changes in metadata, skipping the index rebuild."
+          echo "No changes in metadata, skipping the index rebuild."
         fi
 
     - name: Display Index Changes (git)

From 29c2eb6fba6dea5ece437eaea908db4a7d32e051 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:26:29 -0600
Subject: [PATCH 086/126] gha: skip rebuilding index for packs with invalid
 pack metadata

---
 .github/workflows/index-update.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index ebc2cf44..9d57c1e8 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -171,8 +171,10 @@ jobs:
           pack=${pack_dir#"$PACKS_PREFIX"-}
           echo "::group::pack - ${pack}"
           pushd ${pack_dir} >/dev/null
-          PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml)
-          # !TODO: handle invalid pack metadata
+          if ! PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml); then
+            echo SKIPPING rebuild for ${pack_dir} because pack.yaml is not valid
+            continue
+          fi
           # Rebuild pack index directory
           ${VIRTUALENV_DIR}/bin/python "${CI_DIR}/utils/pack_content.py" --input . --output "${INDEX_DIR}/v1/packs/${PACK_NAME}"
           # Check if an icon has been added or changed

From 32f99f5b44921e38ce2cbcdfc53b55c6f617b3b2 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:37:54 -0600
Subject: [PATCH 087/126] gha: improve output folding to not obscure pack
 errors

---
 .github/workflows/index-update.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 9d57c1e8..4f49fcef 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -140,6 +140,7 @@ jobs:
             echo latestTag = $latestTag
             git -C ${repo_name} checkout $latestTag -b latestTagBranch
           fi
+          echo "::endgroup::" # DELETED notices will not be folded to simplify scanning action output
           if [[ -z "$latestTag" ]]; then
             rm -rf ${repo_name}
             echo "DELETED clone of ${PACKS_ORG}/${repo_name}: missing git tags"
@@ -155,7 +156,6 @@ jobs:
           else
             echo ${repo_name} is a valid pack.
           fi
-          echo "::endgroup::"
         done
 
     # validate.py hard codes the pack repo prefix
@@ -172,6 +172,7 @@ jobs:
           echo "::group::pack - ${pack}"
           pushd ${pack_dir} >/dev/null
           if ! PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml); then
+            echo "::endgroup::" # ensure SKIPPING notice is visible outside of the folded group
             echo SKIPPING rebuild for ${pack_dir} because pack.yaml is not valid
             continue
           fi

From 441b6e9dc31118f530e24194d55d85050e810250 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:42:30 -0600
Subject: [PATCH 088/126] gha: caching apt deps is not working. disable for
 now.

---
 .github/actions/apt-dependencies/action.yaml | 40 ++++++++++----------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index 7ff4e9b6..dac15a7d 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -18,17 +18,18 @@ runs:
   using: "composite"
   steps:
 
-    - name: Create a directory for debian packages so we can cache it
-      # this is what we did on CircleCI. Not sure if it'll work on GHA
-      shell: bash
-      run: >
-        sudo rm -rf /var/cache/apt/archives /var/lib/apt/lists
-        && sudo ln -s ~/apt_cache/archives /var/cache/apt/archives
-        && sudo ln -s ~/apt_cache/lists /var/lib/apt/lists
-        && mkdir -p ~/apt_cache/archives/partial ~/apt_cache/lists
+    # TODO: not working on GHA. Need to revisit caching.
+    #- name: Create a directory for debian packages so we can cache it
+    #  # this is what we did on CircleCI. Not sure if it'll work on GHA
+    #  shell: bash
+    #  run: >
+    #    sudo rm -rf /var/cache/apt/archives /var/lib/apt/lists
+    #    && sudo ln -s ~/apt_cache/archives /var/cache/apt/archives
+    #    && sudo ln -s ~/apt_cache/lists /var/lib/apt/lists
+    #    && mkdir -p ~/apt_cache/archives/partial ~/apt_cache/lists
 
     # hashFiles only reads files relative to GITHUB_WORKSPACE
-    - name: Copy apt-packages.txt for caching
+    - name: Construct apt-packages.txt
       shell: bash
       run: |
         cp ${{ github.action_path }}/apt-packages.txt ${GITHUB_WORKSPACE}/
@@ -36,20 +37,21 @@ runs:
           cat ${{ inputs.extra-apt-packages-file }} >> ${GITHUB_WORKSPACE}/apt-packages.txt
         fi
 
-    - name: Cache APT Dependencies
-      id: cache-apt-deps
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/apt_cache/*.deb
-        key: ${{ inputs.cache-version }}-apt-archives-and-lists-${{ hashFiles('apt-packages.txt') }}
-        restore-keys: |
-          ${{ inputs.cache-version }}-apt-archives-and-lists-
+    #- name: Cache APT Dependencies
+    #  id: cache-apt-deps
+    #  uses: actions/cache@v2
+    #  with:
+    #    path: |
+    #      ~/apt_cache/*.deb
+    #    key: ${{ inputs.cache-version }}-apt-archives-and-lists-${{ hashFiles('apt-packages.txt') }}
+    #    restore-keys: |
+    #      ${{ inputs.cache-version }}-apt-archives-and-lists-
 
     - name: Install APT Dependencies
       shell: bash
       env:
-        CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+    #    CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
+        CACHE_HIT: "false"
         APT_PACKAGES_FILE_PATH: apt-packages.txt
         DEBIAN_FRONTEND: noninteractive
       run: |

From f799863ffbe05ab079ef9ac4bd7fb2b8a14761c7 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:45:27 -0600
Subject: [PATCH 089/126] gha: drop superfluous set -ex

---
 .github/actions/apt-dependencies/action.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/actions/apt-dependencies/action.yaml b/.github/actions/apt-dependencies/action.yaml
index dac15a7d..69bccf86 100644
--- a/.github/actions/apt-dependencies/action.yaml
+++ b/.github/actions/apt-dependencies/action.yaml
@@ -67,7 +67,6 @@ runs:
       shell: bash
       run: |
         echo "::group::Print Versions"
-        set -ex
         jq --version
         gh --version
         echo "::endgroup::"

From 3852efdf8e09bed7471509b17648c2febbcd5ec9 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 22:47:04 -0600
Subject: [PATCH 090/126] gha: only print pack clone errors. assume valid
 otherwise

---
 .github/workflows/index-update.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 4f49fcef..6364c266 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -153,8 +153,6 @@ jobs:
             rm -rf ${repo_name}
             echo "DELETED clone of ${PACKS_ORG}/${repo_name}: pack excluded via index.git/v1/exclude_packs.txt"
             echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
-          else
-            echo ${repo_name} is a valid pack.
           fi
         done
 

From aac209eef90bfad0672d92ac99223bace057557e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 23:06:30 -0600
Subject: [PATCH 091/126] gha: improve index-update output to better identify
 errors and to avoid output interleaving

newlines (plain echo) before endgroup will hopefully flush output so gha does not interleave output
---
 .github/workflows/index-update.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 6364c266..4acf8287 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -140,6 +140,7 @@ jobs:
             echo latestTag = $latestTag
             git -C ${repo_name} checkout $latestTag -b latestTagBranch
           fi
+          echo
           echo "::endgroup::" # DELETED notices will not be folded to simplify scanning action output
           if [[ -z "$latestTag" ]]; then
             rm -rf ${repo_name}
@@ -155,6 +156,7 @@ jobs:
             echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
           fi
         done
+        cat ${GITHUB_WORKSPACE}/bad_pack_repos || true
 
     # validate.py hard codes the pack repo prefix
     # pack_content.py makes no CI assumptions
@@ -162,6 +164,7 @@ jobs:
     - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
       env:
+        PACKS_ORG: ${{ inputs.packs_org }}
         PACKS_PREFIX: ${{ inputs.pack_repo_prefix }}
       working-directory: packs
       run: |
@@ -170,6 +173,8 @@ jobs:
           echo "::group::pack - ${pack}"
           pushd ${pack_dir} >/dev/null
           if ! PACK_NAME=$(${VIRTUALENV_DIR}/bin/python "${CI_DIR}/.circle/validate.py" "${pack_dir}" pack.yaml); then
+            echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
+            echo
             echo "::endgroup::" # ensure SKIPPING notice is visible outside of the folded group
             echo SKIPPING rebuild for ${pack_dir} because pack.yaml is not valid
             continue
@@ -187,8 +192,10 @@ jobs:
             echo "Icon copied and optimized."
           fi
           popd >/dev/null
+          echo
           echo "::endgroup::"
         done
+        cat ${GITHUB_WORKSPACE}/bad_pack_repos || true
 
     - name: Display Index Changes (git)
       shell: bash

From f781d20130038808d2be05121f4ee669f32304dc Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 23:22:57 -0600
Subject: [PATCH 092/126] gha: improve index-update output to better identify
 errors

---
 .github/workflows/index-update.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 4acf8287..aac70003 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -156,7 +156,10 @@ jobs:
             echo "${PACKS_ORG}/${repo_name}" >> ${GITHUB_WORKSPACE}/bad_pack_repos
           fi
         done
-        cat ${GITHUB_WORKSPACE}/bad_pack_repos || true
+        if [[ -f ${GITHUB_WORKSPACE}/bad_pack_repos ]]; then
+          echo "These repos were DELETED. See above for why."
+          cat ${GITHUB_WORKSPACE}/bad_pack_repos
+        fi
 
     # validate.py hard codes the pack repo prefix
     # pack_content.py makes no CI assumptions
@@ -195,7 +198,10 @@ jobs:
           echo
           echo "::endgroup::"
         done
-        cat ${GITHUB_WORKSPACE}/bad_pack_repos || true
+        if [[ -f ${GITHUB_WORKSPACE}/bad_pack_repos ]]; then
+          echo "These repos were SKIPPED or DELETED. See above or in checkout step logs for why."
+          cat ${GITHUB_WORKSPACE}/bad_pack_repos
+        fi
 
     - name: Display Index Changes (git)
       shell: bash

From 1bf21915610accf2f2634070c5a1181526724ba7 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 6 Dec 2021 23:30:42 -0600
Subject: [PATCH 093/126] gha: debug index changes

---
 .circle/validate.py                 | 1 +
 .github/workflows/index-update.yaml | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/.circle/validate.py b/.circle/validate.py
index 52b04051..68f1ee5b 100644
--- a/.circle/validate.py
+++ b/.circle/validate.py
@@ -63,6 +63,7 @@ def validate_repo_name(instance, repo_name):
 
     # TODO: Figure out why this wasn't previously executed, and execute it
     #       stackstorm-test-content-version repo is test_content_version pack
+    #       stackstorm-test2 repo is test pack
     # validate_repo_name(pack_meta, repo_name)
 
     # raises ValidationError if pack_meta doesn't validate against PACK_SCHEMA
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index aac70003..71dd1ba7 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -161,6 +161,13 @@ jobs:
           cat ${GITHUB_WORKSPACE}/bad_pack_repos
         fi
 
+    - name: Display Index Changes (git)
+      shell: bash
+      working-directory: index
+      run: |
+        git status
+        git diff
+
     # validate.py hard codes the pack repo prefix
     # pack_content.py makes no CI assumptions
     # TODO: maybe use imagemagick instead of gmic+optipng

From 97983079d40dc2300e2354b3cf34a68851e9b3c9 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 7 Dec 2021 00:28:19 -0600
Subject: [PATCH 094/126] adjust index.py and validate.py to not assume
 CircleCI and pack repo prefix

---
 .circle/index.py                    | 40 ++++++++++++++++++++++++++---
 .circle/validate.py                 |  2 +-
 .github/workflows/index-update.yaml | 18 +++++--------
 3 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/.circle/index.py b/.circle/index.py
index 8da5756c..018e2c7f 100644
--- a/.circle/index.py
+++ b/.circle/index.py
@@ -17,11 +17,22 @@
 
 from st2common.util.pack import get_pack_ref_from_metadata
 
-EXCHANGE_NAME = "StackStorm-Exchange"
-EXCHANGE_PREFIX = "stackstorm"
+EXCHANGE_NAME = os.environ.get("PACKS_ORG", "StackStorm-Exchange")
+EXCHANGE_PREFIX = os.environ.get("PACKS_PREFIX", "stackstorm")
 
+if os.environ.get("CIRCLECI"):
+    CI = "CircleCI"
+elif os.environ.get("GITHUB_ACTIONS"):
+    CI = "GHA"
+else:
+    CI = "unknown"
+
+
+# TODO: drop GITHUB_USERNAME once we drop support for CircleCI
 GITHUB_USERNAME = os.environ.get('MACHINE_USER')
+# TODO: drop MACHINE_PASSWORD once we drop support for CircleCI. Keep GH_TOKEN.
 GITHUB_PASSWORD = os.environ.get("MACHINE_PASSWORD", os.environ.get("GH_TOKEN"))
+# TODO: drop ACTIVE_PACK_NAME once we drop support for CircleCI. Only used for CircleCI-specific error message.
 ACTIVE_PACK_NAME = os.environ.get('PACK_NAME', "unknown")
 
 SESSION = requests.Session()
@@ -107,7 +118,8 @@ def build_index(path_glob, output_path):
                   separators=(',', ': '))
 
     failed_message = ''
-    if failed_count > 0:
+    # TODO: drop CircleCI error message once we drop support for CircleCI
+    if failed_count > 0 and CI == "CircleCI":
         failed_message = (
             ', {failed_count} packs failed to update.\n'
             'The GitHub Personal Access Tokens for CircleCI for the pack may '
@@ -120,6 +132,14 @@ def build_index(path_glob, output_path):
             'will need to ask a member of the StackStorm TSC to update the Personal\n'
             'Access Token on your behalf.'
         ).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME)
+    elif failed_count > 0:
+        # If an issue is reported on GitHub Actions, update this error message
+        # to explain common causes and how to fix them.
+        failed_message = (
+            ', {failed_count} packs failed to update.\n'
+            'Please investigate why this failed and report an issue on:\n'
+            '  https://github.com/{exchange_name}/ci\n'
+        ).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME)
 
     print('')
     print('Processed %s packs%s.' % (counter, failed_message))
@@ -152,13 +172,20 @@ def get_available_versions():
         proc.kill()
         outs, _ = proc.communicate()
     result = outs.decode().strip()
-    if proc.returncode != 0:
+    # TODO: drop CircleCI error message once we drop support for CircleCI
+    if proc.returncode != 0 and CI == "CircleCI":
         sys.exit(
             "Error retrieving data with github graphql API.\n"
             "The GitHub PAT might need to be regenerated:\n"
             "https://github.com/settings/tokens/new?scopes=public_repo"
             "&description=CircleCI%3A%20stackstorm-" + ACTIVE_PACK_NAME
         )
+    elif proc.returncode != 0:
+        # If an issue is reported on GitHub Actions, update this error message
+        # to explain common causes and how to fix them.
+        sys.exit(
+            "Error retrieving data with github graphql API.\n"
+        )
 
     # https://stackoverflow.com/a/43807246/1134951
     decoder = json.JSONDecoder()
@@ -194,7 +221,12 @@ def get_available_versions_for_pack(pack_ref):
 
     NOTE: This function uses Github API.
     """
+    # TODO: remove this if block once we discontinue CircleCI support. Keep the else block.
     if pack_ref not in PACK_VERSIONS:
+        # This will fail in GitHub Actions because the GITHUB_TOKEN there must be
+        # provided as bearer auth instead of basic auth. But using graphql is better
+        # anyway, so this is only needed as a backup on CircleCI if graphql doesn't work.
+        # graphql should always work on GHA.
         url = ('https://api.github.com/repos/%s/%s-%s/tags' %
                (EXCHANGE_NAME, EXCHANGE_PREFIX, pack_ref))
         resp = SESSION.get(url)
diff --git a/.circle/validate.py b/.circle/validate.py
index 68f1ee5b..046683d5 100644
--- a/.circle/validate.py
+++ b/.circle/validate.py
@@ -22,7 +22,7 @@
 from st2common.util import schema as util_schema
 from st2common.util.pack import get_pack_ref_from_metadata
 
-PREFIX = 'stackstorm'
+PREFIX = os.environ.get("PACKS_PREFIX", "stackstorm")
 PACK_SCHEMA = PackAPI.schema
 
 
diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 71dd1ba7..5ab360c2 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -168,8 +168,8 @@ jobs:
         git status
         git diff
 
-    # validate.py hard codes the pack repo prefix
-    # pack_content.py makes no CI assumptions
+    # validate.py and pack_content.py make no CI assumptions
+    # TODO: once we drop CircleCI move validate.py out of the .circle directory.
     # TODO: maybe use imagemagick instead of gmic+optipng
     - name: Rebuild index/v1/packs and index/v1/icons
       shell: bash
@@ -217,20 +217,14 @@ jobs:
         git status
         git diff
 
-    # index.py has some bad assumptions:
-    # - error messages explain old CircleCI + GitHub PAT process.
-    # - error message assumes we're only updating one pack and
-    #   uses PACK_NAME env var to enhance the error message.
-    # - uses MACHINE_USER/MACHINE_PASSWORD env vars to determine
-    #   github api credentials. These were CircleCI secrets.
-    #   However, the GITHUB_TOKEN generated for GitHub actions
-    #   requires Bearer auth instead of basic auth.
-    #   Note: gh graphql credentials already do the right thing.
-    # - EXCHANGE ORG and pack repo prefix are hard coded.
+    # TODO: once we drop CircleCI, update index.py to remove CircleCI-specific bits
+    #       and move out of the .circle directory.
     - name: Rebuild index/v1/index.json
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
+        PACKS_ORG: ${{ inputs.packs_org }}
+        PACKS_PREFIX: ${{ inputs.pack_repo_prefix }}
         FORCE_REBUILD_INDEX: ${{ inputs.force-rebuild-index }}
       run: |
         if [[ -n "$(git -C ${INDEX_DIR} status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "true" ]]; then

From 3e0cbbc3469b65e410dd98198098165c61590c1a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 7 Dec 2021 10:57:13 -0600
Subject: [PATCH 095/126] gha: drop unused composite action skeleton

---
 .github/actions/deploy/action.yaml | 43 ------------------------------
 1 file changed, 43 deletions(-)
 delete mode 100644 .github/actions/deploy/action.yaml

diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml
deleted file mode 100644
index 5d690974..00000000
--- a/.github/actions/deploy/action.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-name: Deploy pack
-description: |
-  Deploy pack to StackStorm-Exchange index.
-  Before using this, make sure to run
-  StackStorm-Exchange/ci/.github/actions/checkout,
-  StackStorm-Exchange/ci/.github/actions/apt-dependencies,
-  StackStorm-Exchange/ci/.github/actions/py-dependencies, and
-  StackStorm-Exchange/ci/.github/actions/test.
-author: StackStorm
-
-inputs: {}
-outputs:
-  pack-name:
-    description: The pack name pulled from pack.yaml
-    value: ${{ steps.pack-name.outputs.pack-name }}
-
-runs:
-  using: "composite"
-  steps:
-
-    # The .github/actions/test action retrieves the PACK_NAME, so it is already present in the env at this point.
-
-    # checkout pack and index repos
-
-    # create version tags
-
-    # webhook that triggers a workflow in index repo to regen
-    #  - rebuild pack index directory
-    #    - convert resource metadata to json
-    #    - convert config schema to json
-    #    - add resource stats to updated pack.yaml
-    #  - rebuild index JSON
-    #  - copy and optimize icon
-
-    - name: Update exchange.stackstorm.org
-      shell: bash
-      env:
-        GH_TOKEN: ${{ github.token }}
-      # TODO: This Makefile has CircleCI assumptions
-      run: |
-        source ${VIRTUALENV_DIR}/bin/activate
-        # ???

From ad7e7ae2a412733f1bd23dc5774c605922ae4945 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Tue, 7 Dec 2021 11:02:12 -0600
Subject: [PATCH 096/126] skeleton pack-tag_release.yaml workflow

---
 .github/workflows/pack-tag_release.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .github/workflows/pack-tag_release.yaml

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
new file mode 100644
index 00000000..531febea
--- /dev/null
+++ b/.github/workflows/pack-tag_release.yaml
@@ -0,0 +1,15 @@
+name: Tag Release
+
+on:
+  workflow_call:
+#    inputs:
+
+jobs:
+  tag_release:
+    runs-on: ubuntu-latest
+    name: 'Tag Release'
+    steps:
+      - name: Checkout Pack Repo
+        
+      - name: 
+        run: |

From f120b47bf77bea864dbcb80a169901fccbef5311 Mon Sep 17 00:00:00 2001
From: lm-ydubler <92544319+lm-ydubler@users.noreply.github.com>
Date: Wed, 8 Dec 2021 19:32:53 -0700
Subject: [PATCH 097/126] Updates to Tag Release Workflow (#120)

* Update pack-tag_release.yaml
  - basic auto-tagging workflow if tag doesn't exist for version in pack.yaml on push to master.
---
 .github/workflows/pack-tag_release.yaml | 72 ++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index 531febea..213ef8e7 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -10,6 +10,74 @@ jobs:
     name: 'Tag Release'
     steps:
       - name: Checkout Pack Repo
-        
-      - name: 
+        uses: actions/checkout@v2
+        with:
+          repository: lm-ydubler/logicmonitor-stackstorm-pack
+          fetch-depth: 0
+          #repo-token: ${{ secrets.GITHUB_TOKEN }} # <------- NOT REQUIRED
+
+      - name: Get version from pack.yaml
+        shell: bash
+        id: pack-version
+        #uses: mikefarah/yq@master
+        #with:
+           #cmd: echo ::set-output name=result::$(yq eval '.version' pack.yaml)
+        run: |
+          pwd
+          ls
+          #############
+          # INSTALL YQ
+          #############
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+          sudo add-apt-repository ppa:rmescandon/yq
+          sudo apt update
+          sudo apt install yq -y
+          ###################################
+          # MAKE VERSION AN OUTPUT PARAMETER
+          ###################################
+          echo ::set-output name=RESULT::$(yq eval '.version' pack.yaml)
+
+      - name: Get latest git tag
+        shell: bash
+        id: latest-tag
+        run: |
+          ################################
+          # GET LATEST TAG FROM PACK REPO
+          ################################
+          if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
+            # echo "LATEST_TAG=${LATEST_TAG}" | tee -a ${GITHUB_ENVIRONMENT} # doesn't work
+            echo LATEST_TAG=$LATEST_TAG
+            echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> $GITHUB_ENV
+            #echo ::set-output name=LATEST_TAG::$LATEST_TAG
+          else
+            # echo 'LATEST_TAG=""' >> ${GITHUB_ENVIRONMENT} # doesn't work
+            echo 'LATEST_TAG=""' >> $GITHUB_ENV
+            echo There are no tags yet.
+          fi
+
+      - name: Add Tag If Required
+        if:  ${{ env.LATEST_TAG != format('v{0}', steps.pack-version.outputs.RESULT) && steps.pack-version.outputs.RESULT != '' }}
+        env:
+          PACK_VERSION: ${{steps.pack-version.outputs.RESULT}}
+        shell: bash
         run: |
+          ###################################################
+          # Create new tag on most recent commit
+          ###################################################
+          echo LATEST_TAG=${{env.LATEST_TAG}}
+          echo PACK_VERSION=v${PACK_VERSION}
+          echo LATEST_TAG != PACK_VERSION!
+          echo Creating tag named v${PACK_VERSION} on most recent commit aka HEAD ...
+          git tag "v${PACK_VERSION}" HEAD
+          ##############################################################
+          # Verify tag was created and exists on the most recent commit
+          ##############################################################
+          echo Look at the list of tags to make sure it exists:
+          git tag
+          echo Review most recent commit to make ensure the tag is associated with it:
+          git log --pretty='%h %d' --max-count=1
+          ##########################################
+          # Push tag to origin repository (success!)
+          ##########################################
+          echo Pushing new tag to origin...
+          git push origin "v${PACK_VERSION}"

From 8d576e3f4475d9df1e3c7cc49edccef1223a2ac6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 20:34:03 -0600
Subject: [PATCH 098/126] gha: simplify checkout - pack is the current repo

---
 .github/workflows/pack-tag_release.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index 213ef8e7..5b081071 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -11,10 +11,6 @@ jobs:
     steps:
       - name: Checkout Pack Repo
         uses: actions/checkout@v2
-        with:
-          repository: lm-ydubler/logicmonitor-stackstorm-pack
-          fetch-depth: 0
-          #repo-token: ${{ secrets.GITHUB_TOKEN }} # <------- NOT REQUIRED
 
       - name: Get version from pack.yaml
         shell: bash

From 418c94b0b6402766a4b8cbdec4196d784653ee3b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 20:38:05 -0600
Subject: [PATCH 099/126] gha: simplify yq install

---
 .github/workflows/pack-tag_release.yaml | 25 +++++--------------------
 1 file changed, 5 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index 5b081071..b24dea3a 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -14,24 +14,11 @@ jobs:
 
       - name: Get version from pack.yaml
         shell: bash
-        id: pack-version
-        #uses: mikefarah/yq@master
-        #with:
-           #cmd: echo ::set-output name=result::$(yq eval '.version' pack.yaml)
         run: |
-          pwd
-          ls
-          #############
-          # INSTALL YQ
-          #############
-          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
-          sudo add-apt-repository ppa:rmescandon/yq
-          sudo apt update
-          sudo apt install yq -y
-          ###################################
-          # MAKE VERSION AN OUTPUT PARAMETER
-          ###################################
-          echo ::set-output name=RESULT::$(yq eval '.version' pack.yaml)
+          wget https://github.com/mikefarah/yq/releases/download/v4.16.1/yq_linux_amd64 -O ~/yq
+          chmod +x ~/yq
+          PACK_VERSION=$(~/yq eval '.version' pack.yaml)
+          echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> $GITHUB_ENV
 
       - name: Get latest git tag
         shell: bash
@@ -52,9 +39,7 @@ jobs:
           fi
 
       - name: Add Tag If Required
-        if:  ${{ env.LATEST_TAG != format('v{0}', steps.pack-version.outputs.RESULT) && steps.pack-version.outputs.RESULT != '' }}
-        env:
-          PACK_VERSION: ${{steps.pack-version.outputs.RESULT}}
+        if:  ${{ env.LATEST_TAG != format('v{0}', env.PACK_VERSION) && env.PACK_VERSION != '' }}
         shell: bash
         run: |
           ###################################################

From cbc013ccc3118423255d590f74d697b8c5b1289e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 20:57:57 -0600
Subject: [PATCH 100/126] gha: cleanup debug lines

---
 .github/workflows/pack-tag_release.yaml | 29 ++++++++-----------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index b24dea3a..8ff93397 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -24,9 +24,6 @@ jobs:
         shell: bash
         id: latest-tag
         run: |
-          ################################
-          # GET LATEST TAG FROM PACK REPO
-          ################################
           if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
             # echo "LATEST_TAG=${LATEST_TAG}" | tee -a ${GITHUB_ENVIRONMENT} # doesn't work
             echo LATEST_TAG=$LATEST_TAG
@@ -38,27 +35,19 @@ jobs:
             echo There are no tags yet.
           fi
 
-      - name: Add Tag If Required
+      - name: Tag HEAD commit if pack.yaml version has changed
         if:  ${{ env.LATEST_TAG != format('v{0}', env.PACK_VERSION) && env.PACK_VERSION != '' }}
         shell: bash
         run: |
-          ###################################################
-          # Create new tag on most recent commit
-          ###################################################
-          echo LATEST_TAG=${{env.LATEST_TAG}}
-          echo PACK_VERSION=v${PACK_VERSION}
-          echo LATEST_TAG != PACK_VERSION!
-          echo Creating tag named v${PACK_VERSION} on most recent commit aka HEAD ...
+          echo "::group::create v${PACK_VERSION} tag on HEAD"
           git tag "v${PACK_VERSION}" HEAD
-          ##############################################################
-          # Verify tag was created and exists on the most recent commit
-          ##############################################################
-          echo Look at the list of tags to make sure it exists:
+          echo "::endgroup::"
+          echo "::group::Display details about tags"
+          set -x
           git tag
-          echo Review most recent commit to make ensure the tag is associated with it:
           git log --pretty='%h %d' --max-count=1
-          ##########################################
-          # Push tag to origin repository (success!)
-          ##########################################
-          echo Pushing new tag to origin...
+          set +x
+          echo "::endgroup::"
+          echo "::group::push v${PACK_VERSION} tag"
           git push origin "v${PACK_VERSION}"
+          echo "::endgroup::"

From c8ad5f9c660f703a5cf86c7e46c9b45f6681b6f1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 22:52:01 -0600
Subject: [PATCH 101/126] gha: cleanup setting env

---
 .github/workflows/pack-tag_release.yaml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index 8ff93397..f023cbc4 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -15,23 +15,21 @@ jobs:
       - name: Get version from pack.yaml
         shell: bash
         run: |
+          echo "::group::install yq to query pack.yaml"
           wget https://github.com/mikefarah/yq/releases/download/v4.16.1/yq_linux_amd64 -O ~/yq
           chmod +x ~/yq
+          echo "::endgroup::"
           PACK_VERSION=$(~/yq eval '.version' pack.yaml)
-          echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> $GITHUB_ENV
+          echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> ${GITHUB_ENV}
 
       - name: Get latest git tag
         shell: bash
         id: latest-tag
         run: |
           if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
-            # echo "LATEST_TAG=${LATEST_TAG}" | tee -a ${GITHUB_ENVIRONMENT} # doesn't work
-            echo LATEST_TAG=$LATEST_TAG
-            echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> $GITHUB_ENV
-            #echo ::set-output name=LATEST_TAG::$LATEST_TAG
+            echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> ${GITHUB_ENV}
           else
-            # echo 'LATEST_TAG=""' >> ${GITHUB_ENVIRONMENT} # doesn't work
-            echo 'LATEST_TAG=""' >> $GITHUB_ENV
+            echo 'LATEST_TAG=""' >> ${GITHUB_ENV}
             echo There are no tags yet.
           fi
 

From 0482d96858de41efcfb9c49283e328240863b0b8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 22:53:37 -0600
Subject: [PATCH 102/126] gha: do full clone during deploy step

---
 .github/workflows/pack-tag_release.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index f023cbc4..ea3a32a2 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -11,6 +11,10 @@ jobs:
     steps:
       - name: Checkout Pack Repo
         uses: actions/checkout@v2
+        with:
+          # A full clone is required to get the tags.
+          # Alternatively, we could use the github API to get a list of tags.
+          fetch-depth: 0
 
       - name: Get version from pack.yaml
         shell: bash

From 786e018b5f4989a95b5746730cfde0e6decb3363 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:04:19 -0600
Subject: [PATCH 103/126] gha: add tag_release workflow outputs

---
 .github/workflows/pack-tag_release.yaml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index ea3a32a2..36955f2b 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -2,12 +2,27 @@ name: Tag Release
 
 on:
   workflow_call:
-#    inputs:
+    outputs:
+      pack_version:
+        description: The current pack version according to pack.yaml
+        value: ${{ jobs.tag_release.outputs.pack_version }}
+      previous_tag:
+        description: The latest tag before running this workflow
+        value: ${{ jobs.tag_release.outputs.previous_tag }}
+      created_tag:
+        description: true if a new tag was pushed to the pack repo
+        value: ${{ jobs.tag_release.outputs.created_tag }}
 
 jobs:
   tag_release:
     runs-on: ubuntu-latest
     name: 'Tag Release'
+
+    outputs:
+      pack_version: ${{ env.PACK_VERSION }}
+      previous_tag: ${{ env.LATEST_TAG }}
+      created_tag: ${{ steps.git-tag.conclusion == 'skipped' }}
+
     steps:
       - name: Checkout Pack Repo
         uses: actions/checkout@v2
@@ -28,7 +43,6 @@ jobs:
 
       - name: Get latest git tag
         shell: bash
-        id: latest-tag
         run: |
           if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
             echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> ${GITHUB_ENV}
@@ -38,6 +52,7 @@ jobs:
           fi
 
       - name: Tag HEAD commit if pack.yaml version has changed
+        id: git-tag
         if:  ${{ env.LATEST_TAG != format('v{0}', env.PACK_VERSION) && env.PACK_VERSION != '' }}
         shell: bash
         run: |

From c161f9fb6a1c090925d4b0cf46007bb5cd5b6cce Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:14:18 -0600
Subject: [PATCH 104/126] minimize required dependencies for semver.py

---
 .circle/semver.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.circle/semver.py b/.circle/semver.py
index 546bf639..c64490f5 100644
--- a/.circle/semver.py
+++ b/.circle/semver.py
@@ -16,8 +16,7 @@
 
 import sys
 import re
-
-import validate
+import yaml
 
 SEMVER_REGEX = re.compile(r"""^(?:0|[1-9]\d*)
                               \.
@@ -31,6 +30,13 @@
 DOUBLE_VERSION_REGEX = re.compile(r"^\d+\.\d+$")
 
 
+def load_yaml_file(path):
+    with open(path, 'r') as stream:
+        text = yaml.safe_load(stream)
+
+    return text
+
+
 def get_semver_string(version):
     if SINGLE_VERSION_REGEX.match(str(version)):
         semver = "%s.0.0" % version
@@ -44,5 +50,5 @@ def get_semver_string(version):
 
 
 if __name__ == '__main__':
-    pack = validate.load_yaml_file(sys.argv[1])
+    pack = load_yaml_file(sys.argv[1])
     print(get_semver_string(pack['version']))

From c9ba51885d070f3057703a140cb075e58786b59c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:31:04 -0600
Subject: [PATCH 105/126] turn tag_release workflow into a composite action

this allows us to use scripts in the ci repo
---
 .../{workflows/pack-tag_release.yaml => actions/tag/action.yaml}  | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/{workflows/pack-tag_release.yaml => actions/tag/action.yaml} (100%)

diff --git a/.github/workflows/pack-tag_release.yaml b/.github/actions/tag/action.yaml
similarity index 100%
rename from .github/workflows/pack-tag_release.yaml
rename to .github/actions/tag/action.yaml

From 699768bd61780d8aa1af814ee458f75913c9c16d Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:35:42 -0600
Subject: [PATCH 106/126] turn tag_release workflow into a composite action

---
 .github/actions/tag/action.yaml | 118 +++++++++++++++-----------------
 1 file changed, 54 insertions(+), 64 deletions(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index 36955f2b..e5165686 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -1,70 +1,60 @@
-name: Tag Release
+---
+name: Tag Pack Repo
+description: |
+  Tag Pack Repo with semver style tag using
+  the version in pack.yaml.
+author: StackStorm
 
-on:
-  workflow_call:
-    outputs:
-      pack_version:
-        description: The current pack version according to pack.yaml
-        value: ${{ jobs.tag_release.outputs.pack_version }}
-      previous_tag:
-        description: The latest tag before running this workflow
-        value: ${{ jobs.tag_release.outputs.previous_tag }}
-      created_tag:
-        description: true if a new tag was pushed to the pack repo
-        value: ${{ jobs.tag_release.outputs.created_tag }}
+outputs:
+  pack_version: ${{ env.PACK_VERSION }}
+  previous_tag: ${{ env.LATEST_TAG }}
+  created_tag: ${{ steps.git-tag.conclusion == 'skipped' }}
 
-jobs:
-  tag_release:
-    runs-on: ubuntu-latest
-    name: 'Tag Release'
+runs:
+  using: "composite"
+  steps:
 
-    outputs:
-      pack_version: ${{ env.PACK_VERSION }}
-      previous_tag: ${{ env.LATEST_TAG }}
-      created_tag: ${{ steps.git-tag.conclusion == 'skipped' }}
+    - name: Checkout Pack Repo
+      uses: actions/checkout@v2
+      with:
+        # A full clone is required to get the tags.
+        # Alternatively, we could use the github API to get a list of tags.
+        fetch-depth: 0
 
-    steps:
-      - name: Checkout Pack Repo
-        uses: actions/checkout@v2
-        with:
-          # A full clone is required to get the tags.
-          # Alternatively, we could use the github API to get a list of tags.
-          fetch-depth: 0
+    - name: Get version from pack.yaml
+      shell: bash
+      run: |
+        echo "::group::install yq to query pack.yaml"
+        wget https://github.com/mikefarah/yq/releases/download/v4.16.1/yq_linux_amd64 -O ~/yq
+        chmod +x ~/yq
+        echo "::endgroup::"
+        PACK_VERSION=$(~/yq eval '.version' pack.yaml)
+        echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> ${GITHUB_ENV}
 
-      - name: Get version from pack.yaml
-        shell: bash
-        run: |
-          echo "::group::install yq to query pack.yaml"
-          wget https://github.com/mikefarah/yq/releases/download/v4.16.1/yq_linux_amd64 -O ~/yq
-          chmod +x ~/yq
-          echo "::endgroup::"
-          PACK_VERSION=$(~/yq eval '.version' pack.yaml)
-          echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> ${GITHUB_ENV}
+    - name: Get latest git tag
+      shell: bash
+      run: |
+        if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
+          echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> ${GITHUB_ENV}
+        else
+          echo 'LATEST_TAG=""' >> ${GITHUB_ENV}
+          echo There are no tags yet.
+        fi
 
-      - name: Get latest git tag
-        shell: bash
-        run: |
-          if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
-            echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> ${GITHUB_ENV}
-          else
-            echo 'LATEST_TAG=""' >> ${GITHUB_ENV}
-            echo There are no tags yet.
-          fi
-
-      - name: Tag HEAD commit if pack.yaml version has changed
-        id: git-tag
-        if:  ${{ env.LATEST_TAG != format('v{0}', env.PACK_VERSION) && env.PACK_VERSION != '' }}
-        shell: bash
-        run: |
-          echo "::group::create v${PACK_VERSION} tag on HEAD"
-          git tag "v${PACK_VERSION}" HEAD
-          echo "::endgroup::"
-          echo "::group::Display details about tags"
-          set -x
-          git tag
-          git log --pretty='%h %d' --max-count=1
-          set +x
-          echo "::endgroup::"
-          echo "::group::push v${PACK_VERSION} tag"
-          git push origin "v${PACK_VERSION}"
-          echo "::endgroup::"
+    - name: Tag HEAD commit if pack.yaml version has changed
+      id: git-tag
+      if:  ${{ env.LATEST_TAG != format('v{0}', env.PACK_VERSION) && env.PACK_VERSION != '' }}
+      shell: bash
+      run: |
+        echo "::group::create v${PACK_VERSION} tag on HEAD"
+        git tag "v${PACK_VERSION}" HEAD
+        echo "::endgroup::"
+        echo "::group::Display details about tags"
+        set -x
+        git tag
+        git log --pretty='%h %d' --max-count=1
+        set +x
+        echo "::endgroup::"
+        echo "::group::push v${PACK_VERSION} tag"
+        git push origin "v${PACK_VERSION}"
+        echo "::endgroup::"

From 0122e0df3d204722327a0efaf91b49c9e4ef45a2 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:42:52 -0600
Subject: [PATCH 107/126] gha: use semver.py to extract semver-style version
 from pack.yaml

---
 .github/actions/tag/action.yaml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index e5165686..053287e6 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -14,6 +14,11 @@ runs:
   using: "composite"
   steps:
 
+    - name: Add checkout path to env context
+      shell: bash
+      run: |
+        echo "CI_DIR=$(realpath ${{ github.action_path }}/../../..)" >> ${GITHUB_ENV}
+
     - name: Checkout Pack Repo
       uses: actions/checkout@v2
       with:
@@ -21,14 +26,18 @@ runs:
         # Alternatively, we could use the github API to get a list of tags.
         fetch-depth: 0
 
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        # use whatever github has in its local cache to speed this up
+        python-version: 3.8
+
+    # semver.py only uses stdlib deps, so no virtualenv required.
+
     - name: Get version from pack.yaml
       shell: bash
       run: |
-        echo "::group::install yq to query pack.yaml"
-        wget https://github.com/mikefarah/yq/releases/download/v4.16.1/yq_linux_amd64 -O ~/yq
-        chmod +x ~/yq
-        echo "::endgroup::"
-        PACK_VERSION=$(~/yq eval '.version' pack.yaml)
+        PACK_VERSION=$(python ${CI_DIR}/.circle/semver.py pack.yaml)
         echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> ${GITHUB_ENV}
 
     - name: Get latest git tag

From 57f10befee6129253f22e9d2cfe2c6f63c47dcb3 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 8 Dec 2021 23:51:39 -0600
Subject: [PATCH 108/126] gha: better division between workflow and composite
 action

---
 .github/actions/tag/action.yaml         | 18 ++---------
 .github/workflows/pack-tag_release.yaml | 42 +++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 15 deletions(-)
 create mode 100644 .github/workflows/pack-tag_release.yaml

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index 053287e6..f3b324ec 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -1,8 +1,9 @@
 ---
 name: Tag Pack Repo
 description: |
-  Tag Pack Repo with semver style tag using
-  the version in pack.yaml.
+  Tag Pack Repo with semver style tag using the version in pack.yaml.
+  You must clone the pack to the default location (fetch-depth: 0)
+  and install python before running this.
 author: StackStorm
 
 outputs:
@@ -19,19 +20,6 @@ runs:
       run: |
         echo "CI_DIR=$(realpath ${{ github.action_path }}/../../..)" >> ${GITHUB_ENV}
 
-    - name: Checkout Pack Repo
-      uses: actions/checkout@v2
-      with:
-        # A full clone is required to get the tags.
-        # Alternatively, we could use the github API to get a list of tags.
-        fetch-depth: 0
-
-    - name: Setup Python
-      uses: actions/setup-python@v2
-      with:
-        # use whatever github has in its local cache to speed this up
-        python-version: 3.8
-
     # semver.py only uses stdlib deps, so no virtualenv required.
 
     - name: Get version from pack.yaml
diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
new file mode 100644
index 00000000..adcd8ab6
--- /dev/null
+++ b/.github/workflows/pack-tag_release.yaml
@@ -0,0 +1,42 @@
+name: Tag Release
+
+on:
+  workflow_call:
+    outputs:
+      pack_version:
+        description: The current pack version according to pack.yaml
+        value: ${{ jobs.tag_release.outputs.pack_version }}
+      previous_tag:
+        description: The latest tag before running this workflow
+        value: ${{ jobs.tag_release.outputs.previous_tag }}
+      created_tag:
+        description: true if a new tag was pushed to the pack repo
+        value: ${{ jobs.tag_release.outputs.created_tag }}
+
+jobs:
+  tag_release:
+    runs-on: ubuntu-latest
+    name: 'Tag Release'
+
+    outputs:
+      pack_version: ${{ steps.tag-pack.outputs.pack_version }}
+      previous_tag: ${{ steps.tag-pack.outputs.previous_tag }}
+      created_tag: ${{ steps.tag-pack.outputs.created_tag }}
+
+    steps:
+      - name: Checkout Pack Repo
+        uses: actions/checkout@v2
+        with:
+          # A full clone is required to get the tags.
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          # use whatever github has in its local cache to speed this up
+          python-version: 3.8
+
+      # eventually replace @gha with @master
+      - name: Check for and tag new version
+        id: tag-pack
+        uses: StackStorm-Exchange/ci/.github/actions/tag@gha

From 86095c0ce3bd463663d90ce95cb76964d29ba464 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 00:06:50 -0600
Subject: [PATCH 109/126] fix bug in validate.py

---
 .circle/validate.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circle/validate.py b/.circle/validate.py
index 046683d5..8e2245e3 100644
--- a/.circle/validate.py
+++ b/.circle/validate.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 from __future__ import print_function
 
+import os
 import sys
 
 import yaml

From 7eaf4ee0fd922985619c99f81b22f68f48b23dd1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 09:39:05 -0600
Subject: [PATCH 110/126] fix composite action syntax

---
 .github/actions/tag/action.yaml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index f3b324ec..699c18d2 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -7,9 +7,15 @@ description: |
 author: StackStorm
 
 outputs:
-  pack_version: ${{ env.PACK_VERSION }}
-  previous_tag: ${{ env.LATEST_TAG }}
-  created_tag: ${{ steps.git-tag.conclusion == 'skipped' }}
+  pack_version:
+    description: The current pack version according to pack.yaml
+    value: ${{ env.PACK_VERSION }}
+  previous_tag:
+    description: The latest tag before running this workflow
+    value: ${{ env.LATEST_TAG }}
+  created_tag:
+    description: true if a new tag was pushed to the pack repo
+    value: ${{ steps.git-tag.conclusion == 'skipped' }}
 
 runs:
   using: "composite"

From a5224dbc929a3f57cfe402aba458279930ad99e1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 09:42:13 -0600
Subject: [PATCH 111/126] gha: install deps for semver.py

---
 .github/actions/tag/action.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index 699c18d2..bcf21c6d 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -26,7 +26,10 @@ runs:
       run: |
         echo "CI_DIR=$(realpath ${{ github.action_path }}/../../..)" >> ${GITHUB_ENV}
 
-    # semver.py only uses stdlib deps, so no virtualenv required.
+    - name: Install semver.py deps
+      shell: bash
+      run: |
+        pip install --user pyyaml
 
     - name: Get version from pack.yaml
       shell: bash

From f7335bf1f592a9d48a864c0c908f836616656125 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 09:43:41 -0600
Subject: [PATCH 112/126] fix tee usage

---
 .github/actions/tag/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index bcf21c6d..dfdb2e54 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -35,13 +35,13 @@ runs:
       shell: bash
       run: |
         PACK_VERSION=$(python ${CI_DIR}/.circle/semver.py pack.yaml)
-        echo "PACK_VERSION=${PACK_VERSION}" | tee -a >> ${GITHUB_ENV}
+        echo "PACK_VERSION=${PACK_VERSION}" | tee -a ${GITHUB_ENV}
 
     - name: Get latest git tag
       shell: bash
       run: |
         if LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`); then
-          echo "LATEST_TAG=${LATEST_TAG}" | tee -a >> ${GITHUB_ENV}
+          echo "LATEST_TAG=${LATEST_TAG}" | tee -a ${GITHUB_ENV}
         else
           echo 'LATEST_TAG=""' >> ${GITHUB_ENV}
           echo There are no tags yet.

From c46da9ece7e578ace67b1b070b1e77720763a6dd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 09:47:52 -0600
Subject: [PATCH 113/126] gha: improve output grouping

---
 .github/actions/tag/action.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index dfdb2e54..c8545797 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -53,7 +53,9 @@ runs:
       shell: bash
       run: |
         echo "::group::create v${PACK_VERSION} tag on HEAD"
+        set -x
         git tag "v${PACK_VERSION}" HEAD
+        set +x
         echo "::endgroup::"
         echo "::group::Display details about tags"
         set -x

From a2f1d75b96ca7d9e50418e52dad282c0e2dc689e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 10:50:08 -0600
Subject: [PATCH 114/126] fix created_tag logic

---
 .github/actions/tag/action.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/actions/tag/action.yaml b/.github/actions/tag/action.yaml
index c8545797..d3be5e74 100644
--- a/.github/actions/tag/action.yaml
+++ b/.github/actions/tag/action.yaml
@@ -15,7 +15,8 @@ outputs:
     value: ${{ env.LATEST_TAG }}
   created_tag:
     description: true if a new tag was pushed to the pack repo
-    value: ${{ steps.git-tag.conclusion == 'skipped' }}
+    # skipped if task did not run
+    value: ${{ steps.git-tag.conclusion == 'success' }}
 
 runs:
   using: "composite"

From a12e6367d438d0152504bb81b3f1002a5c09023d Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 10:50:38 -0600
Subject: [PATCH 115/126] gha: add pack repo metadata update action

---
 .github/actions/repo-meta/action.yaml | 67 +++++++++++++++++++++++++++
 .github/actions/repo-meta/extract.py  | 41 ++++++++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 .github/actions/repo-meta/action.yaml
 create mode 100644 .github/actions/repo-meta/extract.py

diff --git a/.github/actions/repo-meta/action.yaml b/.github/actions/repo-meta/action.yaml
new file mode 100644
index 00000000..6cd01051
--- /dev/null
+++ b/.github/actions/repo-meta/action.yaml
@@ -0,0 +1,67 @@
+---
+name: Update Pack Repo Metadata
+description: |
+  Update Pack Repo Metadata based on pack.yaml.
+  You must clone the pack to the default location
+  and install python before running this.
+author: StackStorm
+
+inputs:
+  homepage:
+    required: false
+    # env vars are available here (see Update repo homepage task)
+    default: "https://exchange.stackstorm.com/#${PACK_NAME}"
+
+outputs:
+  pack_description:
+    description: The current pack description according to pack.yaml
+    value: ${{ env.PACK_DESCRIPTION }}
+  original_description:
+    description: The original description before running this workflow
+    value: ${{ env.ORIGINAL_DESCRIPTION }}
+  original_homepage:
+    description: The original homepage before running this workflow
+    value: ${{ env.ORIGINAL_HOMEPAGE }}
+  updated:
+    description: true if metadata was updated on the pack repo
+    # skipped if task did not run
+    value: ${{ steps.gh-description.conclusion == 'success' || steps.gh-homepage.conclusion == 'success' }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Install extract.py deps
+      shell: bash
+      run: |
+        pip install --user pyyaml
+
+    - name: Get metadata from pack.yaml
+      shell: bash
+      run: |
+        PACK_NAME=$(python ${ACTION_PATH}/extract.py pack.yaml name)
+        echo "PACK_NAME=${PACK_NAME}" | tee -a ${GITHUB_ENV}
+        PACK_DESCRIPTION=$(python ${ACTION_PATH}/extract.py pack.yaml description)
+        echo "PACK_DESCRIPTION=${PACK_DESCRIPTION}" | tee -a ${GITHUB_ENV}
+
+    - name: Get latest repo metadata
+      shell: bash
+      run: |
+        ORIGINAL_DESCRIPTION=$(gh repo view --json description -q .description)
+        echo "ORIGINAL_DESCRIPTION=${ORIGINAL_DESCRIPTION}" | tee -a ${GITHUB_ENV}
+        ORIGINAL_HOMEPAGE=$(gh repo view --json homepage -q .homepage)
+        echo "ORIGINAL_HOMEPAGE=${ORIGINAL_HOMEPAGE}" | tee -a ${GITHUB_ENV}
+
+    - name: Update repo description
+      id: gh-description
+      if:  ${{ env.ORIGINAL_DESCRIPTION != env.PACK_DESCRIPTION && env.PACK_DESCRIPTION != '' }}
+      shell: bash
+      run: |
+        gh api -X PATCH 'repos/{owner}/{repo}' -f "description=${PACK_DESCRIPTION}"
+
+    - name: Update repo homepage
+      id: gh-homepage
+      if:  ${{ env.ORIGINAL_HOMEPAGE != inputs.homepage && inputs.homepage != '' }}
+      shell: bash
+      run: |
+        gh api -X PATCH 'repos/{owner}/{repo}' -f "homepage=${{ inputs.homepage }}"
diff --git a/.github/actions/repo-meta/extract.py b/.github/actions/repo-meta/extract.py
new file mode 100644
index 00000000..7e8f1496
--- /dev/null
+++ b/.github/actions/repo-meta/extract.py
@@ -0,0 +1,41 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import typing
+import sys
+import yaml
+import json
+
+
+def load_yaml_file(path):
+    with open(path, "r") as stream:
+        text = yaml.safe_load(stream)
+
+    return text
+
+
+def dump(value):
+    if isinstance(value, typing.Collection):
+        return json.dumps(value)
+    else:  # simple int or str (without json quotes)
+        return value
+
+
+if __name__ == "__main__":
+    pack = load_yaml_file(sys.argv[1])
+    key = sys.argv[2]
+    value = pack.get(key, "")
+    print(dump(value))

From 732c37b847a64f23a7ded299dfa4270af65de530 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 11:03:13 -0600
Subject: [PATCH 116/126] gha: add pack-repo_meta.yaml workflow

---
 .github/actions/repo-meta/action.yaml |  3 ++
 .github/workflows/pack-repo_meta.yaml | 48 +++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 .github/workflows/pack-repo_meta.yaml

diff --git a/.github/actions/repo-meta/action.yaml b/.github/actions/repo-meta/action.yaml
index 6cd01051..cda101db 100644
--- a/.github/actions/repo-meta/action.yaml
+++ b/.github/actions/repo-meta/action.yaml
@@ -16,6 +16,9 @@ outputs:
   pack_description:
     description: The current pack description according to pack.yaml
     value: ${{ env.PACK_DESCRIPTION }}
+  repo_homepage:
+    description: The resulting repo homepage
+    value: ${{ inputs.homepage }}
   original_description:
     description: The original description before running this workflow
     value: ${{ env.ORIGINAL_DESCRIPTION }}
diff --git a/.github/workflows/pack-repo_meta.yaml b/.github/workflows/pack-repo_meta.yaml
new file mode 100644
index 00000000..eef22ec0
--- /dev/null
+++ b/.github/workflows/pack-repo_meta.yaml
@@ -0,0 +1,48 @@
+name: Repo Metadata
+
+on:
+  workflow_call:
+    outputs:
+      pack_description:
+        description: The current pack description according to pack.yaml
+        value: ${{ jobs.repo_meta.outputs.pack_description }}
+      repo_homepage:
+        description: The resulting repo homepage
+        value: ${{ jobs.repo_meta.outputs.repo_homepage }}
+      original_description:
+        description: The original description before running this workflow
+        value: ${{ jobs.repo_meta.outputs.original_description }}
+      original_homepage:
+        description: The original homepage before running this workflow
+        value: ${{ jobs.repo_meta.outputs.original_homepage }}
+      updated:
+        description: true if metadata was updated on the pack repo
+        value: ${{ jobs.repo_meta.outputs.updated }}
+
+jobs:
+  repo_meta:
+    runs-on: ubuntu-latest
+    name: 'Repo Metadata'
+
+    outputs:
+      pack_description: ${{ steps.repo-meta.outputs.pack_description }}
+      original_description: ${{ steps.repo-meta.outputs.original_description }}
+      repo_homepage: ${{ steps.repo-meta.outputs.repo_homepage }}
+      original_homepage: ${{ steps.repo-meta.outputs.original_homepage }}
+      updated_metadata: ${{ steps.repo-meta.outputs.updated }}
+
+    steps:
+      - name: Checkout Pack Repo
+        uses: actions/checkout@v2
+        # only the latest commit is needed
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          # use whatever github has in its local cache to speed this up
+          python-version: 3.8
+
+      # eventually replace @gha with @master
+      - name: Update repo metadata
+        id: repo-meta
+        uses: StackStorm-Exchange/ci/.github/actions/repo-meta@gha

From 38f7c5c8a09a4f4607f2a44fc7573abb60cc2a16 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 11:13:28 -0600
Subject: [PATCH 117/126] gha: correct GITHUB_ACTION_PATH var

---
 .github/actions/repo-meta/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/repo-meta/action.yaml b/.github/actions/repo-meta/action.yaml
index cda101db..8e3c56e8 100644
--- a/.github/actions/repo-meta/action.yaml
+++ b/.github/actions/repo-meta/action.yaml
@@ -42,9 +42,9 @@ runs:
     - name: Get metadata from pack.yaml
       shell: bash
       run: |
-        PACK_NAME=$(python ${ACTION_PATH}/extract.py pack.yaml name)
+        PACK_NAME=$(python ${GITHUB_ACTION_PATH}/extract.py pack.yaml name)
         echo "PACK_NAME=${PACK_NAME}" | tee -a ${GITHUB_ENV}
-        PACK_DESCRIPTION=$(python ${ACTION_PATH}/extract.py pack.yaml description)
+        PACK_DESCRIPTION=$(python ${GITHUB_ACTION_PATH}/extract.py pack.yaml description)
         echo "PACK_DESCRIPTION=${PACK_DESCRIPTION}" | tee -a ${GITHUB_ENV}
 
     - name: Get latest repo metadata

From cf2f98743838c25071c2780d28e3abd50326f25f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 12:44:11 -0600
Subject: [PATCH 118/126] switch from gha branch to master

---
 .github/workflows/index-update.yaml        | 4 ++--
 .github/workflows/pack-build_and_test.yaml | 9 ++++-----
 .github/workflows/pack-repo_meta.yaml      | 5 +++--
 .github/workflows/pack-tag_release.yaml    | 3 +--
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 5ab360c2..99eec97c 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -109,14 +109,14 @@ jobs:
         fetch-depth: 1
 
     - name: Install APT Dependencies
-      uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@gha
+      uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@master
       with:
         cache-version: ${{ inputs.apt-cache-version }}
         # this has dependencies for icon optimization
         extra-apt-packages-file: ci/.github/actions/apt-dependencies/index-apt-packages.txt
 
     - name: Install Python Dependencies
-      uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
+      uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@master
       with:
         mode: index # ie: skip pack-specific deps
         cache-version: ${{ inputs.py-cache-version }}
diff --git a/.github/workflows/pack-build_and_test.yaml b/.github/workflows/pack-build_and_test.yaml
index 2855558b..9ae7ae54 100644
--- a/.github/workflows/pack-build_and_test.yaml
+++ b/.github/workflows/pack-build_and_test.yaml
@@ -31,23 +31,22 @@ jobs:
           - python-version-short: 3.6
             python-version: 3.6.13
     steps:
-      # eventually replace @gha with @master
       - name: Checkout Pack Repo and CI Repos
-        uses: StackStorm-Exchange/ci/.github/actions/checkout@gha
+        uses: StackStorm-Exchange/ci/.github/actions/checkout@master
         
       - name: Install APT Dependencies
-        uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@gha
+        uses: StackStorm-Exchange/ci/.github/actions/apt-dependencies@master
         with:
           cache-version: ${{ inputs.apt-cache-version }}
 
       - name: Install Python Dependencies
-        uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@gha
+        uses: StackStorm-Exchange/ci/.github/actions/py-dependencies@master
         with:
           cache-version: ${{ inputs.py-cache-version }}
           python-version: ${{ matrix.python-version }}
 
       - name: Run pack tests
-        uses: StackStorm-Exchange/ci/.github/actions/test@gha
+        uses: StackStorm-Exchange/ci/.github/actions/test@master
         with:
           enable-common-libs: ${{ inputs.enable-common-libs }}
 
diff --git a/.github/workflows/pack-repo_meta.yaml b/.github/workflows/pack-repo_meta.yaml
index eef22ec0..9a7871d3 100644
--- a/.github/workflows/pack-repo_meta.yaml
+++ b/.github/workflows/pack-repo_meta.yaml
@@ -1,4 +1,6 @@
 name: Repo Metadata
+# TODO: this workflow is not function yet because GITHUB_TOKEN
+#       is not allowed to edit repo metadata (so far).
 
 on:
   workflow_call:
@@ -42,7 +44,6 @@ jobs:
           # use whatever github has in its local cache to speed this up
           python-version: 3.8
 
-      # eventually replace @gha with @master
       - name: Update repo metadata
         id: repo-meta
-        uses: StackStorm-Exchange/ci/.github/actions/repo-meta@gha
+        uses: StackStorm-Exchange/ci/.github/actions/repo-meta@master
diff --git a/.github/workflows/pack-tag_release.yaml b/.github/workflows/pack-tag_release.yaml
index adcd8ab6..e555d4b5 100644
--- a/.github/workflows/pack-tag_release.yaml
+++ b/.github/workflows/pack-tag_release.yaml
@@ -36,7 +36,6 @@ jobs:
           # use whatever github has in its local cache to speed this up
           python-version: 3.8
 
-      # eventually replace @gha with @master
       - name: Check for and tag new version
         id: tag-pack
-        uses: StackStorm-Exchange/ci/.github/actions/tag@gha
+        uses: StackStorm-Exchange/ci/.github/actions/tag@master

From 2319b64221e1fa83880f5627eb99c59979d0761c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 12:57:43 -0600
Subject: [PATCH 119/126] remove old comments

---
 .github/actions/apt-dependencies/apt-packages.txt | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/actions/apt-dependencies/apt-packages.txt b/.github/actions/apt-dependencies/apt-packages.txt
index 7800030f..68ad6cea 100644
--- a/.github/actions/apt-dependencies/apt-packages.txt
+++ b/.github/actions/apt-dependencies/apt-packages.txt
@@ -8,10 +8,3 @@ libldap2-dev
 libsasl2-dev
 # st2 also installs
 #libssl-dev libyaml-dev ldap-utils
-
-# not sure if we still need this
-#python3-dev
-
-# deploy dependencies (maybe use pre-installed imagemagick instead)
-#gmic
-#optipng

From 7143e1f5178f93cc885458708c13a03afbaff2ef Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 13:06:48 -0600
Subject: [PATCH 120/126] drop unrelated changes

---
 utils/exchange-bootstrap.sh | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/utils/exchange-bootstrap.sh b/utils/exchange-bootstrap.sh
index 21c9831c..fea53a54 100755
--- a/utils/exchange-bootstrap.sh
+++ b/utils/exchange-bootstrap.sh
@@ -39,7 +39,6 @@ then
 fi
 
 EXCHANGE_ORG="${EXCHANGE_ORG:-StackStorm-Exchange}"
-EXCHANGE_TEAM="${EXCHANGE_TEAM:-tsc}"
 EXCHANGE_PREFIX="${EXCHANGE_PREFIX:-stackstorm}"
 PACK="${1/${EXCHANGE_PREFIX}-/}"  # Ensure that PACK is just the bare pack name
 REPO_ALIAS=${PACK}
@@ -169,13 +168,6 @@ then
        "https://api.github.com/repos/${EXCHANGE_ORG}/${REPO_NAME}/hooks"
 fi
 
-echo "GitHub: Allow ${EXCHANGE_TEAM} team to 'maintain' ${EXCHANGE_ORG}/${REPO_NAME}"
-curl -sS --fail -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" -X PUT \
-     --header "Content-Type: application/json" \
-     --header "Accept: application/vnd.github.v3+json" \
-     -d '{"permission": "maintain"}' \
-     "https://api.github.com/orgs/${EXCHANGE_ORG}/teams/${EXCHANGE_TEAM}/repos/${EXCHANGE_ORG}/${REPO_NAME}"
-
 # This will open a private tab in the user's browser to the PAT page, with the
 # name field already filled in and the scope fields already checked, and direct
 # the user to click the "Generate Token" button, then copy and paste the
@@ -185,12 +177,12 @@ curl -sS --fail -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" -X PUT \
 # PAT
 ${CI_REPO_ROOT}/tools/reset_github_user_token_and_update_circleci.sh --set-user $PACK
 
-# XXX: this API request was failing, but it is working now. Not sure why.
-# CircleCI: follow the project (we need this, or CircleCI won't watch the repo)
-echo "CircleCI: Following the project (enables CircleCI builds)"
-curl -v -sS --fail -X POST \
-    --header "Circle-Token: ${CIRCLECI_TOKEN}" \
-    "https://circleci.com/api/v1.1/project/github/${EXCHANGE_ORG}/${REPO_NAME}/follow"
+# NO longer needed, this API request fails everytime we call it
+# # CircleCI: follow the project
+# echo "CircleCI: Following the project"
+# curl -v -sS --fail -X POST \
+#      --header "Circle-Token: ${CIRCLECI_TOKEN}" \
+#      "https://circleci.com/api/v1.1/project/github/${EXCHANGE_ORG}/${REPO_NAME}/follow"
 
 
 # CircleCI: upload the read-write key

From ce10f59e91ded53060812a6833f2bc79f4ac2d92 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 15:03:02 -0600
Subject: [PATCH 121/126] satisfy lint

---
 .circle/index.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circle/index.py b/.circle/index.py
index 018e2c7f..a5971d18 100644
--- a/.circle/index.py
+++ b/.circle/index.py
@@ -32,7 +32,8 @@
 GITHUB_USERNAME = os.environ.get('MACHINE_USER')
 # TODO: drop MACHINE_PASSWORD once we drop support for CircleCI. Keep GH_TOKEN.
 GITHUB_PASSWORD = os.environ.get("MACHINE_PASSWORD", os.environ.get("GH_TOKEN"))
-# TODO: drop ACTIVE_PACK_NAME once we drop support for CircleCI. Only used for CircleCI-specific error message.
+# TODO: drop ACTIVE_PACK_NAME once we drop support for CircleCI.
+#       Only used for CircleCI-specific error message.
 ACTIVE_PACK_NAME = os.environ.get('PACK_NAME', "unknown")
 
 SESSION = requests.Session()

From b4bb6270cab716ba8a8562334dbf74c9c0d921da Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 15:04:21 -0600
Subject: [PATCH 122/126] switch back to StackStorm-Exchange/exchange-tools
 instead of fork

---
 .github/workflows/index-update.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index 99eec97c..f6615769 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -90,11 +90,7 @@ jobs:
     - name: Checkout exchange-tools repo
       uses: actions/checkout@v2
       with:
-        # TODO: switch to StackStorm-Exchange once these are merged:
-        # https://github.com/StackStorm-Exchange/exchange-tools/pull/3
-        # https://github.com/StackStorm-Exchange/exchange-tools/pull/2
-        #repository: StackStorm-Exchange/exchange-tools
-        repository: cognifloyd/exchange-tools
+        repository: StackStorm-Exchange/exchange-tools
         ref: ${{ inputs.exchange_tools_branch }}
         path: exchange-tools
         fetch-depth: 1

From 5f5f057544bb7472a9023b237fa84bfbca76f7d8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 15:05:48 -0600
Subject: [PATCH 123/126] gha: switch branch of checkouts to master

---
 .github/workflows/index-update.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/index-update.yaml b/.github/workflows/index-update.yaml
index f6615769..191fdc47 100644
--- a/.github/workflows/index-update.yaml
+++ b/.github/workflows/index-update.yaml
@@ -18,13 +18,11 @@ on:
       ci_branch:
         required: false
         type: string
-        # TODO: switch to master
-        default: gha
+        default: master
       exchange_tools_branch:
         required: false
         type: string
-        # TODO: switch to master
-        default: add-more-tools
+        default: master
       packs_org:
         required: false
         type: string

From d26754f3b691b8375f4428fa0f171d53c144c570 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 15:13:11 -0600
Subject: [PATCH 124/126] silence pylint about new file

---
 .github/actions/repo-meta/extract.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/actions/repo-meta/extract.py b/.github/actions/repo-meta/extract.py
index 7e8f1496..8ac1a91c 100644
--- a/.github/actions/repo-meta/extract.py
+++ b/.github/actions/repo-meta/extract.py
@@ -14,14 +14,15 @@
 # limitations under the License.
 from __future__ import print_function
 
-import typing
+import json
 import sys
+import typing
+
 import yaml
-import json
 
 
 def load_yaml_file(path):
-    with open(path, "r") as stream:
+    with open(path, "r", encoding="utf8") as stream:
         text = yaml.safe_load(stream)
 
     return text
@@ -29,9 +30,9 @@ def load_yaml_file(path):
 
 def dump(value):
     if isinstance(value, typing.Collection):
-        return json.dumps(value)
-    else:  # simple int or str (without json quotes)
-        return value
+        value = json.dumps(value)
+    # else it is a simple int or str (pass as is w/o json quotes)
+    return value
 
 
 if __name__ == "__main__":

From 0e111b840027e0a1904f02e925216e4e600a345b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 15:16:13 -0600
Subject: [PATCH 125/126] satisfy pylint for files used in gha

---
 .circle/index.py    | 4 ++--
 .circle/semver.py   | 2 +-
 .circle/validate.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.circle/index.py b/.circle/index.py
index a5971d18..db1339ac 100644
--- a/.circle/index.py
+++ b/.circle/index.py
@@ -75,7 +75,7 @@ def build_index(path_glob, output_path):
     counter = 0
     failed_count = 0
     for filename in generator:
-        with open(filename, 'r') as pack:
+        with open(filename, 'r', encoding="utf8") as pack:
             pack_meta = yaml.safe_load(pack)
 
         pack_name = pack_meta['name']
@@ -114,7 +114,7 @@ def build_index(path_glob, output_path):
     result['metadata']['hash'] = data_hash.hexdigest()
 
     output_path = os.path.expanduser(os.path.join(output_path, 'index.json'))
-    with open(output_path, 'w') as outfile:
+    with open(output_path, 'w', encoding="utf8") as outfile:
         json.dump(result, outfile, indent=4, sort_keys=True,
                   separators=(',', ': '))
 
diff --git a/.circle/semver.py b/.circle/semver.py
index c64490f5..0dc86fb1 100644
--- a/.circle/semver.py
+++ b/.circle/semver.py
@@ -31,7 +31,7 @@
 
 
 def load_yaml_file(path):
-    with open(path, 'r') as stream:
+    with open(path, 'r', encoding="utf8") as stream:
         text = yaml.safe_load(stream)
 
     return text
diff --git a/.circle/validate.py b/.circle/validate.py
index 8e2245e3..65df7915 100644
--- a/.circle/validate.py
+++ b/.circle/validate.py
@@ -28,7 +28,7 @@
 
 
 def load_yaml_file(path):
-    with open(path, 'r') as stream:
+    with open(path, 'r', encoding="utf8") as stream:
         text = yaml.safe_load(stream)
 
     return text

From b911fde4830d09152a167e49c72e99ec859d32f3 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 9 Dec 2021 16:53:35 -0600
Subject: [PATCH 126/126] use an f-string

---
 .circle/index.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circle/index.py b/.circle/index.py
index db1339ac..32c0c74c 100644
--- a/.circle/index.py
+++ b/.circle/index.py
@@ -137,10 +137,10 @@ def build_index(path_glob, output_path):
         # If an issue is reported on GitHub Actions, update this error message
         # to explain common causes and how to fix them.
         failed_message = (
-            ', {failed_count} packs failed to update.\n'
+            f', {failed_count} packs failed to update.\n'
             'Please investigate why this failed and report an issue on:\n'
-            '  https://github.com/{exchange_name}/ci\n'
-        ).format(failed_count=failed_count, exchange_name=EXCHANGE_NAME)
+            f'  https://github.com/{EXCHANGE_NAME}/ci\n'
+        )
 
     print('')
     print('Processed %s packs%s.' % (counter, failed_message))