Automated generation of two test central repositories (#41)

* Add script to generate two test central repos

Two test central repos are created that can be run using docker:
- a small central repo with 4 plugins
- a large central repo with 99 plugins (but only 4 can be installed)

This allows to test the new Central Repo feature.

See the hack/central-repo/README.md for details.

Makefile

@@ -204,6 +204,19 @@ choco-package: ## Build a Chocolatey package
 test: fmt ## Run Tests
 	${GO} test ./... -timeout 60m -race -coverprofile coverage.txt ${GOTEST_VERBOSE}
 
+.PHONY: start-test-central-repo
+start-test-central-repo: stop-test-central-repo ## Starts up a test central repository locally with docker
+	if [ ! -d $(ROOT_DIR)/hack/central-repo/registry-content ]; then \
+		(cd $(ROOT_DIR)/hack/central-repo && tar xzf registry-content.bz2 || true;) \
+	fi
+	docker run --rm -d -p 9876:5000 --name central \
+		-v $(ROOT_DIR)/hack/central-repo/registry-content:/var/lib/registry \
+		mirror.gcr.io/library/registry:2
+
+.PHONY: stop-test-central-repo
+stop-test-central-repo: ## Stops and removes the local test central repository
+	docker container stop central 2> /dev/null || true
+
 .PHONY: fmt
 fmt: $(GOIMPORTS) ## Run goimports
 	$(GOIMPORTS) -w -local github.com/vmware-tanzu ./

hack/central-repo/.gitignore

@@ -0,0 +1 @@
+registry-content

hack/central-repo/README.md

@@ -0,0 +1,66 @@
+# Test Central Repository
+
+## Using the test Central Repo
+
+From the root directory of the tanzu-cli repo, run `make start-test-central-repo`.  This will start an OCI registry
+containing two test central repositories:
+
+1. a small repo with 4 plugins which can be simpler to test with: localhost:9876/tanzu-cli/plugins/central:small
+1. a large one with 99 plugins: localhost:9876/tanzu-cli/plugins/central:large
+
+The steps to follow to use this test central repo are:
+
+1. start the test repo
+1. enable the central repository feature flag (this is a temporary flag which will be removed eventually)
+1. configure the plugin source for the test central repo
+1. allow the use of a local registry with the `ALLOWED_REGISTRY` variable
+
+```bash
+cd tanzu-cli
+make build
+make start-test-central-repo
+tz config set features.global.central-repository true
+tz plugin source add -n default -t oci -u localhost:9876/tanzu-cli/plugins/central:small || true
+tz plugin source update default -t oci -u localhost:9876/tanzu-cli/plugins/central:small
+export ALLOWED_REGISTRY=localhost:9876
+
+tz plugin list
+tz plugin install twotargets --target tmc
+```
+
+To use the large test central repo instead:
+
+```bash
+tz plugin source update default -t oci -u localhost:9876/tanzu-cli/plugins/central:large
+```
+
+To stop the central repos: `make stop-test-central-repo`.
+
+Note that the registry is pre-configured through the existing `hack/central-repo/registry-content` directory.
+
+Limitations:
+
+1. Only certain plugins can be installed.  This avoids having to push binaries for all plugins, for efficiency.
+1. The plugins that can be installed are two plugins named `twotargets`, and `kubeplugin` and `tmcplugin`.
+
+## Generate the test Central Repo
+
+This should only be done if a new version of the registry should be generated.
+Normally, the content of the repo is persisted on disk under `hack/central-repo/registry-content`
+which avoids having to regenerate the repo.  When using the Makefile target `make start-test-central-repo`
+the directory `registry-content` is extracted from the `registry-content.bz2` tarball.
+A tarball is used to dramatically reduce the size saved in git.
+
+If it is necessary to re-generate a new test central repo, it took around 1 minute on a Mac M1.
+The procedure follows:
+
+```bash
+cd tanzu-cli
+make stop-test-central-repo
+cd hack/central-repo
+\rm -rf registry-content registry-content.bz2
+./generate-central.sh
+tar cjf registry-content.bz2 registry-content
+git add registry-content.bz2
+git commit -m "Regenerated the test central repos"
+```

hack/central-repo/fakeplugin.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2023 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Minimally Viable Dummy Tanzu CLI 'Plugin'
+
+info() {
+   cat << EOF
+{
+  "name": "__NAME__",
+  "target": "__TARGET__",
+  "description": "__NAME__ functionality",
+  "version": "__VERSION__",
+  "buildSHA": "01234567",
+  "group": "System",
+  "hidden": false,
+  "aliases": [],
+  "completionType": 0
+}
+EOF
+  exit 0
+}
+
+case "$1" in
+    info)  $1 "$@";;
+    help|-h) cat << EOF
+Plugin __NAME__ for __TARGET__ does nothing
+EOF
+       ;;
+    *) cat << EOF
+Plugin __NAME__ for __TARGET__ does nothing
+EOF
+    ;;
+esac

hack/central-repo/generate-central.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+ROOT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd)
+
+# Start a registry
+make -C $ROOT_DIR/../.. start-test-central-repo
+rm -f /tmp/central.db
+
+# Build a run a docker image that contains imgpkg and sqlite3
+# to avoid having to install them locally
+echo "========================================"
+echo "Setting up image with imgpkg and sqlite3"
+echo "========================================"
+IMAGE=build-central
+docker build -t ${IMAGE} ${ROOT_DIR} -f - <<- EOF
+   FROM ubuntu
+   RUN apt update && \
+       apt install -y curl \
+                      sqlite3 \
+                      libdigest-sha-perl
+
+    RUN mkdir /tmp/carvel/ && \
+        curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=/tmp/carvel bash && \
+        install /tmp/carvel/imgpkg /usr/bin
+
+   WORKDIR /work
+   COPY upload-plugins.sh .
+   COPY fakeplugin.sh .
+EOF
+
+# Generate both the small and large test central repositories
+docker run --rm ${IMAGE} ./upload-plugins.sh --fast host.docker.internal:9876/tanzu-cli/plugins/central:small 
+docker run --rm ${IMAGE} ./upload-plugins.sh host.docker.internal:9876/tanzu-cli/plugins/central:large
+
+# Stop the registry
+make -C $ROOT_DIR/../.. stop-test-central-repo

hack/central-repo/upload-plugins.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 VMware, Inc. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+ROOT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}"); pwd)
+
+usage() {
+    echo "generate_central.sh [-h | -d | --dry-run | --fast] REPO_URI"
+    echo
+    echo "Push 99 plugins to a test repository located at REPO_URI (e.g., localhost:9998/central:small"
+    echo "  -h               Print this help"
+    echo "  -d, --dry-run    Only print the commands that would be executed"
+    echo "      --fast       Only install 4 plugins"
+    exit 0
+}
+
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+    usage
+fi
+
+dry_run=""
+if [ "$1" = "-d" ] || [ "$1" = "--dry-run" ]; then
+    dry_run=echo
+    shift
+fi
+
+fast=off
+if [ "$1" = "--fast" ]; then
+    fast=on
+    shift
+fi
+
+if [ $# -eq 0 ] || [[ $1 == "-"* ]]; then
+    usage
+fi
+
+content_image=$1
+repoBasePath=$(dirname $content_image)
+database=/tmp/central.db
+publisher="vmware/tkg"
+
+# Create db table
+cat << EOF | sqlite3 -batch $database
+CREATE TABLE IF NOT EXISTS "PluginBinaries" (
+		"PluginName"         TEXT NOT NULL,
+		"Target"             TEXT NOT NULL,
+		"RecommendedVersion" TEXT NOT NULL,
+		"Version"            TEXT NOT NULL,
+		"Hidden"             INTEGER NOT NULL,
+		"Description"        TEXT NOT NULL,
+		"Publisher"          TEXT NOT NULL,
+		"Vendor"             TEXT NOT NULL,
+		"OS"                 TEXT NOT NULL,
+		"Architecture"       TEXT NOT NULL,
+		"Digest"             TEXT NOT NULL,
+		"URI"                TEXT NOT NULL,
+		PRIMARY KEY("PluginName", "Target", "Version", "OS", "Architecture")
+	);
+EOF
+
+addPlugin() {
+    name=$1
+    target=$2
+
+    tmpPluginPhase1="/tmp/fakeplugin1.sh"
+    tmpPluginPhase2="/tmp/fakeplugin2.sh"
+
+    # Start preparing the plugin file with the name and target.
+    # Start here to avoir repeating in the loop.
+    sed -e "s/__NAME__/$name/" -e "s/__TARGET__/$target/" $ROOT_DIR/fakeplugin.sh > $tmpPluginPhase1
+
+    # Define 10 versions for the plugin
+    for v in {0..9}; do
+        version="v$v.$v.$v"
+
+        # Put printout to show progress
+        echo "Inserting $name version $version for target $target"
+
+        # Create the plugin file with the correct version
+        sed -e "s/__VERSION__/$version/" $tmpPluginPhase1 > $tmpPluginPhase2
+        digest=$(sha256sum $tmpPluginPhase2 | cut -f1 -d' ')
+
+        for os in darwin linux windows; do
+            for arch in amd64 arm64; do
+                if [ $arch = arm64 ] && [ $os != darwin ]; then
+                    # Only support darwin with arm64 for now
+                    continue
+                fi
+
+                image_path=$publisher/$os/$arch/$target/$name
+
+                sql_cmd="INSERT INTO PluginBinaries VALUES('$name','$target','v9.9.9','$version','FALSE','Desc for $name','TKG','VMware','$os','$arch','$digest','$image_path:$version');"
+                if [ "$dry_run" = "echo" ]; then
+                    echo $sql_cmd 
+                else 
+                    echo $sql_cmd | sqlite3 -batch $database
+                fi
+
+                # For efficiency, only push the plugin binaries of the two latest versions for 4 plugins
+                # The plugins are: twotargets (for both targets), kubeplugin, tmcplugin 
+                if [ $name = "twotargets" ] || [ $name = "kubeplugin" ] || [ $name = "tmcplugin" ]; then
+                    if [ $version = "v8.8.8" ] || [ $version = "v9.9.9" ]; then
+                        ${dry_run} imgpkg push -i $repoBasePath/$image_path:$version -f $tmpPluginPhase2 --registry-insecure
+                    fi
+                fi
+            done
+        done
+    done
+}
+
+# Push the plugins that have two targets
+addPlugin twotargets tmc
+addPlugin twotargets k8s
+
+# Push another two plugins for the small repo
+addPlugin kubeplugin k8s
+addPlugin tmcplugin tmc
+
+# Push 95 more plugins, for a total of 99
+if [ $fast = "off" ]; then
+    for idx in {0..94}; do
+        target_rand=$(($RANDOM % 3))
+        case $target_rand in
+        0) target=global
+           ;;
+        1) target=k8s
+           ;;
+        2) target=tmc
+           ;;
+        esac
+
+        addPlugin plugin$idx $target
+    done
+fi
+
+# Push content file
+${dry_run} imgpkg push -i $content_image -f $database --registry-insecure
+rm -f $database
+