Skip to content

Commit

Permalink
Create a DiscoveryBackend interface
Browse files Browse the repository at this point in the history 
A DiscoveryBackend is the content management logic behind a plugin
Discovery.  This commit creates a DiscoveryBackend implementation for
the SQLite backend of the Central Repository.

The DiscoveryBackend support is included in the "discovery" package at
the moment, so that it can use the Discovered type.  Making the new
DiscoveryBackend part of its own package would prevent it from using
the Discovered type due to a cyclical dependency.  This would force us
to create yet another type to characterize plugins for the
DiscoveryBackend to use.  This new type would then need to be converted
to the Discovered type by the Discovery.

An alternative would be to move the Discovered type to its own package
that could be imported by both the "discovery" package, and a new
"discoverybackend" package.

Signed-off-by: Marc Khouzam <kmarc@vmware.com>
  • Loading branch information
@marckhouzam
marckhouzam committed Feb 1, 2023
1 parent 54e2364 commit 9b2edee
Show file tree
Hide file tree
Showing 4 changed files with 235 additions and 180 deletions.
6 changes: 0 additions & 6 deletions pkg/common/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,3 @@ const (

// CoreName is the name of the core binary.
const CoreName = "core"

const (
// CentralRepoDBFileName is the name of the DB file that is stored in
// the OCI image describing the content of the Central Repository
CentralRepoDBFileName = "central.db"
)
10 changes: 10 additions & 0 deletions pkg/discovery/discoverybackend.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Copyright 2023 VMware, Inc. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package discovery

// DiscoveryBackend is the interface to extract plugin information from
// a backend used for plugin discovery.
type DiscoveryBackend interface {
GetAllPlugins() ([]*Discovered, error)
}
200 changes: 26 additions & 174 deletions pkg/discovery/oci_central.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,20 @@
package discovery

import (
"database/sql"
"fmt"
"os"
"path"
"path/filepath"
"strings"

// Import the sqlite3 driver
_ "github.com/mattn/go-sqlite3"
"github.com/pkg/errors"

"github.com/vmware-tanzu/tanzu-cli/pkg/carvelhelpers"
"github.com/vmware-tanzu/tanzu-cli/pkg/catalog"
"github.com/vmware-tanzu/tanzu-cli/pkg/common"
"github.com/vmware-tanzu/tanzu-cli/pkg/distribution"
configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types"
)

// contentDirName is the name of the directory where the file(s) describing
// the content of the discovery will be downloaded and stored.
// It should be a sub-directory of the cache directory.
const contentDirName = "plugin_data"

// DBBackedOCIDiscovery is an artifact discovery utilizing an OCI image
// which contains an SQLite database listing all available plugins.
type DBBackedOCIDiscovery struct {
Expand Down Expand Up @@ -60,191 +56,47 @@ func (od *DBBackedOCIDiscovery) Type() string {
return common.DiscoveryTypeOCI
}

// List available plugins.
func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) {
dbFilePath, err := od.fetchContentDB()
if err != nil {
return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name())
}

return od.getPluginsFromDB(dbFilePath)
}

// Describe a plugin.
func (od *DBBackedOCIDiscovery) Describe(name string) (Discovered, error) {
return Discovered{}, errors.Errorf("NOT IMPLEMENTED")
}

// fetchContentDB downloads the content DB to the cache directory and returns
// the path to the DB file to use.
func (od *DBBackedOCIDiscovery) fetchContentDB() (string, error) {
// TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache
pluginDBDir := filepath.Join(common.DefaultCacheDir, "plugin_db")
if err := carvelhelpers.DownloadDBImage(od.image, pluginDBDir); err != nil {
return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name())
}
return filepath.Join(pluginDBDir, common.CentralRepoDBFileName), nil
}

// getPluginsFromDB returns all plugins found in the DB located at the specified dbFilePath
func (od *DBBackedOCIDiscovery) getPluginsFromDB(dbFilePath string) ([]Discovered, error) {
db, err := sql.Open("sqlite3", dbFilePath)
// List available plugins.
func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) {
pluginDataDir, err := od.fetchContentImage()
if err != nil {
return nil, errors.Wrapf(err, "failed to open the DB for discovery '%s'", od.Name())
return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name())
}
defer db.Close()

// We need to order the results properly because the logic of processRows()
// expects an ordering of PluginName, then Target, then Version.
// The column order must also match the order used in getNextRow().
dbQuery := "SELECT PluginName,Target,RecommendedVersion,Version,Hidden,Description,Publisher,Vendor,OS,Architecture,Digest,URI FROM PluginBinaries ORDER BY PluginName,Target,Version;"
rows, err := db.Query(dbQuery)
if err != nil {
return nil, errors.Wrapf(err, "unable to setup DB query for discovery '%s'", od.Name())
}
defer rows.Close()
// The central repository uses relative image URIs to be future-proof.
// Determine the image prefix from the Central Repository main image.
// E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest
// then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/
imagePrefix := path.Dir(od.image)
backend := NewSQLiteBackend(od.Name(), pluginDataDir, imagePrefix)

allPluginPtrs, err := od.processRows(rows)
allPluginPtrs, err := backend.GetAllPlugins()
if err != nil {
return nil, err
}

// Convert from plugin pointers to plugins
// TODO(khouzam): continue optimizing by converting every call
// to using pointers
// TODO(khouzam): continue optimizing by converting every call to using pointers
var allPlugins []Discovered
for _, pluginPtr := range allPluginPtrs {
allPlugins = append(allPlugins, *pluginPtr)
}
return allPlugins, nil
}

func (od *DBBackedOCIDiscovery) processRows(rows *sql.Rows) ([]*Discovered, error) {
// The central repository uses relative image URIs to be future-proof.
// Determine the image prefix from the Central Repository main image.
// E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest
// then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/
imagePrefix := path.Dir(od.image)

currentPluginID := ""
currentVersion := ""
var currentPlugin *Discovered
allPlugins := make([]*Discovered, 0)
var artifactList distribution.ArtifactList
var artifacts distribution.Artifacts

for rows.Next() {
row, err := getNextRow(rows)
if err != nil {
return allPlugins, err
}

target := convertTargetFromDB(row.target)
pluginIDFromRow := catalog.PluginNameTarget(row.name, target)
if currentPluginID != pluginIDFromRow {
// Found a new plugin.
// Store the current one in the array and prepare the new one.
if currentPlugin != nil {
artifacts[currentVersion] = artifactList
artifactList = distribution.ArtifactList{}
currentPlugin.Distribution = artifacts
allPlugins = appendPlugin(allPlugins, currentPlugin)
}
currentPluginID = pluginIDFromRow

currentPlugin = &Discovered{
Name: row.name,
Description: row.description,
RecommendedVersion: row.recommendedVersion,
InstalledVersion: "", // TODO(khouzam)
SupportedVersions: []string{},
Optional: false,
Scope: common.PluginScopeStandalone,
Source: "Central Repository",
ContextName: "", // TODO(khouzam) this is used when creating the catalog. Concept needs updating
DiscoveryType: common.DiscoveryTypeOCI,
Target: target,
Status: common.PluginStatusNotInstalled,
}
currentVersion = ""
artifacts = distribution.Artifacts{}
}

// Check if we have a new version
if currentVersion != row.version {
// This is a new version of our current plugin. Add it to the array of versions.
// We can do this without verifying if the version is already there because
// we have requested the list of plugins from the database ordered by version.
currentPlugin.SupportedVersions = append(currentPlugin.SupportedVersions, row.version)

// Also store the list of artifacts for the previous version then start building
// the artifact list for the new version
if currentVersion != "" {
artifacts[currentVersion] = artifactList
artifactList = distribution.ArtifactList{}
}
currentVersion = row.version
}

// The central repository uses relative URIs to be future-proof.
// Build the full URI before creating the artifact.
fullImagePath := fmt.Sprintf("%s/%s", imagePrefix, row.uri)
// Create the artifact for this row.
artifact := distribution.Artifact{
Image: fullImagePath,
URI: "",
Digest: row.digest,
OS: row.os,
Arch: row.arch,
}
artifactList = append(artifactList, artifact)
}
// Don't forget to store the very last plugin we were building
artifacts[currentVersion] = artifactList
currentPlugin.Distribution = artifacts
allPlugins = appendPlugin(allPlugins, currentPlugin)
return allPlugins, rows.Err()
}

func getNextRow(rows *sql.Rows) (*centralRepoRow, error) {
var row centralRepoRow
// The order of the fields MUST match the order specified in the
// SELECT query that generated the rows.
err := rows.Scan(
&row.name,
&row.target,
&row.recommendedVersion,
&row.version,
&row.hidden,
&row.description,
&row.publisher,
&row.vendor,
&row.os,
&row.arch,
&row.digest,
&row.uri,
)
return &row, err
}

func convertTargetFromDB(target string) configtypes.Target {
target = strings.ToLower(target)
if target == "global" {
target = ""
}
return configtypes.StringToTarget(target)
}

func appendPlugin(allPlugins []*Discovered, plugin *Discovered) []*Discovered {
// Now that we are done gathering the information for the plugin
// we need to compute the recommendedVersion if it wasn't provided
// by the database
if err := SortVersions(plugin.SupportedVersions); err != nil {
fmt.Fprintf(os.Stderr, "error parsing supported versions for plugin %s: %v", plugin.Name, err)
}
if plugin.RecommendedVersion == "" {
plugin.RecommendedVersion = plugin.SupportedVersions[len(plugin.SupportedVersions)-1]
// fetchContentImage downloads the OCI image containing the information about the
// content of this discovery and stores it in the cache directory.
// It returns the path to the exact directory used.
func (od *DBBackedOCIDiscovery) fetchContentImage() (string, error) {
// TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache
pluginDataDir := filepath.Join(common.DefaultCacheDir, contentDirName)
if err := carvelhelpers.DownloadDBImage(od.image, pluginDataDir); err != nil {
return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name())
}
allPlugins = append(allPlugins, plugin)
return allPlugins
return pluginDataDir, nil
}
Loading

0 comments on commit 9b2edee

Please sign in to comment.