From 9b2edee304820573baae351118590b682330f550 Mon Sep 17 00:00:00 2001 From: Marc Khouzam Date: Wed, 1 Feb 2023 07:49:32 -0500 Subject: [PATCH] Create a DiscoveryBackend interface A DiscoveryBackend is the content management logic behind a plugin Discovery. This commit creates a DiscoveryBackend implementation for the SQLite backend of the Central Repository. The DiscoveryBackend support is included in the "discovery" package at the moment, so that it can use the Discovered type. Making the new DiscoveryBackend part of its own package would prevent it from using the Discovered type due to a cyclical dependency. This would force us to create yet another type to characterize plugins for the DiscoveryBackend to use. This new type would then need to be converted to the Discovered type by the Discovery. An alternative would be to move the Discovered type to its own package that could be imported by both the "discovery" package, and a new "discoverybackend" package. Signed-off-by: Marc Khouzam --- pkg/common/constants.go | 6 - pkg/discovery/discoverybackend.go | 10 ++ pkg/discovery/oci_central.go | 200 ++++-------------------------- pkg/discovery/sqlite_backend.go | 199 +++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 180 deletions(-) create mode 100644 pkg/discovery/discoverybackend.go create mode 100644 pkg/discovery/sqlite_backend.go diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 77cd5b524..47915a82a 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -30,9 +30,3 @@ const ( // CoreName is the name of the core binary. const CoreName = "core" - -const ( - // CentralRepoDBFileName is the name of the DB file that is stored in - // the OCI image describing the content of the Central Repository - CentralRepoDBFileName = "central.db" -) diff --git a/pkg/discovery/discoverybackend.go b/pkg/discovery/discoverybackend.go new file mode 100644 index 000000000..03179f064 --- /dev/null +++ b/pkg/discovery/discoverybackend.go @@ -0,0 +1,10 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +// DiscoveryBackend is the interface to extract plugin information from +// a backend used for plugin discovery. +type DiscoveryBackend interface { + GetAllPlugins() ([]*Discovered, error) +} diff --git a/pkg/discovery/oci_central.go b/pkg/discovery/oci_central.go index dfa63893e..b7c4ded40 100644 --- a/pkg/discovery/oci_central.go +++ b/pkg/discovery/oci_central.go @@ -4,24 +4,20 @@ package discovery import ( - "database/sql" - "fmt" - "os" "path" "path/filepath" - "strings" - // Import the sqlite3 driver - _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vmware-tanzu/tanzu-cli/pkg/carvelhelpers" - "github.com/vmware-tanzu/tanzu-cli/pkg/catalog" "github.com/vmware-tanzu/tanzu-cli/pkg/common" - "github.com/vmware-tanzu/tanzu-cli/pkg/distribution" - configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types" ) +// contentDirName is the name of the directory where the file(s) describing +// the content of the discovery will be downloaded and stored. +// It should be a sub-directory of the cache directory. +const contentDirName = "plugin_data" + // DBBackedOCIDiscovery is an artifact discovery utilizing an OCI image // which contains an SQLite database listing all available plugins. type DBBackedOCIDiscovery struct { @@ -60,58 +56,32 @@ func (od *DBBackedOCIDiscovery) Type() string { return common.DiscoveryTypeOCI } -// List available plugins. -func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) { - dbFilePath, err := od.fetchContentDB() - if err != nil { - return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name()) - } - - return od.getPluginsFromDB(dbFilePath) -} - // Describe a plugin. func (od *DBBackedOCIDiscovery) Describe(name string) (Discovered, error) { return Discovered{}, errors.Errorf("NOT IMPLEMENTED") } -// fetchContentDB downloads the content DB to the cache directory and returns -// the path to the DB file to use. -func (od *DBBackedOCIDiscovery) fetchContentDB() (string, error) { - // TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache - pluginDBDir := filepath.Join(common.DefaultCacheDir, "plugin_db") - if err := carvelhelpers.DownloadDBImage(od.image, pluginDBDir); err != nil { - return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name()) - } - return filepath.Join(pluginDBDir, common.CentralRepoDBFileName), nil -} - -// getPluginsFromDB returns all plugins found in the DB located at the specified dbFilePath -func (od *DBBackedOCIDiscovery) getPluginsFromDB(dbFilePath string) ([]Discovered, error) { - db, err := sql.Open("sqlite3", dbFilePath) +// List available plugins. +func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) { + pluginDataDir, err := od.fetchContentImage() if err != nil { - return nil, errors.Wrapf(err, "failed to open the DB for discovery '%s'", od.Name()) + return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name()) } - defer db.Close() - // We need to order the results properly because the logic of processRows() - // expects an ordering of PluginName, then Target, then Version. - // The column order must also match the order used in getNextRow(). - dbQuery := "SELECT PluginName,Target,RecommendedVersion,Version,Hidden,Description,Publisher,Vendor,OS,Architecture,Digest,URI FROM PluginBinaries ORDER BY PluginName,Target,Version;" - rows, err := db.Query(dbQuery) - if err != nil { - return nil, errors.Wrapf(err, "unable to setup DB query for discovery '%s'", od.Name()) - } - defer rows.Close() + // The central repository uses relative image URIs to be future-proof. + // Determine the image prefix from the Central Repository main image. + // E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest + // then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/ + imagePrefix := path.Dir(od.image) + backend := NewSQLiteBackend(od.Name(), pluginDataDir, imagePrefix) - allPluginPtrs, err := od.processRows(rows) + allPluginPtrs, err := backend.GetAllPlugins() if err != nil { return nil, err } // Convert from plugin pointers to plugins - // TODO(khouzam): continue optimizing by converting every call - // to using pointers + // TODO(khouzam): continue optimizing by converting every call to using pointers var allPlugins []Discovered for _, pluginPtr := range allPluginPtrs { allPlugins = append(allPlugins, *pluginPtr) @@ -119,132 +89,14 @@ func (od *DBBackedOCIDiscovery) getPluginsFromDB(dbFilePath string) ([]Discovere return allPlugins, nil } -func (od *DBBackedOCIDiscovery) processRows(rows *sql.Rows) ([]*Discovered, error) { - // The central repository uses relative image URIs to be future-proof. - // Determine the image prefix from the Central Repository main image. - // E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest - // then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/ - imagePrefix := path.Dir(od.image) - - currentPluginID := "" - currentVersion := "" - var currentPlugin *Discovered - allPlugins := make([]*Discovered, 0) - var artifactList distribution.ArtifactList - var artifacts distribution.Artifacts - - for rows.Next() { - row, err := getNextRow(rows) - if err != nil { - return allPlugins, err - } - - target := convertTargetFromDB(row.target) - pluginIDFromRow := catalog.PluginNameTarget(row.name, target) - if currentPluginID != pluginIDFromRow { - // Found a new plugin. - // Store the current one in the array and prepare the new one. - if currentPlugin != nil { - artifacts[currentVersion] = artifactList - artifactList = distribution.ArtifactList{} - currentPlugin.Distribution = artifacts - allPlugins = appendPlugin(allPlugins, currentPlugin) - } - currentPluginID = pluginIDFromRow - - currentPlugin = &Discovered{ - Name: row.name, - Description: row.description, - RecommendedVersion: row.recommendedVersion, - InstalledVersion: "", // TODO(khouzam) - SupportedVersions: []string{}, - Optional: false, - Scope: common.PluginScopeStandalone, - Source: "Central Repository", - ContextName: "", // TODO(khouzam) this is used when creating the catalog. Concept needs updating - DiscoveryType: common.DiscoveryTypeOCI, - Target: target, - Status: common.PluginStatusNotInstalled, - } - currentVersion = "" - artifacts = distribution.Artifacts{} - } - - // Check if we have a new version - if currentVersion != row.version { - // This is a new version of our current plugin. Add it to the array of versions. - // We can do this without verifying if the version is already there because - // we have requested the list of plugins from the database ordered by version. - currentPlugin.SupportedVersions = append(currentPlugin.SupportedVersions, row.version) - - // Also store the list of artifacts for the previous version then start building - // the artifact list for the new version - if currentVersion != "" { - artifacts[currentVersion] = artifactList - artifactList = distribution.ArtifactList{} - } - currentVersion = row.version - } - - // The central repository uses relative URIs to be future-proof. - // Build the full URI before creating the artifact. - fullImagePath := fmt.Sprintf("%s/%s", imagePrefix, row.uri) - // Create the artifact for this row. - artifact := distribution.Artifact{ - Image: fullImagePath, - URI: "", - Digest: row.digest, - OS: row.os, - Arch: row.arch, - } - artifactList = append(artifactList, artifact) - } - // Don't forget to store the very last plugin we were building - artifacts[currentVersion] = artifactList - currentPlugin.Distribution = artifacts - allPlugins = appendPlugin(allPlugins, currentPlugin) - return allPlugins, rows.Err() -} - -func getNextRow(rows *sql.Rows) (*centralRepoRow, error) { - var row centralRepoRow - // The order of the fields MUST match the order specified in the - // SELECT query that generated the rows. - err := rows.Scan( - &row.name, - &row.target, - &row.recommendedVersion, - &row.version, - &row.hidden, - &row.description, - &row.publisher, - &row.vendor, - &row.os, - &row.arch, - &row.digest, - &row.uri, - ) - return &row, err -} - -func convertTargetFromDB(target string) configtypes.Target { - target = strings.ToLower(target) - if target == "global" { - target = "" - } - return configtypes.StringToTarget(target) -} - -func appendPlugin(allPlugins []*Discovered, plugin *Discovered) []*Discovered { - // Now that we are done gathering the information for the plugin - // we need to compute the recommendedVersion if it wasn't provided - // by the database - if err := SortVersions(plugin.SupportedVersions); err != nil { - fmt.Fprintf(os.Stderr, "error parsing supported versions for plugin %s: %v", plugin.Name, err) - } - if plugin.RecommendedVersion == "" { - plugin.RecommendedVersion = plugin.SupportedVersions[len(plugin.SupportedVersions)-1] +// fetchContentImage downloads the OCI image containing the information about the +// content of this discovery and stores it in the cache directory. +// It returns the path to the exact directory used. +func (od *DBBackedOCIDiscovery) fetchContentImage() (string, error) { + // TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache + pluginDataDir := filepath.Join(common.DefaultCacheDir, contentDirName) + if err := carvelhelpers.DownloadDBImage(od.image, pluginDataDir); err != nil { + return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name()) } - allPlugins = append(allPlugins, plugin) - return allPlugins + return pluginDataDir, nil } diff --git a/pkg/discovery/sqlite_backend.go b/pkg/discovery/sqlite_backend.go new file mode 100644 index 000000000..282534dbb --- /dev/null +++ b/pkg/discovery/sqlite_backend.go @@ -0,0 +1,199 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + + // Import the sqlite3 driver + _ "github.com/mattn/go-sqlite3" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/tanzu-cli/pkg/catalog" + "github.com/vmware-tanzu/tanzu-cli/pkg/common" + "github.com/vmware-tanzu/tanzu-cli/pkg/distribution" + configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types" +) + +const ( + // sqliteDBFileName is the name of the DB file that is stored in + // the OCI image describing the content of the Central Repository + sqliteDBFileName = "central.db" +) + +// SQLiteBackend is a backend using SQLite for managing the data for the +// content of the Central Repository. +type SQLiteBackend struct { + // discoveryName is the name of the discovery powered by this backend + discoveryName string + // contentFile represents the full path to the SQLite DB file + contentFile string + // uriPrefix is the prefix that must be added to the extracted URIs. + // The central repository uses relative image URIs to be future-proof. + uriPrefix string +} + +// NewSQLiteBackend returns a new DiscoveryBackend using the data found at 'contentDir'. +func NewSQLiteBackend(discoveryName, contentDir, prefix string) DiscoveryBackend { + return &SQLiteBackend{ + discoveryName: discoveryName, + contentFile: filepath.Join(contentDir, sqliteDBFileName), + uriPrefix: prefix, + } +} + +// GetAllPlugins returns all plugins discovered in this backend. +func (b *SQLiteBackend) GetAllPlugins() ([]*Discovered, error) { + return b.getPluginsFromDB() +} + +// getPluginsFromDB returns all plugins found in the DB located at the specified dbFilePath +func (b *SQLiteBackend) getPluginsFromDB() ([]*Discovered, error) { + db, err := sql.Open("sqlite3", b.contentFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to open the DB for discovery '%s'", b.discoveryName) + } + defer db.Close() + + // We need to order the results properly because the logic of processRows() + // expects an ordering of PluginName, then Target, then Version. + // The column order must also match the order used in getNextRow(). + dbQuery := "SELECT PluginName,Target,RecommendedVersion,Version,Hidden,Description,Publisher,Vendor,OS,Architecture,Digest,URI FROM PluginBinaries ORDER BY PluginName,Target,Version;" + rows, err := db.Query(dbQuery) + if err != nil { + return nil, errors.Wrapf(err, "unable to setup DB query for discovery '%s'", b.discoveryName) + } + defer rows.Close() + + return b.extractPluginsFromRows(rows) +} + +func (b *SQLiteBackend) extractPluginsFromRows(rows *sql.Rows) ([]*Discovered, error) { + currentPluginID := "" + currentVersion := "" + var currentPlugin *Discovered + allPlugins := make([]*Discovered, 0) + var artifactList distribution.ArtifactList + var artifacts distribution.Artifacts + + for rows.Next() { + row, err := getNextRow(rows) + if err != nil { + return allPlugins, err + } + + target := convertTargetFromDB(row.target) + pluginIDFromRow := catalog.PluginNameTarget(row.name, target) + if currentPluginID != pluginIDFromRow { + // Found a new plugin. + // Store the current one in the array and prepare the new one. + if currentPlugin != nil { + artifacts[currentVersion] = artifactList + artifactList = distribution.ArtifactList{} + currentPlugin.Distribution = artifacts + allPlugins = appendPlugin(allPlugins, currentPlugin) + } + currentPluginID = pluginIDFromRow + + currentPlugin = &Discovered{ + Name: row.name, + Description: row.description, + RecommendedVersion: row.recommendedVersion, + InstalledVersion: "", // TODO(khouzam) + SupportedVersions: []string{}, + Optional: false, + Scope: common.PluginScopeStandalone, + Source: "Central Repository", + ContextName: "", // TODO(khouzam) this is used when creating the catalog. Concept needs updating + DiscoveryType: common.DiscoveryTypeOCI, + Target: target, + Status: common.PluginStatusNotInstalled, + } + currentVersion = "" + artifacts = distribution.Artifacts{} + } + + // Check if we have a new version + if currentVersion != row.version { + // This is a new version of our current plugin. Add it to the array of versions. + // We can do this without verifying if the version is already there because + // we have requested the list of plugins from the database ordered by version. + currentPlugin.SupportedVersions = append(currentPlugin.SupportedVersions, row.version) + + // Also store the list of artifacts for the previous version then start building + // the artifact list for the new version + if currentVersion != "" { + artifacts[currentVersion] = artifactList + artifactList = distribution.ArtifactList{} + } + currentVersion = row.version + } + + // The central repository uses relative URIs to be future-proof. + // Build the full URI before creating the artifact. + fullImagePath := fmt.Sprintf("%s/%s", b.uriPrefix, row.uri) + // Create the artifact for this row. + artifact := distribution.Artifact{ + Image: fullImagePath, + URI: "", + Digest: row.digest, + OS: row.os, + Arch: row.arch, + } + artifactList = append(artifactList, artifact) + } + // Don't forget to store the very last plugin we were building + artifacts[currentVersion] = artifactList + currentPlugin.Distribution = artifacts + allPlugins = appendPlugin(allPlugins, currentPlugin) + return allPlugins, rows.Err() +} + +func getNextRow(rows *sql.Rows) (*centralRepoRow, error) { + var row centralRepoRow + // The order of the fields MUST match the order specified in the + // SELECT query that generated the rows. + err := rows.Scan( + &row.name, + &row.target, + &row.recommendedVersion, + &row.version, + &row.hidden, + &row.description, + &row.publisher, + &row.vendor, + &row.os, + &row.arch, + &row.digest, + &row.uri, + ) + return &row, err +} + +func convertTargetFromDB(target string) configtypes.Target { + target = strings.ToLower(target) + if target == "global" { + target = "" + } + return configtypes.StringToTarget(target) +} + +func appendPlugin(allPlugins []*Discovered, plugin *Discovered) []*Discovered { + // Now that we are done gathering the information for the plugin + // we need to compute the recommendedVersion if it wasn't provided + // by the database + if err := SortVersions(plugin.SupportedVersions); err != nil { + fmt.Fprintf(os.Stderr, "error parsing supported versions for plugin %s: %v", plugin.Name, err) + } + if plugin.RecommendedVersion == "" { + plugin.RecommendedVersion = plugin.SupportedVersions[len(plugin.SupportedVersions)-1] + } + allPlugins = append(allPlugins, plugin) + return allPlugins +}