diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 77cd5b524..47915a82a 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -30,9 +30,3 @@ const ( // CoreName is the name of the core binary. const CoreName = "core" - -const ( - // CentralRepoDBFileName is the name of the DB file that is stored in - // the OCI image describing the content of the Central Repository - CentralRepoDBFileName = "central.db" -) diff --git a/pkg/discovery/discoverybackend.go b/pkg/discovery/discoverybackend.go new file mode 100644 index 000000000..03179f064 --- /dev/null +++ b/pkg/discovery/discoverybackend.go @@ -0,0 +1,10 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +// DiscoveryBackend is the interface to extract plugin information from +// a backend used for plugin discovery. +type DiscoveryBackend interface { + GetAllPlugins() ([]*Discovered, error) +} diff --git a/pkg/discovery/oci_central.go b/pkg/discovery/oci_central.go index dfa63893e..b7c4ded40 100644 --- a/pkg/discovery/oci_central.go +++ b/pkg/discovery/oci_central.go @@ -4,24 +4,20 @@ package discovery import ( - "database/sql" - "fmt" - "os" "path" "path/filepath" - "strings" - // Import the sqlite3 driver - _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" "github.com/vmware-tanzu/tanzu-cli/pkg/carvelhelpers" - "github.com/vmware-tanzu/tanzu-cli/pkg/catalog" "github.com/vmware-tanzu/tanzu-cli/pkg/common" - "github.com/vmware-tanzu/tanzu-cli/pkg/distribution" - configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types" ) +// contentDirName is the name of the directory where the file(s) describing +// the content of the discovery will be downloaded and stored. +// It should be a sub-directory of the cache directory. +const contentDirName = "plugin_data" + // DBBackedOCIDiscovery is an artifact discovery utilizing an OCI image // which contains an SQLite database listing all available plugins. type DBBackedOCIDiscovery struct { @@ -60,58 +56,32 @@ func (od *DBBackedOCIDiscovery) Type() string { return common.DiscoveryTypeOCI } -// List available plugins. -func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) { - dbFilePath, err := od.fetchContentDB() - if err != nil { - return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name()) - } - - return od.getPluginsFromDB(dbFilePath) -} - // Describe a plugin. func (od *DBBackedOCIDiscovery) Describe(name string) (Discovered, error) { return Discovered{}, errors.Errorf("NOT IMPLEMENTED") } -// fetchContentDB downloads the content DB to the cache directory and returns -// the path to the DB file to use. -func (od *DBBackedOCIDiscovery) fetchContentDB() (string, error) { - // TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache - pluginDBDir := filepath.Join(common.DefaultCacheDir, "plugin_db") - if err := carvelhelpers.DownloadDBImage(od.image, pluginDBDir); err != nil { - return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name()) - } - return filepath.Join(pluginDBDir, common.CentralRepoDBFileName), nil -} - -// getPluginsFromDB returns all plugins found in the DB located at the specified dbFilePath -func (od *DBBackedOCIDiscovery) getPluginsFromDB(dbFilePath string) ([]Discovered, error) { - db, err := sql.Open("sqlite3", dbFilePath) +// List available plugins. +func (od *DBBackedOCIDiscovery) List() (plugins []Discovered, err error) { + pluginDataDir, err := od.fetchContentImage() if err != nil { - return nil, errors.Wrapf(err, "failed to open the DB for discovery '%s'", od.Name()) + return nil, errors.Wrapf(err, "unable to fetch the content of discovery '%s'", od.Name()) } - defer db.Close() - // We need to order the results properly because the logic of processRows() - // expects an ordering of PluginName, then Target, then Version. - // The column order must also match the order used in getNextRow(). - dbQuery := "SELECT PluginName,Target,RecommendedVersion,Version,Hidden,Description,Publisher,Vendor,OS,Architecture,Digest,URI FROM PluginBinaries ORDER BY PluginName,Target,Version;" - rows, err := db.Query(dbQuery) - if err != nil { - return nil, errors.Wrapf(err, "unable to setup DB query for discovery '%s'", od.Name()) - } - defer rows.Close() + // The central repository uses relative image URIs to be future-proof. + // Determine the image prefix from the Central Repository main image. + // E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest + // then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/ + imagePrefix := path.Dir(od.image) + backend := NewSQLiteBackend(od.Name(), pluginDataDir, imagePrefix) - allPluginPtrs, err := od.processRows(rows) + allPluginPtrs, err := backend.GetAllPlugins() if err != nil { return nil, err } // Convert from plugin pointers to plugins - // TODO(khouzam): continue optimizing by converting every call - // to using pointers + // TODO(khouzam): continue optimizing by converting every call to using pointers var allPlugins []Discovered for _, pluginPtr := range allPluginPtrs { allPlugins = append(allPlugins, *pluginPtr) @@ -119,132 +89,14 @@ func (od *DBBackedOCIDiscovery) getPluginsFromDB(dbFilePath string) ([]Discovere return allPlugins, nil } -func (od *DBBackedOCIDiscovery) processRows(rows *sql.Rows) ([]*Discovered, error) { - // The central repository uses relative image URIs to be future-proof. - // Determine the image prefix from the Central Repository main image. - // E.g., if the main image is at project.registry.vmware.com/tanzu-cli/plugins/plugin_database:latest - // then the image prefix should be project.registry.vmware.com/tanzu-cli/plugins/ - imagePrefix := path.Dir(od.image) - - currentPluginID := "" - currentVersion := "" - var currentPlugin *Discovered - allPlugins := make([]*Discovered, 0) - var artifactList distribution.ArtifactList - var artifacts distribution.Artifacts - - for rows.Next() { - row, err := getNextRow(rows) - if err != nil { - return allPlugins, err - } - - target := convertTargetFromDB(row.target) - pluginIDFromRow := catalog.PluginNameTarget(row.name, target) - if currentPluginID != pluginIDFromRow { - // Found a new plugin. - // Store the current one in the array and prepare the new one. - if currentPlugin != nil { - artifacts[currentVersion] = artifactList - artifactList = distribution.ArtifactList{} - currentPlugin.Distribution = artifacts - allPlugins = appendPlugin(allPlugins, currentPlugin) - } - currentPluginID = pluginIDFromRow - - currentPlugin = &Discovered{ - Name: row.name, - Description: row.description, - RecommendedVersion: row.recommendedVersion, - InstalledVersion: "", // TODO(khouzam) - SupportedVersions: []string{}, - Optional: false, - Scope: common.PluginScopeStandalone, - Source: "Central Repository", - ContextName: "", // TODO(khouzam) this is used when creating the catalog. Concept needs updating - DiscoveryType: common.DiscoveryTypeOCI, - Target: target, - Status: common.PluginStatusNotInstalled, - } - currentVersion = "" - artifacts = distribution.Artifacts{} - } - - // Check if we have a new version - if currentVersion != row.version { - // This is a new version of our current plugin. Add it to the array of versions. - // We can do this without verifying if the version is already there because - // we have requested the list of plugins from the database ordered by version. - currentPlugin.SupportedVersions = append(currentPlugin.SupportedVersions, row.version) - - // Also store the list of artifacts for the previous version then start building - // the artifact list for the new version - if currentVersion != "" { - artifacts[currentVersion] = artifactList - artifactList = distribution.ArtifactList{} - } - currentVersion = row.version - } - - // The central repository uses relative URIs to be future-proof. - // Build the full URI before creating the artifact. - fullImagePath := fmt.Sprintf("%s/%s", imagePrefix, row.uri) - // Create the artifact for this row. - artifact := distribution.Artifact{ - Image: fullImagePath, - URI: "", - Digest: row.digest, - OS: row.os, - Arch: row.arch, - } - artifactList = append(artifactList, artifact) - } - // Don't forget to store the very last plugin we were building - artifacts[currentVersion] = artifactList - currentPlugin.Distribution = artifacts - allPlugins = appendPlugin(allPlugins, currentPlugin) - return allPlugins, rows.Err() -} - -func getNextRow(rows *sql.Rows) (*centralRepoRow, error) { - var row centralRepoRow - // The order of the fields MUST match the order specified in the - // SELECT query that generated the rows. - err := rows.Scan( - &row.name, - &row.target, - &row.recommendedVersion, - &row.version, - &row.hidden, - &row.description, - &row.publisher, - &row.vendor, - &row.os, - &row.arch, - &row.digest, - &row.uri, - ) - return &row, err -} - -func convertTargetFromDB(target string) configtypes.Target { - target = strings.ToLower(target) - if target == "global" { - target = "" - } - return configtypes.StringToTarget(target) -} - -func appendPlugin(allPlugins []*Discovered, plugin *Discovered) []*Discovered { - // Now that we are done gathering the information for the plugin - // we need to compute the recommendedVersion if it wasn't provided - // by the database - if err := SortVersions(plugin.SupportedVersions); err != nil { - fmt.Fprintf(os.Stderr, "error parsing supported versions for plugin %s: %v", plugin.Name, err) - } - if plugin.RecommendedVersion == "" { - plugin.RecommendedVersion = plugin.SupportedVersions[len(plugin.SupportedVersions)-1] +// fetchContentImage downloads the OCI image containing the information about the +// content of this discovery and stores it in the cache directory. +// It returns the path to the exact directory used. +func (od *DBBackedOCIDiscovery) fetchContentImage() (string, error) { + // TODO(khouzam): Improve by checking if we really need to download again or if we can use the cache + pluginDataDir := filepath.Join(common.DefaultCacheDir, contentDirName) + if err := carvelhelpers.DownloadDBImage(od.image, pluginDataDir); err != nil { + return "", errors.Wrapf(err, "failed to download OCI image from discovery '%s'", od.Name()) } - allPlugins = append(allPlugins, plugin) - return allPlugins + return pluginDataDir, nil } diff --git a/pkg/discovery/sqlite_backend.go b/pkg/discovery/sqlite_backend.go new file mode 100644 index 000000000..282534dbb --- /dev/null +++ b/pkg/discovery/sqlite_backend.go @@ -0,0 +1,199 @@ +// Copyright 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + + // Import the sqlite3 driver + _ "github.com/mattn/go-sqlite3" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/tanzu-cli/pkg/catalog" + "github.com/vmware-tanzu/tanzu-cli/pkg/common" + "github.com/vmware-tanzu/tanzu-cli/pkg/distribution" + configtypes "github.com/vmware-tanzu/tanzu-plugin-runtime/config/types" +) + +const ( + // sqliteDBFileName is the name of the DB file that is stored in + // the OCI image describing the content of the Central Repository + sqliteDBFileName = "central.db" +) + +// SQLiteBackend is a backend using SQLite for managing the data for the +// content of the Central Repository. +type SQLiteBackend struct { + // discoveryName is the name of the discovery powered by this backend + discoveryName string + // contentFile represents the full path to the SQLite DB file + contentFile string + // uriPrefix is the prefix that must be added to the extracted URIs. + // The central repository uses relative image URIs to be future-proof. + uriPrefix string +} + +// NewSQLiteBackend returns a new DiscoveryBackend using the data found at 'contentDir'. +func NewSQLiteBackend(discoveryName, contentDir, prefix string) DiscoveryBackend { + return &SQLiteBackend{ + discoveryName: discoveryName, + contentFile: filepath.Join(contentDir, sqliteDBFileName), + uriPrefix: prefix, + } +} + +// GetAllPlugins returns all plugins discovered in this backend. +func (b *SQLiteBackend) GetAllPlugins() ([]*Discovered, error) { + return b.getPluginsFromDB() +} + +// getPluginsFromDB returns all plugins found in the DB located at the specified dbFilePath +func (b *SQLiteBackend) getPluginsFromDB() ([]*Discovered, error) { + db, err := sql.Open("sqlite3", b.contentFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to open the DB for discovery '%s'", b.discoveryName) + } + defer db.Close() + + // We need to order the results properly because the logic of processRows() + // expects an ordering of PluginName, then Target, then Version. + // The column order must also match the order used in getNextRow(). + dbQuery := "SELECT PluginName,Target,RecommendedVersion,Version,Hidden,Description,Publisher,Vendor,OS,Architecture,Digest,URI FROM PluginBinaries ORDER BY PluginName,Target,Version;" + rows, err := db.Query(dbQuery) + if err != nil { + return nil, errors.Wrapf(err, "unable to setup DB query for discovery '%s'", b.discoveryName) + } + defer rows.Close() + + return b.extractPluginsFromRows(rows) +} + +func (b *SQLiteBackend) extractPluginsFromRows(rows *sql.Rows) ([]*Discovered, error) { + currentPluginID := "" + currentVersion := "" + var currentPlugin *Discovered + allPlugins := make([]*Discovered, 0) + var artifactList distribution.ArtifactList + var artifacts distribution.Artifacts + + for rows.Next() { + row, err := getNextRow(rows) + if err != nil { + return allPlugins, err + } + + target := convertTargetFromDB(row.target) + pluginIDFromRow := catalog.PluginNameTarget(row.name, target) + if currentPluginID != pluginIDFromRow { + // Found a new plugin. + // Store the current one in the array and prepare the new one. + if currentPlugin != nil { + artifacts[currentVersion] = artifactList + artifactList = distribution.ArtifactList{} + currentPlugin.Distribution = artifacts + allPlugins = appendPlugin(allPlugins, currentPlugin) + } + currentPluginID = pluginIDFromRow + + currentPlugin = &Discovered{ + Name: row.name, + Description: row.description, + RecommendedVersion: row.recommendedVersion, + InstalledVersion: "", // TODO(khouzam) + SupportedVersions: []string{}, + Optional: false, + Scope: common.PluginScopeStandalone, + Source: "Central Repository", + ContextName: "", // TODO(khouzam) this is used when creating the catalog. Concept needs updating + DiscoveryType: common.DiscoveryTypeOCI, + Target: target, + Status: common.PluginStatusNotInstalled, + } + currentVersion = "" + artifacts = distribution.Artifacts{} + } + + // Check if we have a new version + if currentVersion != row.version { + // This is a new version of our current plugin. Add it to the array of versions. + // We can do this without verifying if the version is already there because + // we have requested the list of plugins from the database ordered by version. + currentPlugin.SupportedVersions = append(currentPlugin.SupportedVersions, row.version) + + // Also store the list of artifacts for the previous version then start building + // the artifact list for the new version + if currentVersion != "" { + artifacts[currentVersion] = artifactList + artifactList = distribution.ArtifactList{} + } + currentVersion = row.version + } + + // The central repository uses relative URIs to be future-proof. + // Build the full URI before creating the artifact. + fullImagePath := fmt.Sprintf("%s/%s", b.uriPrefix, row.uri) + // Create the artifact for this row. + artifact := distribution.Artifact{ + Image: fullImagePath, + URI: "", + Digest: row.digest, + OS: row.os, + Arch: row.arch, + } + artifactList = append(artifactList, artifact) + } + // Don't forget to store the very last plugin we were building + artifacts[currentVersion] = artifactList + currentPlugin.Distribution = artifacts + allPlugins = appendPlugin(allPlugins, currentPlugin) + return allPlugins, rows.Err() +} + +func getNextRow(rows *sql.Rows) (*centralRepoRow, error) { + var row centralRepoRow + // The order of the fields MUST match the order specified in the + // SELECT query that generated the rows. + err := rows.Scan( + &row.name, + &row.target, + &row.recommendedVersion, + &row.version, + &row.hidden, + &row.description, + &row.publisher, + &row.vendor, + &row.os, + &row.arch, + &row.digest, + &row.uri, + ) + return &row, err +} + +func convertTargetFromDB(target string) configtypes.Target { + target = strings.ToLower(target) + if target == "global" { + target = "" + } + return configtypes.StringToTarget(target) +} + +func appendPlugin(allPlugins []*Discovered, plugin *Discovered) []*Discovered { + // Now that we are done gathering the information for the plugin + // we need to compute the recommendedVersion if it wasn't provided + // by the database + if err := SortVersions(plugin.SupportedVersions); err != nil { + fmt.Fprintf(os.Stderr, "error parsing supported versions for plugin %s: %v", plugin.Name, err) + } + if plugin.RecommendedVersion == "" { + plugin.RecommendedVersion = plugin.SupportedVersions[len(plugin.SupportedVersions)-1] + } + allPlugins = append(allPlugins, plugin) + return allPlugins +}