WIP: VERY ugly but tests pass :)

src/cmd/root.go

@@ -6,10 +6,10 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/defenseunicorns/zarf/src/cmd/common"
 	"github.com/defenseunicorns/zarf/src/cmd/tools"
-	zarfConfig "github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/utils/exec"
 	"github.com/spf13/cobra"
@@ -83,14 +83,16 @@ func init() {
 	v.SetDefault(V_NO_LOG_FILE, false)
 	v.SetDefault(V_NO_PROGRESS, false)
 	v.SetDefault(V_INSECURE, false)
-	v.SetDefault(V_ZARF_CACHE, zarfConfig.ZarfDefaultCachePath)
 	v.SetDefault(V_TMP_DIR, "")
 
+	homeDir, _ := os.UserHomeDir()
+	v.SetDefault(V_UDS_CACHE, filepath.Join(homeDir, config.UDSCache))
+
 	rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", v.GetString(V_LOG_LEVEL), lang.RootCmdFlagLogLevel)
 	rootCmd.PersistentFlags().StringVarP(&config.CLIArch, "architecture", "a", v.GetString(common.VArchitecture), lang.RootCmdFlagArch)
 	rootCmd.PersistentFlags().BoolVar(&config.SkipLogFile, "no-log-file", v.GetBool(V_NO_LOG_FILE), lang.RootCmdFlagSkipLogFile)
 	rootCmd.PersistentFlags().BoolVar(&message.NoProgress, "no-progress", v.GetBool(V_NO_PROGRESS), lang.RootCmdFlagNoProgress)
-	rootCmd.PersistentFlags().StringVar(&config.CommonOptions.CachePath, "zarf-cache", v.GetString(V_ZARF_CACHE), lang.RootCmdFlagCachePath)
+	rootCmd.PersistentFlags().StringVar(&config.CommonOptions.CachePath, "uds-cache", v.GetString(V_UDS_CACHE), lang.RootCmdFlagCachePath)
 	rootCmd.PersistentFlags().StringVar(&config.CommonOptions.TempDirectory, "tmpdir", v.GetString(V_TMP_DIR), lang.RootCmdFlagTempDir)
 	rootCmd.PersistentFlags().BoolVar(&config.CommonOptions.Insecure, "insecure", v.GetBool(V_INSECURE), lang.RootCmdFlagInsecure)
 }

src/cmd/uds.go

@@ -258,7 +258,8 @@ func configureZarf() {
 		TempDirectory:  config.CommonOptions.TempDirectory,
 		OCIConcurrency: config.CommonOptions.OCIConcurrency,
 		Confirm:        config.CommonOptions.Confirm,
-		CachePath:      config.CommonOptions.CachePath,
+		// todo: decouple Zarf cache?
+		CachePath: config.CommonOptions.CachePath,
 	}
 }
 

src/cmd/viper.go

@@ -23,7 +23,7 @@ const (
 	V_ARCHITECTURE = "architecture"
 	V_NO_LOG_FILE  = "no_log_file"
 	V_NO_PROGRESS  = "no_progress"
-	V_ZARF_CACHE   = "zarf_cache"
+	V_UDS_CACHE    = "uds_cache"
 	V_TMP_DIR      = "tmp_dir"
 	V_INSECURE     = "insecure"
 

src/config/config.go

@@ -44,6 +44,9 @@ const (
 	// ChecksumsTxt is the name of the checksums.txt file in a Zarf pkg
 	ChecksumsTxt = "checksums.txt"
 
+	// UDSCache is the directory containing cached bundle layers
+	UDSCache = ".uds-cache"
+
 	// TasksYAML is the default name of the uds run cmd file
 	TasksYAML = "tasks.yaml"
 )

src/pkg/bundle/bundle.go

@@ -57,7 +57,7 @@ func Create(b *Bundler, signature []byte) error {
 
 		if pkg.Repository != "" {
 			url := fmt.Sprintf("%s:%s", pkg.Repository, pkg.Ref)
-			remoteBundler, err := bundler.NewRemoteBundler(pkg, url, store, nil)
+			remoteBundler, err := bundler.NewRemoteBundler(pkg, url, store, nil, b.tmp)
 			if err != nil {
 				return err
 			}
@@ -69,21 +69,31 @@ func Create(b *Bundler, signature []byte) error {
 			if err != nil {
 				return err
 			}
-
-			// append zarf pkg manifest to root manifest and grab path for archiving
-			rootManifest.Layers = append(rootManifest.Layers, pkgManifestDesc)
-			digest := pkgManifestDesc.Digest.Encoded()
-			artifactPathMap[filepath.Join(b.tmp, config.BlobsDir, digest)] = filepath.Join(config.BlobsDir, digest)
-
-			message.Debugf("Pushed %s sub-manifest into %s: %s", url, b.tmp, message.JSONValue(pkgManifestDesc))
 			layerDescs, err := remoteBundler.PushLayers(fetchSpinner, i+1, len(bundle.ZarfPackages))
 			if err != nil {
 				return err
 			}
 
 			// grab layers for archiving
 			for _, layerDesc := range layerDescs {
-				digest = layerDesc.Digest.Encoded()
+				if layerDesc.MediaType == ocispec.MediaTypeImageManifest {
+					// rewrite the Zarf image manifest to have media type of Zarf blob
+					err = os.Remove(filepath.Join(b.tmp, config.BlobsDir, layerDesc.Digest.Encoded()))
+					if err != nil {
+						return err
+					}
+					layerBytes, err := remoteBundler.RemoteSrc.FetchLayer(layerDesc)
+					if err != nil {
+						return err
+					}
+					rootPkgDescBytes := content.NewDescriptorFromBytes(oci.ZarfLayerMediaTypeBlob, layerBytes)
+					if err = store.Push(context.TODO(), rootPkgDescBytes, bytes.NewReader(layerBytes)); err != nil {
+						return err
+					}
+					layerDesc.MediaType = oci.ZarfLayerMediaTypeBlob
+					rootManifest.Layers = append(rootManifest.Layers, layerDesc)
+				}
+				digest := layerDesc.Digest.Encoded()
 				artifactPathMap[filepath.Join(b.tmp, config.BlobsDir, digest)] = filepath.Join(config.BlobsDir, digest)
 			}
 		} else if pkg.Path != "" {
@@ -188,6 +198,11 @@ func Create(b *Bundler, signature []byte) error {
 	if err != nil {
 		return err
 	}
+	err = cleanIndexJSON(b.tmp, ref)
+	if err != nil {
+		return err
+	}
+
 	// tarball the bundle
 	err = writeTarball(bundle, artifactPathMap)
 	if err != nil {
@@ -209,7 +224,7 @@ func CreateAndPublish(remoteDst *oci.OrasRemote, bundle *types.UDSBundle, signat
 
 	for i, pkg := range bundle.ZarfPackages {
 		url := fmt.Sprintf("%s:%s", pkg.Repository, pkg.Ref)
-		remoteBundler, err := bundler.NewRemoteBundler(pkg, url, nil, remoteDst)
+		remoteBundler, err := bundler.NewRemoteBundler(pkg, url, nil, remoteDst, "")
 		if err != nil {
 			return err
 		}
@@ -274,9 +289,7 @@ func CreateAndPublish(remoteDst *oci.OrasRemote, bundle *types.UDSBundle, signat
 	message.Debug("Pushed config:", message.JSONValue(configDesc))
 
 	rootManifest.Config = configDesc
-
 	rootManifest.SchemaVersion = 2
-
 	rootManifest.Annotations = manifestAnnotationsFromMetadata(&bundle.Metadata) // maps to registry UI
 	b, err := json.Marshal(rootManifest)
 	if err != nil {
@@ -470,3 +483,38 @@ func pushBundleSignature(ctx context.Context, store *ocistore.Store, signature [
 	}
 	return signatureDesc, err
 }
+
+// rebuild index.json because copying remote Zarf pkgs adds unnecessary entries
+// this is due to root manifest in Zarf packages having an image manifest media type
+func cleanIndexJSON(tmpDir, ref string) error {
+	indexBytes, err := os.ReadFile(filepath.Join(tmpDir, "index.json"))
+	if err != nil {
+		return err
+	}
+	var index ocispec.Index
+	if err := json.Unmarshal(indexBytes, &index); err != nil {
+		return err
+	}
+
+	for _, manifestDesc := range index.Manifests {
+		if manifestDesc.Annotations[ocispec.AnnotationRefName] == ref {
+			index.Manifests = []ocispec.Descriptor{manifestDesc}
+			break
+		}
+	}
+
+	bundleIndexBytes, err := json.Marshal(index)
+	if err != nil {
+		return err
+	}
+	indexFile, err := os.Create(filepath.Join(tmpDir, "index.json"))
+	if err != nil {
+		return err
+	}
+	defer indexFile.Close()
+	_, err = indexFile.Write(bundleIndexBytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}

src/pkg/bundle/common.go

@@ -131,7 +131,7 @@ func (b *Bundler) ValidateBundleResources(bundle *types.UDSBundle, spinner *mess
 			if strings.Contains(pkg.Ref, "@sha256:") {
 				url = fmt.Sprintf("%s:%s", pkg.Repository, pkg.Ref)
 			}
-			remotePkg, err := bundler.NewRemoteBundler(pkg, url, nil, nil)
+			remotePkg, err := bundler.NewRemoteBundler(pkg, url, nil, nil, b.tmp)
 			if err != nil {
 				return err
 			}

src/pkg/bundle/remote.go

@@ -222,6 +222,5 @@ func (op *ociProvider) LoadBundle(_ int) (PathMap, error) {
 
 func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error {
 	// todo: implement moving bundles from one registry to another
-	message.Warnf("moving bundles in between remote registries not yet supported")
-	return nil
+	return fmt.Errorf("moving bundles in between remote registries not yet supported")
 }

src/pkg/bundler/remote.go

@@ -8,18 +8,24 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"path/filepath"
+	"strings"
+	"sync"
 
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/oci"
-	"github.com/defenseunicorns/zarf/src/pkg/utils"
+	zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils"
 	zarfTypes "github.com/defenseunicorns/zarf/src/types"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/content"
 	ocistore "oras.land/oras-go/v2/content/oci"
 
 	"github.com/defenseunicorns/uds-cli/src/config"
+	"github.com/defenseunicorns/uds-cli/src/pkg/cache"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
 	"github.com/defenseunicorns/uds-cli/src/types"
 )
 
@@ -31,10 +37,11 @@ type RemoteBundler struct {
 	RemoteSrc       *oci.OrasRemote
 	RemoteDst       *oci.OrasRemote
 	localDst        *ocistore.Store
+	tmpDir          string
 }
 
 // NewRemoteBundler creates a bundler to pull remote Zarf pkgs
-func NewRemoteBundler(pkg types.BundleZarfPackage, url string, localDst *ocistore.Store, remoteDst *oci.OrasRemote) (RemoteBundler, error) {
+func NewRemoteBundler(pkg types.BundleZarfPackage, url string, localDst *ocistore.Store, remoteDst *oci.OrasRemote, tmpDir string) (RemoteBundler, error) {
 	src, err := oci.NewOrasRemote(url)
 	if err != nil {
 		return RemoteBundler{}, err
@@ -44,7 +51,7 @@ func NewRemoteBundler(pkg types.BundleZarfPackage, url string, localDst *ocistor
 		return RemoteBundler{}, err
 	}
 	if localDst != nil {
-		return RemoteBundler{ctx: context.TODO(), RemoteSrc: src, localDst: localDst, PkgRootManifest: pkgRootManifest, pkg: pkg}, err
+		return RemoteBundler{ctx: context.TODO(), RemoteSrc: src, localDst: localDst, PkgRootManifest: pkgRootManifest, pkg: pkg, tmpDir: tmpDir}, err
 	}
 	return RemoteBundler{ctx: context.TODO(), RemoteSrc: src, RemoteDst: remoteDst, PkgRootManifest: pkgRootManifest, pkg: pkg}, err
 }
@@ -62,7 +69,7 @@ func (b *RemoteBundler) GetMetadata(url string, tmpDir string) (zarfTypes.ZarfPa
 	}
 	zarfYAML := zarfTypes.ZarfPackage{}
 	zarfYAMLPath := filepath.Join(tmpDir, config.ZarfYAML)
-	err = utils.ReadYaml(zarfYAMLPath, &zarfYAML)
+	err = zarfUtils.ReadYaml(zarfYAMLPath, &zarfYAML)
 	if err != nil {
 		return zarfTypes.ZarfPackage{}, err
 	}
@@ -93,28 +100,28 @@ func (b *RemoteBundler) PushManifest() (ocispec.Descriptor, error) {
 func (b *RemoteBundler) PushLayers(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) {
 	// get only the layers that are required by the components
 	spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", b.pkg.Name, currentPackageIter, totalPackages)
-	layersToCopy, err := getZarfLayers(b.RemoteSrc, b.pkg, b.PkgRootManifest)
+	layersToCopy, err := getZarfLayers(b.RemoteSrc, b.pkg, b.PkgRootManifest, b.tmpDir)
 	if err != nil {
 		return nil, err
 	}
 	if b.localDst != nil {
-		layerDescs, err := handleLocalCopy(layersToCopy, b, spinner, currentPackageIter, totalPackages)
+		layerDescs, err := b.handleLocalCopy(layersToCopy, spinner, currentPackageIter, totalPackages)
 		if err != nil {
 			return nil, err
 		}
 		// return layer descriptor so we can copy them into the tarball path map
 		return layerDescs, err
 	}
 	spinner.Updatef("Pushing package %s layers to registry (package %d of %d)", b.pkg.Name, currentPackageIter, totalPackages)
-	err = handleRemoteCopy(b, layersToCopy)
+	err = b.handleRemoteCopy(layersToCopy)
 	if err != nil {
 		return nil, err
 	}
 	return nil, err
 }
 
 // handleRemoteCopy copies a remote Zarf pkg to a remote OCI registry
-func handleRemoteCopy(b *RemoteBundler, layersToCopy []ocispec.Descriptor) error {
+func (b *RemoteBundler) handleRemoteCopy(layersToCopy []ocispec.Descriptor) error {
 	// stream copy if different registry
 	srcRef := b.RemoteSrc.Repo().Reference
 	dstRef := b.RemoteDst.Repo().Reference
@@ -157,36 +164,87 @@ func handleRemoteCopy(b *RemoteBundler, layersToCopy []ocispec.Descriptor) error
 }
 
 // handleLocalCopy copies a remote Zarf pkg to a local OCI store
-func handleLocalCopy(layersToCopy []ocispec.Descriptor, b *RemoteBundler, spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) {
+func (b *RemoteBundler) handleLocalCopy(layersToCopy []ocispec.Descriptor, spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) {
 	// pull layers from remote and write to OCI artifact dir
-	var layerDescs []ocispec.Descriptor
-	for i, layer := range layersToCopy {
+	var layerDescsToArchive []ocispec.Descriptor
+	var layersToPull []ocispec.Descriptor
+	estimatedBytes := int64(0)
+	for _, layer := range layersToCopy {
 		if layer.Digest == "" {
 			continue
 		}
 		// check if layer already exists
-		if exists, err := b.localDst.Exists(b.ctx, layer); exists {
+		if exists, _ := b.localDst.Exists(b.ctx, layer); exists {
 			continue
-		} else if err != nil {
+		} else if cache.Exists(layer.Digest.Encoded()) {
+			err := cache.Use(layer.Digest.Encoded(), filepath.Join(b.tmpDir, config.BlobsDir))
+			if err != nil {
+				return nil, err
+			}
+			layerDescsToArchive = append(layerDescsToArchive, layer)
+			continue
+		} else {
+			// grab layer to pull from OCI
+			if layer.MediaType != ocispec.MediaTypeImageManifest {
+				layersToPull = append(layersToPull, layer)
+				layerDescsToArchive = append(layerDescsToArchive, layer)
+				estimatedBytes += layer.Size
+				continue
+			}
+			// todo: what is the purpose of if block above again?
+			// todo: losing pkg root manifests when caching bc oras.copy gets them for us, but we don't retrieve from cache
+
+			layerDescsToArchive = append(layerDescsToArchive, layer)
+		}
+	}
+
+	// todo: weird flickering in TUI during Copy op (its bc of the spinner up top)
+
+	if len(layersToPull) > 0 {
+		// copy bundle
+		copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency)
+		// Create a thread to update a progress bar as we save the package to disk
+		doneSaving := make(chan int)
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go zarfUtils.RenderProgressBarForLocalDirWrite(b.tmpDir, estimatedBytes, &wg, doneSaving, fmt.Sprintf("Pulling bundle: %s", b.pkg.Name), fmt.Sprintf("Successfully pulled bundle: %s", b.pkg.Name))
+		rootPkgDesc, err := oras.Copy(context.TODO(), b.RemoteSrc.Repo(), b.RemoteSrc.Repo().Reference.String(), b.localDst, "", copyOpts)
+		if err != nil {
+			doneSaving <- 1
 			return nil, err
 		}
+		doneSaving <- 1
+		wg.Wait()
 
-		spinner.Updatef("Fetching %s layer %d of %d (package %d of %d)", b.pkg.Name, i+1, len(layersToCopy), currentPackageIter, totalPackages)
-		layerBytes, err := b.RemoteSrc.FetchLayer(layer)
+		// grab pkg root manifest for archiving
+		layerDescsToArchive = append(layerDescsToArchive, rootPkgDesc)
+
+		// cache only the image layers that were just pulled
+		for _, layer := range layersToPull {
+			if strings.Contains(layer.Annotations[ocispec.AnnotationTitle], config.BlobsDir) {
+				err = cache.Add(filepath.Join(b.tmpDir, config.BlobsDir, layer.Digest.Encoded()))
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	} else {
+		// need to grab pkg root manifest manually bc we didn't use oras.Copy()
+		pkgManifestBytes, err := json.Marshal(b.PkgRootManifest)
 		if err != nil {
 			return nil, err
 		}
-		layerDesc := content.NewDescriptorFromBytes(oci.ZarfLayerMediaTypeBlob, layerBytes)
-		if err := b.localDst.Push(b.ctx, layerDesc, bytes.NewReader(layerBytes)); err != nil {
+		pkgManifestDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, pkgManifestBytes)
+		if err = b.localDst.Push(context.TODO(), pkgManifestDesc, bytes.NewReader(pkgManifestBytes)); err != nil {
 			return nil, err
 		}
-		layerDescs = append(layerDescs, layerDesc)
+		layerDescsToArchive = append(layerDescsToArchive, pkgManifestDesc)
 	}
-	return layerDescs, nil
+	return layerDescsToArchive, nil
 }
 
 // getZarfLayers grabs the necessary Zarf pkg layers from a remote OCI registry
-func getZarfLayers(remote *oci.OrasRemote, pkg types.BundleZarfPackage, pkgRootManifest *oci.ZarfOCIManifest) ([]ocispec.Descriptor, error) {
+func getZarfLayers(remote *oci.OrasRemote, pkg types.BundleZarfPackage, pkgRootManifest *oci.ZarfOCIManifest, tmpDir string) ([]ocispec.Descriptor, error) {
 	layersFromComponents, err := remote.LayersFromRequestedComponents(pkg.OptionalComponents)
 	if err != nil {
 		return nil, err