feat: changes catalog and OCP release heads only workflows to use imp…

…licit ranges

When merging the catalog, unintentional changes to the upgrade and pruned bundles
can be created in the target FBC generated by oc-mirror.

To remove the requirement to merge FBCs, package ranges are being used in metadata for
heads-only workflows to generate FBCs that are scoped per the configuration instead of
generating differential FBCs and merging with previous FBCs. The OCP releases heads only
workflow has been updated work in the same way. To achieve the same optimization in regards
to download images, all images in the image mapping during each run are validated
against the metadata to make sure they do not exist in PastAssociations.

Closes openshift#377

BREAKING CHANGE: This change adds OCPMetadata and modifies OperatorMetadata on the Metadata Spec.
It also removes the catalog merging workflow.

Signed-off-by: Jennifer Power <barnabei.jennifer@gmail.com>

pkg/api/v1alpha2/types_include_config.go

@@ -62,7 +62,6 @@ func (ic *IncludeConfig) ConvertToDiffIncludeConfig() (dic action.DiffIncludeCon
 		case pkg.StartingBundle != "":
 			dpkg.Bundles = []string{pkg.StartingBundle}
 		}
-		dic.Packages = append(dic.Packages, dpkg)
 
 		for chIdx, ch := range pkg.Channels {
 			if ch.Name == "" {
@@ -81,6 +80,7 @@ func (ic *IncludeConfig) ConvertToDiffIncludeConfig() (dic action.DiffIncludeCon
 			}
 			dpkg.Channels = append(dpkg.Channels, dch)
 		}
+		dic.Packages = append(dic.Packages, dpkg)
 	}
 
 	return dic, nil

pkg/api/v1alpha2/types_include_config_test.go

@@ -0,0 +1,118 @@
+package v1alpha2
+
+import (
+	"testing"
+
+	"github.com/blang/semver/v4"
+	"github.com/operator-framework/operator-registry/alpha/action"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertToDiffIncludeConfig(t *testing.T) {
+	type spec struct {
+		name string
+		cfg  IncludeConfig
+		exp  action.DiffIncludeConfig
+	}
+
+	specs := []spec{
+		{
+			name: "Valid/WithChannels",
+			cfg: IncludeConfig{
+				Packages: []IncludePackage{
+					{
+						Name: "bar",
+						Channels: []IncludeChannel{
+							{
+								Name: "stable",
+								IncludeBundle: IncludeBundle{
+									StartingVersion: semver.MustParse("0.1.0"),
+								},
+							},
+						},
+					},
+					{
+						Name: "foo",
+						Channels: []IncludeChannel{
+							{
+								Name: "stable",
+								IncludeBundle: IncludeBundle{
+									StartingVersion: semver.MustParse("0.1.0"),
+								},
+							},
+						},
+					},
+				},
+			},
+			exp: action.DiffIncludeConfig{
+				Packages: []action.DiffIncludePackage{
+					{
+						Name: "bar",
+						Channels: []action.DiffIncludeChannel{
+							{
+								Name: "stable",
+								Versions: []semver.Version{
+									semver.MustParse("0.1.0"),
+								},
+							},
+						},
+					},
+					{
+						Name: "foo",
+						Channels: []action.DiffIncludeChannel{
+							{
+								Name: "stable",
+								Versions: []semver.Version{
+									semver.MustParse("0.1.0"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "Valid/NoChannels",
+			cfg: IncludeConfig{
+				Packages: []IncludePackage{
+					{
+						Name: "bar",
+						IncludeBundle: IncludeBundle{
+							StartingVersion: semver.MustParse("0.1.0"),
+						},
+					},
+					{
+						Name: "foo",
+						IncludeBundle: IncludeBundle{
+							StartingVersion: semver.MustParse("0.1.0"),
+						},
+					},
+				},
+			},
+			exp: action.DiffIncludeConfig{
+				Packages: []action.DiffIncludePackage{
+					{
+						Name: "bar",
+						Versions: []semver.Version{
+							semver.MustParse("0.1.0"),
+						},
+					},
+					{
+						Name: "foo",
+						Versions: []semver.Version{
+							semver.MustParse("0.1.0"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, s := range specs {
+		t.Run(s.name, func(t *testing.T) {
+			dic, err := s.cfg.ConvertToDiffIncludeConfig()
+			require.NoError(t, err)
+			require.Equal(t, s.exp, dic)
+		})
+	}
+}

pkg/api/v1alpha2/types_metadata.go

@@ -46,6 +46,8 @@ type PastMirror struct {
 	Mirror Mirror `json:"mirror"`
 	// Operators are metadata about the set of mirrored operators in a mirror operation.
 	Operators []OperatorMetadata `json:"operators,omitempty"`
+	// OCPReleases are metadata about the set of mirrored OCP release channels in a mirror operation.
+	OCPReleases []OCPMetadata `json:"ocpReleases,omitempty"`
 	// Associations are metadata about the set of mirrored images including
 	// child manifest and layer digest information
 	Associations []Association `json:"associations,omitempty"`
@@ -59,6 +61,22 @@ type OperatorMetadata struct {
 	// This image will be pulled using the pull secret
 	// in the metadata's Mirror config for this catalog.
 	ImagePin string `json:"imagePin"`
+	// IncludeConfig in OperatorMetadata holds the starting
+	// versions of all newly mirrored catalogs. This will
+	// be populated the first time a catalog is mirrored
+	// and copied the remaining runs.
+	IncludeConfig `json:",inline"`
+}
+
+// ReleaseMetadata holds an Release's post-mirror metadata.
+type OCPMetadata struct {
+	// Release references a channel name from the mirror spec.
+	ReleaseChannel string `json:"channel"`
+	// MinVersion in ReleaseMetadata holds the starting
+	// versions of all newly mirrored channels. This will
+	// be populated the first time a channel is mirrored
+	// and copied the remaining runs.
+	MinVersion string `json:"minVersion"`
 }
 
 var _ io.Writer = &InlinedIndex{}

pkg/cli/mirror/catalog_images.go

@@ -10,33 +10,25 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/remotes"
-	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/crane"
-	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/layout"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/openshift/library-go/pkg/image/reference"
-	"github.com/operator-framework/operator-registry/alpha/action"
-	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	"github.com/operator-framework/operator-registry/pkg/containertools"
-	operatorimage "github.com/operator-framework/operator-registry/pkg/image"
 	"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
 	"github.com/sirupsen/logrus"
 
 	"github.com/openshift/oc-mirror/pkg/api/v1alpha2"
+	"github.com/openshift/oc-mirror/pkg/config"
 	"github.com/openshift/oc-mirror/pkg/image"
 	"github.com/openshift/oc-mirror/pkg/image/builder"
-	"github.com/openshift/oc-mirror/pkg/operator"
 	"github.com/openshift/oc/pkg/cli/image/imagesource"
 )
 
 // unpackCatalog will unpack file-based catalogs if they exists
 func (o *MirrorOptions) unpackCatalog(dstDir string, filesInArchive map[string]string) (bool, error) {
 	var found bool
-	if err := unpack(CatalogsDir, dstDir, filesInArchive); err != nil {
+	if err := unpack(config.CatalogsDir, dstDir, filesInArchive); err != nil {
 		nferr := &ErrArchiveFileNotFound{}
 		if errors.As(err, &nferr) || errors.Is(err, os.ErrNotExist) {
 			logrus.Debug("No catalogs found in archive, skipping catalog rebuild")
@@ -64,7 +56,7 @@ func (o *MirrorOptions) rebuildCatalogs(ctx context.Context, dstDir string) (ima
 
 		// Skip the layouts dir because we only need
 		// to process the parent directory one time
-		if filepath.Base(fpath) == LayoutsDir {
+		if filepath.Base(fpath) == config.LayoutsDir {
 			return filepath.SkipDir
 		}
 
@@ -77,9 +69,9 @@ func (o *MirrorOptions) rebuildCatalogs(ctx context.Context, dstDir string) (ima
 		slashPath := filepath.ToSlash(fpath)
 		if base := path.Base(slashPath); base == "index.json" {
 			slashPath = path.Dir(slashPath)
-			slashPath = strings.TrimSuffix(slashPath, IndexDir)
+			slashPath = strings.TrimSuffix(slashPath, config.IndexDir)
 
-			repoPath := strings.TrimPrefix(slashPath, fmt.Sprintf("%s/%s/", dstDir, CatalogsDir))
+			repoPath := strings.TrimPrefix(slashPath, fmt.Sprintf("%s/%s/", dstDir, config.CatalogsDir))
 			regRepoNs, id := path.Split(path.Dir(repoPath))
 			regRepoNs = path.Clean(regRepoNs)
 			var img string
@@ -110,22 +102,13 @@ func (o *MirrorOptions) rebuildCatalogs(ctx context.Context, dstDir string) (ima
 		return nil, err
 	}
 
-	resolver, err := containerdregistry.NewResolver("", o.DestSkipTLS, o.DestPlainHTTP, nil)
-	if err != nil {
-		return nil, fmt.Errorf("error creating image resolver: %v", err)
-	}
-	reg, err := containerdregistry.NewRegistry(
-		containerdregistry.SkipTLSVerify(o.DestSkipTLS),
-		containerdregistry.WithPlainHTTP(o.DestPlainHTTP),
-		containerdregistry.WithCacheDir(filepath.Join(dstDir, "cache")),
-	)
-	if err != nil {
+	if err := o.processCatalogRefs(ctx, catalogsByImage); err != nil {
 		return nil, err
 	}
-	defer reg.Destroy()
 
-	if err := o.processCatalogRefs(ctx, catalogsByImage, reg, resolver); err != nil {
-		return nil, err
+	resolver, err := containerdregistry.NewResolver("", o.DestSkipTLS, o.DestPlainHTTP, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error creating image resolver: %v", err)
 	}
 
 	// Resolve the image's digest for ICSP creation.
@@ -141,7 +124,7 @@ func (o *MirrorOptions) rebuildCatalogs(ctx context.Context, dstDir string) (ima
 	return refs, nil
 }
 
-func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage map[imagesource.TypedImageReference]string, reg operatorimage.Registry, resolver remotes.Resolver) error {
+func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage map[imagesource.TypedImageReference]string) error {
 	for ctlgRef, artifactDir := range catalogsByImage {
 		// An image for a particular catalog may not exist in the mirror registry yet,
 		// ex. when publish is run for the first time for a catalog (full/headsonly).
@@ -159,87 +142,30 @@ func (o *MirrorOptions) processCatalogRefs(ctx context.Context, catalogsByImage
 		// Check push permissions before trying to resolve for Quay compatibility
 		nameOpts := getNameOpts(destInsecure)
 		remoteOpts := getRemoteOpts(ctx, destInsecure)
-		ref, err := name.ParseReference(refExact, nameOpts...)
-		if err != nil {
-			return err
-		}
-		err = remote.CheckPushPermission(ref, authn.DefaultKeychain, createRT(destInsecure))
-		if err != nil {
-			return err
-		}
 		imgBuilder := &builder.ImageBuilder{
 			NameOpts:   nameOpts,
 			RemoteOpts: remoteOpts,
 		}
 
-		switch _, _, rerr := resolver.Resolve(ctx, refExact); {
-		case errors.Is(rerr, errdefs.ErrNotFound):
-			logrus.Infof("Catalog image %q found, building from file with file-based catalog ", refExact)
-
-			add, err := builder.LayerFromPath("/configs", filepath.Join(artifactDir, IndexDir, "index.json"))
-			if err != nil {
-				return fmt.Errorf("error creating add layer: %v", err)
-			}
-
-			// Since we are defining the FBC as index.json, remove
-			// any .yaml files from the initial image to ensure they are not processed instead
-			deleted, err := deleteLayer("/configs/.wh.index.yaml")
-			if err != nil {
-				return fmt.Errorf("error creating deleted layer: %v", err)
-			}
-			layers = append(layers, add, deleted)
-
-			layoutDir := filepath.Join(artifactDir, LayoutsDir)
-			layoutPath, err = imgBuilder.CreateLayout("", layoutDir)
-			if err != nil {
-				return fmt.Errorf("error creating OCI layout: %v", err)
-			}
-		case rerr == nil:
-			logrus.Infof("Catalog image %q found, rendering with file-based catalog", refExact)
+		logrus.Infof("Rendering catalog image %q with file-based catalog ", refExact)
 
-			dcDir := filepath.Join(artifactDir, IndexDir)
-			dc, err := action.Render{
-				// Order the old ctlgRef before dcDir so new packages/channels/bundles overwrite
-				// existing counterparts.
-				Refs:           []string{refExact, dcDir},
-				AllowedRefMask: action.RefAll,
-				Registry:       reg,
-			}.Run(ctx)
-			if err != nil {
-				return err
-			}
-			// Remove any duplicate objects
-			merger := &operator.TwoWayStrategy{}
-			if err := merger.Merge(dc); err != nil {
-				return err
-			}
-			dcDirToBuild := filepath.Join(dcDir, "rendered")
-			if err := os.MkdirAll(dcDirToBuild, os.ModePerm); err != nil {
-				return err
-			}
-			renderedPath := filepath.Join(dcDirToBuild, "index.json")
-			f, err := os.Create(renderedPath)
-			if err != nil {
-				return err
-			}
-			defer f.Close()
-			if err := declcfg.WriteJSON(*dc, f); err != nil {
-				return err
-			}
+		add, err := builder.LayerFromPath("/configs", filepath.Join(artifactDir, config.IndexDir, "index.json"))
+		if err != nil {
+			return fmt.Errorf("error creating add layer: %v", err)
+		}
 
-			add, err := builder.LayerFromPath("/configs", renderedPath)
-			if err != nil {
-				return fmt.Errorf("error creating add layer: %v", err)
-			}
-			layers = append(layers, add)
+		// Since we are defining the FBC as index.json, remove
+		// any .yaml files from the initial image to ensure they are not processed instead
+		deleted, err := deleteLayer("/configs/.wh.index.yaml")
+		if err != nil {
+			return fmt.Errorf("error creating deleted layer: %v", err)
+		}
+		layers = append(layers, add, deleted)
 
-			layoutDir := filepath.Join(dcDir, "layout")
-			layoutPath, err = imgBuilder.CreateLayout(refExact, layoutDir)
-			if err != nil {
-				return fmt.Errorf("error creating OCI layout: %v", err)
-			}
-		default:
-			return fmt.Errorf("error resolving existing catalog image %q: %v", refExact, rerr)
+		layoutDir := filepath.Join(artifactDir, config.LayoutsDir)
+		layoutPath, err = imgBuilder.CreateLayout("", layoutDir)
+		if err != nil {
+			return fmt.Errorf("error creating OCI layout: %v", err)
 		}
 
 		update := func(cfg *v1.ConfigFile) {