Add support for using an OCI image as source

This change defines an OCIImage CRD that allows a user to specify a
given image to use as source.  The contents of the image (including
images with multiple layers) are converted into a TAR and exposed to
consumers following the same conventions as the other source types.

  apiVersion: source.toolkit.fluxcd.io/v1beta1
  kind: OCIImage
  metadata:
    name: ociimage-sample
  spec:
    image:    index.docker.io/stefanprodan/podinfo:latest
    interval: 1m

In addition to the image, interval, timeout, ignore, and suspend keys
(all of which behave consistently with the existing source types) this
CRD also defines both imagePullSecrets and serviceAccountName keys
which provide ways to contribute registry connection credentials for the
specified image.

This change also adds a new way to write to the storage archive by
streaming data from an incoming TAR without writing it to the
filesystem.  A couple of code and test functions were extracted to
reuse common functionality for both archive strategies.

Signed-off-by: Ben Hale <bhale@vmware.com>

.github/workflows/e2e.yaml

@@ -56,6 +56,7 @@ jobs:
           kubectl -n source-system wait gitrepository/gitrepository-sample --for=condition=ready --timeout=1m
           kubectl -n source-system wait helmrepository/helmrepository-sample --for=condition=ready --timeout=1m
           kubectl -n source-system wait helmchart/helmchart-sample --for=condition=ready --timeout=1m
+          kubectl -n source-system wait ociimage/ociimage-sample --for=condition=ready --timeout=1m
           kubectl -n source-system delete -f ./config/samples
       - name: Run HelmChart values file tests
         run: |
@@ -113,6 +114,7 @@ jobs:
           kubectl -n source-system get gitrepositories -oyaml
           kubectl -n source-system get helmrepositories -oyaml
           kubectl -n source-system get helmcharts -oyaml
+          kubectl -n source-system get ociimages -oyaml
           kubectl -n source-system get all
           kubectl -n source-system logs deploy/source-controller
           kubectl -n minio get all

PROJECT

@@ -13,4 +13,7 @@ resources:
 - group: source
   kind: Bucket
   version: v1beta1
+- group: source
+  kind: OCIImage
+  version: v1beta1
 version: "2"

api/v1beta1/ociimage_types.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	"github.com/fluxcd/pkg/apis/meta"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// OCIImageKind is the string representation of a OCIImage.
+	OCIImageKind = "OCIImage"
+)
+
+// OCIImageSpec defines the desired state of OCIImage
+type OCIImageSpec struct {
+
+	// Image is a reference to an image in a remote registry
+	// +required
+	Image string `json:"image"`
+
+	// ImagePullSecrets contains the names of the Kubernetes Secrets containing registry login
+	// information to resolve image metadata.
+	// +optional
+	ImagePullSecrets []meta.LocalObjectReference `json:"imagePullSecrets,omitempty"`
+
+	// ServiceAccountName is the name of the Kubernetes ServiceAccount used to authenticate
+	// the image pull if the service account has attached pull secrets. For more information:
+	// https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account
+	// +optional
+	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// The interval at which to check for image updates.
+	// +required
+	Interval metav1.Duration `json:"interval"`
+
+	// The timeout for remote OCI Image operations like pulling, defaults to 20s.
+	// +kubebuilder:default="20s"
+	// +optional
+	Timeout *metav1.Duration `json:"timeout,omitempty"`
+
+	// Ignore overrides the set of excluded patterns in the .sourceignore format
+	// (which is the same as .gitignore). If not provided, a default will be used,
+	// consult the documentation for your version to find out what those are.
+	// +optional
+	Ignore *string `json:"ignore,omitempty"`
+
+	// This flag tells the controller to suspend the reconciliation of this source.
+	// +optional
+	Suspend bool `json:"suspend,omitempty"`
+}
+
+// OCIImageStatus defines the observed state of OCIImage
+type OCIImageStatus struct {
+	// ObservedGeneration is the last observed generation.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Conditions holds the conditions for the OCIImage.
+	// +optional
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// URL is the download link for the artifact output of the last image
+	// sync.
+	// +optional
+	URL string `json:"url,omitempty"`
+
+	// Artifact represents the output of the last successful image sync.
+	// +optional
+	Artifact *Artifact `json:"artifact,omitempty"`
+
+	meta.ReconcileRequestStatus `json:",inline"`
+}
+
+const (
+	// OCIImageOperationSucceedReason represents the fact that the image pull operation succeeded.
+	OCIImageOperationSucceedReason string = "OCIImageOperationSucceed"
+
+	// OCIImageOperationFailedReason represents the fact that the image pull operation failed.
+	OCIImageOperationFailedReason string = "OCIImageOperationFailed"
+)
+
+// OCIImageProgressing resets the conditions of the OCIImage to
+// metav1.Condition of type meta.ReadyCondition with status 'Unknown' and
+// meta.ProgressingReason reason and message. It returns the modified
+// OCCIImage.
+func OCIImageProgressing(image OCIImage) OCIImage {
+	image.Status.ObservedGeneration = image.Generation
+	image.Status.URL = ""
+	image.Status.Conditions = []metav1.Condition{}
+	meta.SetResourceCondition(&image, meta.ReadyCondition, metav1.ConditionUnknown, meta.ProgressingReason, "reconciliation in progress")
+	return image
+}
+
+// OCIImageReady sets the given Artifact and URL on the OCIImage and
+// sets the meta.ReadyCondition to 'True', with the given reason and message. It
+// returns the modified OCIImage.
+func OCIImageReady(image OCIImage, artifact Artifact, url, reason, message string) OCIImage {
+	image.Status.Artifact = &artifact
+	image.Status.URL = url
+	meta.SetResourceCondition(&image, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
+	return image
+}
+
+// OCIImageNotReady sets the meta.ReadyCondition on the given OCIImage
+// to 'False', with the given reason and message. It returns the modified
+// OCIImage.
+func OCIImageNotReady(image OCIImage, reason, message string) OCIImage {
+	meta.SetResourceCondition(&image, meta.ReadyCondition, metav1.ConditionFalse, reason, message)
+	return image
+}
+
+// OCIImageReadyMessage returns the message of the metav1.Condition of type
+// meta.ReadyCondition with status 'True' if present, or an empty string.
+func OCIImageReadyMessage(image OCIImage) string {
+	if c := apimeta.FindStatusCondition(image.Status.Conditions, meta.ReadyCondition); c != nil {
+		if c.Status == metav1.ConditionTrue {
+			return c.Message
+		}
+	}
+	return ""
+}
+
+// GetArtifact returns the latest artifact from the source if present in the
+// status sub-resource.
+func (in *OCIImage) GetArtifact() *Artifact {
+	return in.Status.Artifact
+}
+
+// GetImagePullSecretNames returns the names of all the configured image pull secrets.
+func (in *OCIImage) GetImagePullSecretNames() []string {
+	var n []string
+
+	for _, i := range in.Spec.ImagePullSecrets {
+		n = append(n, i.Name)
+	}
+
+	return n
+}
+
+// GetStatusConditions returns a pointer to the Status.Conditions slice
+func (in *OCIImage) GetStatusConditions() *[]metav1.Condition {
+	return &in.Status.Conditions
+}
+
+// GetInterval returns the interval at which the source is updated.
+func (in *OCIImage) GetInterval() metav1.Duration {
+	return in.Spec.Interval
+}
+
+// +genclient
+// +genclient:Namespaced
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="Image",type=string,JSONPath=`.spec.image`
+// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description=""
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description=""
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description=""
+
+// OCIImage is the Schema for the ociimages API
+type OCIImage struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   OCIImageSpec   `json:"spec,omitempty"`
+	Status OCIImageStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// OCIImageList contains a list of OCIImage
+type OCIImageList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OCIImage `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&OCIImage{}, &OCIImageList{})
+}