Add support for OCI images

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/jinzhu/gorm v1.9.16
 	github.com/nlopes/slack v0.6.0
 	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.0-rc2
 	github.com/prometheus/client_golang v1.14.0
 	github.com/rusenask/cron v1.1.0
 	github.com/rusenask/docker-registry-client v0.0.0-20200210164146-049272422097
@@ -89,7 +90,7 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
@@ -146,7 +147,6 @@ require (
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -167,8 +167,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
-github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=

registry/docker/json.go

@@ -0,0 +1,48 @@
+package docker
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"regexp"
+)
+
+var ErrNoMorePages = errors.New("no more pages")
+
+// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5)
+// Link header. For example,
+//
+//	<http://registry.example.com/v2/_catalog?n=5&last=tag5>; type="application/json"; rel="next"
+//
+// The URL is _supposed_ to be wrapped by angle brackets `< ... >`,
+// but e.g., quay.io does not include them. Similarly, params like
+// `rel="next"` may not have quoted values in the wild.
+var nextLinkRE = regexp.MustCompile(`^ *<?([^;>]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`)
+
+func getNextLink(resp *http.Response) (string, error) {
+	for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
+		parts := nextLinkRE.FindStringSubmatch(link)
+		if parts != nil {
+			return parts[1], nil
+		}
+	}
+	return "", ErrNoMorePages
+}
+
+// getPaginatedJSON accepts a string and a pointer, and returns the
+// next page URL while updating pointed-to variable with a parsed JSON
+// value. When there are no more pages it returns `ErrNoMorePages`.
+func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) {
+	resp, err := registry.Client.Get(registry.url(url))
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	decoder := json.NewDecoder(resp.Body)
+	err = decoder.Decode(response)
+	if err != nil {
+		return "", err
+	}
+	return getNextLink(resp)
+}

registry/docker/manifest.go

@@ -0,0 +1,48 @@
+package docker
+
+import (
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	manifestv2 "github.com/docker/distribution/manifest/schema2"
+	"github.com/opencontainers/go-digest"
+	oci "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// ManifestDigest - get manifest digest
+func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) {
+	url := registry.url("/v2/%s/manifests/%s", repository, reference)
+	registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("Accept", manifestv2.MediaTypeManifest)
+	resp, err := registry.Client.Do(req)
+	if err != nil {
+		// Try OCI headers if error relates to OCI
+		if strings.Contains(err.Error(), "OCI index found, but accept header does not support OCI indexes") {
+			req.Header.Set("Accept", oci.MediaTypeImageIndex)
+			resp, err = registry.Client.Do(req)
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	defer resp.Body.Close()
+
+	if hdr := resp.Header.Get("Docker-Content-Digest"); hdr != "" {
+		return digest.Parse(hdr)
+	}
+
+	// Try to get digest from body instead, should be equal to what would be presented
+	// in Docker-Content-Digest
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return digest.FromBytes(body), nil
+}

registry/docker/manifest_test.go

@@ -0,0 +1,43 @@
+package docker
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	manifestV2 "github.com/docker/distribution/manifest/schema2"
+)
+
+func TestGetDigest(t *testing.T) {
+
+	req, err := http.NewRequest("GET", "https://registry.opensource.zalan.do/v2/teapot/external-dns/manifests/v0.4.8", nil)
+	if err != nil {
+		t.Fatalf("failed to create request: %s", err)
+	}
+	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("failed to request: %s", err)
+	}
+	defer resp.Body.Close()
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Add("content-type", "application/vnd.docker.distribution.manifest.v2+json; charset=ISO-8859-1")
+		io.Copy(w, resp.Body)
+	}))
+	defer ts.Close()
+
+	reg := New(ts.URL, "", "")
+
+	digest, err := reg.ManifestDigest(ts.URL, "notimportant")
+	if err != nil {
+		t.Errorf("failed to get digest")
+	}
+
+	if digest.String() != "sha256:7aa5175f39a7e8a4172972524302c9a8196f681e40d6ee5d2f6bf0ab7d600fee" {
+		t.Errorf("unexpected digest: %s", digest.String())
+	}
+}

registry/docker/registry.go

@@ -0,0 +1,115 @@
+package docker
+
+import (
+	"crypto/tls"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	drc "github.com/rusenask/docker-registry-client/registry"
+)
+
+type LogfCallback func(format string, args ...interface{})
+
+type Registry struct {
+	URL    string
+	Client *http.Client
+	Logf   LogfCallback
+}
+
+/*
+ * Pass log messages along to Go's "log" module.
+ */
+func Log(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+/*
+ * Create a new Registry with the given URL and credentials, then Ping()s it
+ * before returning it to verify that the registry is available.
+ *
+ * You can, alternately, construct a Registry manually by populating the fields.
+ * This passes http.DefaultTransport to WrapTransport when creating the
+ * http.Client.
+ */
+func New(registryURL, username, password string) *Registry {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		MaxIdleConns:          10,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	return newFromTransport(registryURL, username, password, transport, Log)
+}
+
+/*
+ * Create a new Registry, as with New, using an http.Transport that disables
+ * SSL certificate verification.
+ */
+func NewInsecure(registryURL, username, password string) *Registry {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+		MaxIdleConns:          10,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	return newFromTransport(registryURL, username, password, transport, Log)
+}
+
+func newFromTransport(registryURL, username, password string, transport *http.Transport, logf LogfCallback) *Registry {
+	url := strings.TrimSuffix(registryURL, "/")
+	registry := &Registry{
+		URL: url,
+		Client: &http.Client{
+			Transport: drc.WrapTransport(transport, url, username, password),
+		},
+		Logf: logf,
+	}
+
+	return registry
+}
+
+func (r *Registry) Ping() error {
+	url := r.url("/v2/")
+	r.Logf("registry.ping url=%s", url)
+	resp, err := r.Client.Get(url)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	return err
+}
+
+type tagsResponse struct {
+	Tags []string `json:"tags"`
+}
+
+func (r *Registry) url(pathTemplate string, args ...interface{}) string {
+	pathSuffix := fmt.Sprintf(pathTemplate, args...)
+	if strings.HasPrefix(pathSuffix, r.URL) {
+		return pathSuffix
+	}
+
+	url := fmt.Sprintf("%s%s", r.URL, pathSuffix)
+
+	return url
+}

registry/docker/tags.go

@@ -0,0 +1,21 @@
+package docker
+
+func (registry *Registry) Tags(repository string) (tags []string, err error) {
+	url := registry.url("/v2/%s/tags/list", repository)
+
+	var response tagsResponse
+	for {
+		registry.Logf("registry.tags url=%s repository=%s", url, repository)
+		url, err = registry.getPaginatedJSON(url, &response)
+		switch err {
+		case ErrNoMorePages:
+			tags = append(tags, response.Tags...)
+			return tags, nil
+		case nil:
+			tags = append(tags, response.Tags...)
+			continue
+		default:
+			return nil, err
+		}
+	}
+}

registry/docker/tags_test.go

@@ -0,0 +1,18 @@
+package docker
+
+import (
+	"testing"
+)
+
+func TestGetDigestDockerHub(t *testing.T) {
+	client := New("https://index.docker.io", "", "")
+
+	tags, err := client.Tags("karolisr/keel")
+	if err != nil {
+		t.Errorf("failed to get tags, error: %s", err)
+	}
+
+	if len(tags) == 0 {
+		t.Errorf("no tags?")
+	}
+}

registry/registry.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/rusenask/docker-registry-client/registry"
+	"github.com/keel-hq/keel/registry/docker"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -40,7 +40,7 @@ func New() *DefaultClient {
 	}
 	return &DefaultClient{
 		mu:         &sync.Mutex{},
-		registries: make(map[uint32]*registry.Registry),
+		registries: make(map[uint32]*docker.Registry),
 		insecure:   insecure,
 	}
 }
@@ -49,7 +49,7 @@ func New() *DefaultClient {
 type DefaultClient struct {
 	// a map of registries to reuse for polling
 	mu         *sync.Mutex
-	registries map[uint32]*registry.Registry
+	registries map[uint32]*docker.Registry
 	insecure   bool
 }
 
@@ -71,11 +71,11 @@ func hash(s string) uint32 {
 	return h.Sum32()
 }
 
-func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*registry.Registry, error) {
+func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*docker.Registry, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	var r *registry.Registry
+	var r *docker.Registry
 
 	h := hash(registryAddress + username + password)
 	r, ok := c.registries[h]
@@ -85,9 +85,9 @@ func (c *DefaultClient) getRegistryClient(registryAddress, username, password st
 
 	url := strings.TrimSuffix(registryAddress, "/")
 	if os.Getenv(EnvInsecure) == "true" {
-		r = registry.NewInsecure(url, username, password)
+		r = docker.NewInsecure(url, username, password)
 	} else {
-		r = registry.New(url, username, password)
+		r = docker.New(url, username, password)
 	}
 
 	r.Logf = LogFormatter