Skip to content

Commit

Permalink
Add support for OCI images
Browse files Browse the repository at this point in the history
  • Loading branch information
sjdaws committed Apr 11, 2023
1 parent a58d5a6 commit 2315ace
Show file tree
Hide file tree
Showing 11 changed files with 385 additions and 18 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ require (
github.com/jinzhu/gorm v1.9.16
github.com/nlopes/slack v0.6.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc2
github.com/prometheus/client_golang v1.14.0
github.com/rusenask/cron v1.1.0
github.com/rusenask/docker-registry-client v0.0.0-20200210164146-049272422097
Expand Down Expand Up @@ -89,7 +90,7 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
Expand Down Expand Up @@ -146,7 +147,6 @@ require (
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
Expand Down
48 changes: 48 additions & 0 deletions registry/docker/json.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package docker

import (
"encoding/json"
"errors"
"net/http"
"regexp"
)

var ErrNoMorePages = errors.New("no more pages")

// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5)
// Link header. For example,
//
// <http://registry.example.com/v2/_catalog?n=5&last=tag5>; type="application/json"; rel="next"
//
// The URL is _supposed_ to be wrapped by angle brackets `< ... >`,
// but e.g., quay.io does not include them. Similarly, params like
// `rel="next"` may not have quoted values in the wild.
var nextLinkRE = regexp.MustCompile(`^ *<?([^;>]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`)

func getNextLink(resp *http.Response) (string, error) {
for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
parts := nextLinkRE.FindStringSubmatch(link)
if parts != nil {
return parts[1], nil
}
}
return "", ErrNoMorePages
}

// getPaginatedJSON accepts a string and a pointer, and returns the
// next page URL while updating pointed-to variable with a parsed JSON
// value. When there are no more pages it returns `ErrNoMorePages`.
func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) {
resp, err := registry.Client.Get(registry.url(url))
if err != nil {
return "", err
}
defer resp.Body.Close()

decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return "", err
}
return getNextLink(resp)
}
48 changes: 48 additions & 0 deletions registry/docker/manifest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package docker

import (
"io/ioutil"
"net/http"
"strings"

manifestv2 "github.com/docker/distribution/manifest/schema2"
"github.com/opencontainers/go-digest"
oci "github.com/opencontainers/image-spec/specs-go/v1"
)

// ManifestDigest - get manifest digest
func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) {
url := registry.url("/v2/%s/manifests/%s", repository, reference)
registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)

req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}

req.Header.Set("Accept", manifestv2.MediaTypeManifest)
resp, err := registry.Client.Do(req)
if err != nil {
// Try OCI headers if error relates to OCI
if strings.Contains(err.Error(), "OCI index found, but accept header does not support OCI indexes") {
req.Header.Set("Accept", oci.MediaTypeImageIndex)
resp, err = registry.Client.Do(req)
}
if err != nil {
return "", err
}
}
defer resp.Body.Close()

if hdr := resp.Header.Get("Docker-Content-Digest"); hdr != "" {
return digest.Parse(hdr)
}

// Try to get digest from body instead, should be equal to what would be presented
// in Docker-Content-Digest
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
return digest.FromBytes(body), nil
}
43 changes: 43 additions & 0 deletions registry/docker/manifest_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package docker

import (
"io"
"net/http"
"net/http/httptest"
"testing"

manifestV2 "github.com/docker/distribution/manifest/schema2"
)

func TestGetDigest(t *testing.T) {

req, err := http.NewRequest("GET", "https://registry.opensource.zalan.do/v2/teapot/external-dns/manifests/v0.4.8", nil)
if err != nil {
t.Fatalf("failed to create request: %s", err)
}
req.Header.Set("Accept", manifestV2.MediaTypeManifest)

client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("failed to request: %s", err)
}
defer resp.Body.Close()

ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("content-type", "application/vnd.docker.distribution.manifest.v2+json; charset=ISO-8859-1")
io.Copy(w, resp.Body)
}))
defer ts.Close()

reg := New(ts.URL, "", "")

digest, err := reg.ManifestDigest(ts.URL, "notimportant")
if err != nil {
t.Errorf("failed to get digest")
}

if digest.String() != "sha256:7aa5175f39a7e8a4172972524302c9a8196f681e40d6ee5d2f6bf0ab7d600fee" {
t.Errorf("unexpected digest: %s", digest.String())
}
}
115 changes: 115 additions & 0 deletions registry/docker/registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package docker

import (
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"strings"
"time"

drc "github.com/rusenask/docker-registry-client/registry"
)

type LogfCallback func(format string, args ...interface{})

type Registry struct {
URL string
Client *http.Client
Logf LogfCallback
}

/*
* Pass log messages along to Go's "log" module.
*/
func Log(format string, args ...interface{}) {
log.Printf(format, args...)
}

/*
* Create a new Registry with the given URL and credentials, then Ping()s it
* before returning it to verify that the registry is available.
*
* You can, alternately, construct a Registry manually by populating the fields.
* This passes http.DefaultTransport to WrapTransport when creating the
* http.Client.
*/
func New(registryURL, username, password string) *Registry {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}

return newFromTransport(registryURL, username, password, transport, Log)
}

/*
* Create a new Registry, as with New, using an http.Transport that disables
* SSL certificate verification.
*/
func NewInsecure(registryURL, username, password string) *Registry {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}

return newFromTransport(registryURL, username, password, transport, Log)
}

func newFromTransport(registryURL, username, password string, transport *http.Transport, logf LogfCallback) *Registry {
url := strings.TrimSuffix(registryURL, "/")
registry := &Registry{
URL: url,
Client: &http.Client{
Transport: drc.WrapTransport(transport, url, username, password),
},
Logf: logf,
}

return registry
}

func (r *Registry) Ping() error {
url := r.url("/v2/")
r.Logf("registry.ping url=%s", url)
resp, err := r.Client.Get(url)
if resp != nil {
defer resp.Body.Close()
}
return err
}

type tagsResponse struct {
Tags []string `json:"tags"`
}

func (r *Registry) url(pathTemplate string, args ...interface{}) string {
pathSuffix := fmt.Sprintf(pathTemplate, args...)
if strings.HasPrefix(pathSuffix, r.URL) {
return pathSuffix
}

url := fmt.Sprintf("%s%s", r.URL, pathSuffix)

return url
}
21 changes: 21 additions & 0 deletions registry/docker/tags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package docker

func (registry *Registry) Tags(repository string) (tags []string, err error) {
url := registry.url("/v2/%s/tags/list", repository)

var response tagsResponse
for {
registry.Logf("registry.tags url=%s repository=%s", url, repository)
url, err = registry.getPaginatedJSON(url, &response)
switch err {
case ErrNoMorePages:
tags = append(tags, response.Tags...)
return tags, nil
case nil:
tags = append(tags, response.Tags...)
continue
default:
return nil, err
}
}
}
18 changes: 18 additions & 0 deletions registry/docker/tags_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package docker

import (
"testing"
)

func TestGetDigestDockerHub(t *testing.T) {
client := New("https://index.docker.io", "", "")

tags, err := client.Tags("karolisr/keel")
if err != nil {
t.Errorf("failed to get tags, error: %s", err)
}

if len(tags) == 0 {
t.Errorf("no tags?")
}
}
14 changes: 7 additions & 7 deletions registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"strings"
"sync"

"github.com/rusenask/docker-registry-client/registry"
"github.com/keel-hq/keel/registry/docker"

log "github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -40,7 +40,7 @@ func New() *DefaultClient {
}
return &DefaultClient{
mu: &sync.Mutex{},
registries: make(map[uint32]*registry.Registry),
registries: make(map[uint32]*docker.Registry),
insecure: insecure,
}
}
Expand All @@ -49,7 +49,7 @@ func New() *DefaultClient {
type DefaultClient struct {
// a map of registries to reuse for polling
mu *sync.Mutex
registries map[uint32]*registry.Registry
registries map[uint32]*docker.Registry
insecure bool
}

Expand All @@ -71,11 +71,11 @@ func hash(s string) uint32 {
return h.Sum32()
}

func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*registry.Registry, error) {
func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*docker.Registry, error) {
c.mu.Lock()
defer c.mu.Unlock()

var r *registry.Registry
var r *docker.Registry

h := hash(registryAddress + username + password)
r, ok := c.registries[h]
Expand All @@ -85,9 +85,9 @@ func (c *DefaultClient) getRegistryClient(registryAddress, username, password st

url := strings.TrimSuffix(registryAddress, "/")
if os.Getenv(EnvInsecure) == "true" {
r = registry.NewInsecure(url, username, password)
r = docker.NewInsecure(url, username, password)
} else {
r = registry.New(url, username, password)
r = docker.New(url, username, password)
}

r.Logf = LogFormatter
Expand Down
Loading

0 comments on commit 2315ace

Please sign in to comment.