Skip to content

Commit

Permalink
*: add client support for discovery-srv-name
Browse files Browse the repository at this point in the history
Add support for --discovery-srv-name flag to etcdctl, gRPC proxy, and etcd gateway.
  • Loading branch information
hexfusion authored Nov 9, 2018
2 parents 83304cf + e0f7807 commit 9454c4c
Show file tree
Hide file tree
Showing 18 changed files with 240 additions and 103 deletions.
19 changes: 19 additions & 0 deletions CHANGELOG-3.3.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,25 @@ Previous change logs can be found at [CHANGELOG-3.2](https://github.com/etcd-io/
The [minimum recommended etcd versions to run in **production**](https://groups.google.com/d/msg/etcd-dev/nZQl17RjxHQ/FkC_rZ_4AwAJT) is 3.1.11+, 3.2.10+, and 3.3.0+.


<hr>

## [v3.3.11](https://github.com/coreos/etcd/releases/tag/v3.3.11) (2018-TBD)

See [code changes](https://github.com/coreos/etcd/compare/v3.3.10...v3.3.11) and [v3.3 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**

### etcdctl v3

- Add [`etcdctl --discovery-srv-name`](https://github.com/etcd-io/etcd/pull/10250) flag.

### gRPC Proxy

- Add [`etcd proxy --discovery-srv-name`](https://github.com/etcd-io/etcd/pull/10250) flag.

### etcd gateway

- Add [`etcd gateway --discovery-srv-name`](https://github.com/etcd-io/etcd/pull/10250) flag.


<hr>


Expand Down
6 changes: 3 additions & 3 deletions client/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
// Discoverer is an interface that wraps the Discover method.
type Discoverer interface {
// Discover looks up the etcd servers for the domain.
Discover(domain string) ([]string, error)
Discover(domain string, serviceName string) ([]string, error)
}

type srvDiscover struct{}
Expand All @@ -31,8 +31,8 @@ func NewSRVDiscover() Discoverer {
return &srvDiscover{}
}

func (d *srvDiscover) Discover(domain string) ([]string, error) {
srvs, err := srv.GetClient("etcd-client", domain)
func (d *srvDiscover) Discover(domain string, serviceName string) ([]string, error) {
srvs, err := srv.GetClient("etcd-client", domain, serviceName)
if err != nil {
return nil, err
}
Expand Down
14 changes: 9 additions & 5 deletions etcdctl/ctlv2/command/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,15 @@ func getPeersFlagValue(c *cli.Context) []string {
}

func getDomainDiscoveryFlagValue(c *cli.Context) ([]string, error) {
domainstr, insecure := getDiscoveryDomain(c)
domainstr, insecure, serviceName := getDiscoveryDomain(c)

// If we still don't have domain discovery, return nothing
if domainstr == "" {
return []string{}, nil
}

discoverer := client.NewSRVDiscover()
eps, err := discoverer.Discover(domainstr)
eps, err := discoverer.Discover(domainstr, serviceName)
if err != nil {
return nil, err
}
Expand All @@ -113,15 +113,19 @@ func getDomainDiscoveryFlagValue(c *cli.Context) ([]string, error) {
return ret, err
}

func getDiscoveryDomain(c *cli.Context) (domainstr string, insecure bool) {
func getDiscoveryDomain(c *cli.Context) (domainstr string, insecure bool, serviceName string) {
domainstr = c.GlobalString("discovery-srv")
// Use an environment variable if nothing was supplied on the
// command line
if domainstr == "" {
domainstr = os.Getenv("ETCDCTL_DISCOVERY_SRV")
}
insecure = c.GlobalBool("insecure-discovery") || (os.Getenv("ETCDCTL_INSECURE_DISCOVERY") != "")
return domainstr, insecure
serviceName = c.GlobalString("discovery-srv-name")
if serviceName == "" {
serviceName = os.Getenv("ETCDCTL_DISCOVERY_SRV_NAME")
}
return domainstr, insecure, serviceName
}

func getEndpoints(c *cli.Context) ([]string, error) {
Expand Down Expand Up @@ -168,7 +172,7 @@ func getTransport(c *cli.Context) (*http.Transport, error) {
keyfile = os.Getenv("ETCDCTL_KEY_FILE")
}

discoveryDomain, insecure := getDiscoveryDomain(c)
discoveryDomain, insecure, _ := getDiscoveryDomain(c)
if insecure {
discoveryDomain = ""
}
Expand Down
37 changes: 24 additions & 13 deletions etcdctl/ctlv3/command/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,15 @@ import (
// GlobalFlags are flags that defined globally
// and are inherited to all sub-commands.
type GlobalFlags struct {
Insecure bool
InsecureSkipVerify bool
InsecureDiscovery bool
Endpoints []string
DialTimeout time.Duration
CommandTimeOut time.Duration
KeepAliveTime time.Duration
KeepAliveTimeout time.Duration
Insecure bool
InsecureSkipVerify bool
InsecureDiscovery bool
Endpoints []string
DialTimeout time.Duration
CommandTimeOut time.Duration
KeepAliveTime time.Duration
KeepAliveTimeout time.Duration
DNSClusterServiceName string

TLS transport.TLSInfo

Expand Down Expand Up @@ -75,8 +76,9 @@ type authCfg struct {
}

type discoveryCfg struct {
domain string
insecure bool
domain string
insecure bool
serviceName string
}

var display printer = &simplePrinter{}
Expand Down Expand Up @@ -390,10 +392,19 @@ func discoverySrvFromCmd(cmd *cobra.Command) string {
return domainStr
}

func discoveryDNSClusterServiceNameFromCmd(cmd *cobra.Command) string {
serviceNameStr, err := cmd.Flags().GetString("discovery-srv-name")
if err != nil {
ExitWithError(ExitBadArgs, err)
}
return serviceNameStr
}

func discoveryCfgFromCmd(cmd *cobra.Command) *discoveryCfg {
return &discoveryCfg{
domain: discoverySrvFromCmd(cmd),
insecure: insecureDiscoveryFromCmd(cmd),
domain: discoverySrvFromCmd(cmd),
insecure: insecureDiscoveryFromCmd(cmd),
serviceName: discoveryDNSClusterServiceNameFromCmd(cmd),
}
}

Expand Down Expand Up @@ -422,7 +433,7 @@ func endpointsFromFlagValue(cmd *cobra.Command) ([]string, error) {
return []string{}, nil
}

srvs, err := srv.GetClient("etcd-client", discoveryCfg.domain)
srvs, err := srv.GetClient("etcd-client", discoveryCfg.domain, discoveryCfg.serviceName)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions etcdctl/ctlv3/ctl.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&globalFlags.User, "user", "", "username[:password] for authentication (prompt if password is not supplied)")
rootCmd.PersistentFlags().StringVar(&globalFlags.Password, "password", "", "password for authentication (if this option is used, --user option shouldn't include password)")
rootCmd.PersistentFlags().StringVarP(&globalFlags.TLS.ServerName, "discovery-srv", "d", "", "domain name to query for SRV records describing cluster endpoints")
rootCmd.PersistentFlags().StringVarP(&globalFlags.DNSClusterServiceName, "discovery-srv-name", "", "", "service name to query when using DNS discovery")

rootCmd.AddCommand(
command.NewGetCommand(),
Expand Down
16 changes: 9 additions & 7 deletions etcdmain/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,13 @@ import (
)

var (
gatewayListenAddr string
gatewayEndpoints []string
gatewayDNSCluster string
gatewayInsecureDiscovery bool
getewayRetryDelay time.Duration
gatewayCA string
gatewayListenAddr string
gatewayEndpoints []string
gatewayDNSCluster string
gatewayDNSClusterServiceName string
gatewayInsecureDiscovery bool
getewayRetryDelay time.Duration
gatewayCA string
)

var (
Expand Down Expand Up @@ -68,6 +69,7 @@ func newGatewayStartCommand() *cobra.Command {

cmd.Flags().StringVar(&gatewayListenAddr, "listen-addr", "127.0.0.1:23790", "listen address")
cmd.Flags().StringVar(&gatewayDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster")
cmd.Flags().StringVar(&gatewayDNSClusterServiceName, "discovery-srv-name", "", "service name to query when using DNS discovery")
cmd.Flags().BoolVar(&gatewayInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records")
cmd.Flags().StringVar(&gatewayCA, "trusted-ca-file", "", "path to the client server TLS CA file.")

Expand Down Expand Up @@ -97,7 +99,7 @@ func startGateway(cmd *cobra.Command, args []string) {
os.Exit(1)
}

srvs := discoverEndpoints(lg, gatewayDNSCluster, gatewayCA, gatewayInsecureDiscovery)
srvs := discoverEndpoints(lg, gatewayDNSCluster, gatewayCA, gatewayInsecureDiscovery, gatewayDNSClusterServiceName)
if len(srvs.Endpoints) == 0 {
// no endpoints discovered, fall back to provided endpoints
srvs.Endpoints = gatewayEndpoints
Expand Down
22 changes: 12 additions & 10 deletions etcdmain/grpc_proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,15 @@ import (
)

var (
grpcProxyListenAddr string
grpcProxyMetricsListenAddr string
grpcProxyEndpoints []string
grpcProxyDNSCluster string
grpcProxyInsecureDiscovery bool
grpcProxyDataDir string
grpcMaxCallSendMsgSize int
grpcMaxCallRecvMsgSize int
grpcProxyListenAddr string
grpcProxyMetricsListenAddr string
grpcProxyEndpoints []string
grpcProxyDNSCluster string
grpcProxyDNSClusterServiceName string
grpcProxyInsecureDiscovery bool
grpcProxyDataDir string
grpcMaxCallSendMsgSize int
grpcMaxCallRecvMsgSize int

// tls for connecting to etcd

Expand Down Expand Up @@ -111,7 +112,8 @@ func newGRPCProxyStartCommand() *cobra.Command {
}

cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address")
cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster")
cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "domain name to query for SRV records describing cluster endpoints")
cmd.Flags().StringVar(&grpcProxyDNSClusterServiceName, "discovery-srv-name", "", "service name to query when using DNS discovery")
cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for /metrics requests on an additional interface")
cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records")
cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints")
Expand Down Expand Up @@ -249,7 +251,7 @@ func checkArgs() {
}

func mustNewClient(lg *zap.Logger) *clientv3.Client {
srvs := discoverEndpoints(lg, grpcProxyDNSCluster, grpcProxyCA, grpcProxyInsecureDiscovery)
srvs := discoverEndpoints(lg, grpcProxyDNSCluster, grpcProxyCA, grpcProxyInsecureDiscovery, grpcProxyDNSClusterServiceName)
eps := srvs.Endpoints
if len(eps) == 0 {
eps = grpcProxyEndpoints
Expand Down
4 changes: 2 additions & 2 deletions etcdmain/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ import (
"go.uber.org/zap"
)

func discoverEndpoints(lg *zap.Logger, dns string, ca string, insecure bool) (s srv.SRVClients) {
func discoverEndpoints(lg *zap.Logger, dns string, ca string, insecure bool, serviceName string) (s srv.SRVClients) {
if dns == "" {
return s
}
srvs, err := srv.GetClient("etcd-client", dns)
srvs, err := srv.GetClient("etcd-client", dns, serviceName)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down
18 changes: 15 additions & 3 deletions pkg/srv/srv.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ type SRVClients struct {
}

// GetClient looks up the client endpoints for a service and domain.
func GetClient(service, domain string) (*SRVClients, error) {
func GetClient(service, domain string, serviceName string) (*SRVClients, error) {
var urls []*url.URL
var srvs []*net.SRV

Expand All @@ -115,8 +115,8 @@ func GetClient(service, domain string) (*SRVClients, error) {
return nil
}

errHTTPS := updateURLs(service+"-ssl", "https")
errHTTP := updateURLs(service, "http")
errHTTPS := updateURLs(GetSRVService(service, serviceName, "https"), "https")
errHTTP := updateURLs(GetSRVService(service, serviceName, "http"), "http")

if errHTTPS != nil && errHTTP != nil {
return nil, fmt.Errorf("dns lookup errors: %s and %s", errHTTPS, errHTTP)
Expand All @@ -128,3 +128,15 @@ func GetClient(service, domain string) (*SRVClients, error) {
}
return &SRVClients{Endpoints: endpoints, SRVs: srvs}, nil
}

// GetSRVService generates a SRV service including an optional suffix.
func GetSRVService(service, serviceName string, scheme string) (SRVService string) {
if scheme == "https" {
service = fmt.Sprintf("%s-ssl", service)
}

if serviceName != "" {
return fmt.Sprintf("%s-%s", service, serviceName)
}
return service
}
39 changes: 38 additions & 1 deletion pkg/srv/srv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ func TestSRVDiscover(t *testing.T) {
return "", nil, errors.New("Unknown service in mock")
}

srvs, err := GetClient("etcd-client", "example.com")
srvs, err := GetClient("etcd-client", "example.com", "")
if err != nil {
t.Fatalf("%d: err: %#v", i, err)
}
Expand All @@ -199,3 +199,40 @@ func TestSRVDiscover(t *testing.T) {

}
}

func TestGetSRVService(t *testing.T) {
tests := []struct {
scheme string
serviceName string

expected string
}{
{
"https",
"",
"etcd-client-ssl",
},
{
"http",
"",
"etcd-client",
},
{
"https",
"foo",
"etcd-client-ssl-foo",
},
{
"http",
"bar",
"etcd-client-bar",
},
}

for i, tt := range tests {
service := GetSRVService("etcd-client", tt.serviceName, tt.scheme)
if strings.Compare(service, tt.expected) != 0 {
t.Errorf("#%d: service = %s, want %s", i, service, tt.expected)
}
}
}
6 changes: 6 additions & 0 deletions tests/docker-dns-srv/certs/Procfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,9 @@ etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127
etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --discovery-srv=etcd.local --initial-cluster-state new --peer-cert-file=/certs/server.crt --peer-key-file=/certs/server.key.insecure --peer-trusted-ca-file=/certs/ca.crt --peer-client-cert-auth --cert-file=/certs/server.crt --key-file=/certs/server.key.insecure --trusted-ca-file=/certs/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr

etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --discovery-srv=etcd.local --initial-cluster-state new --peer-cert-file=/certs/server.crt --peer-key-file=/certs/server.key.insecure --peer-trusted-ca-file=/certs/ca.crt --peer-client-cert-auth --cert-file=/certs/server.crt --key-file=/certs/server.key.insecure --trusted-ca-file=/certs/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr

etcd4: ./etcd --name m4 --data-dir /tmp/m4.data --listen-client-urls https://127.0.0.1:13791 --advertise-client-urls https://m4.etcd.local:13791 --listen-peer-urls https://127.0.0.1:13880 --initial-advertise-peer-urls=https://m1.etcd.local:13880 --initial-cluster-token tkn --discovery-srv=etcd.local --discovery-srv-name=c1 --initial-cluster-state new --peer-cert-file=/certs/server.crt --peer-key-file=/certs/server.key.insecure --peer-trusted-ca-file=/certs/ca.crt --peer-client-cert-auth --cert-file=/certs/server.crt --key-file=/certs/server.key.insecure --trusted-ca-file=/certs/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr

etcd5: ./etcd --name m5 --data-dir /tmp/m5.data --listen-client-urls https://127.0.0.1:23791 --advertise-client-urls https://m5.etcd.local:23791 --listen-peer-urls https://127.0.0.1:23880 --initial-advertise-peer-urls=https://m5.etcd.local:23880 --initial-cluster-token tkn --discovery-srv=etcd.local --discovery-srv-name=c1 --initial-cluster-state new --peer-cert-file=/certs/server.crt --peer-key-file=/certs/server.key.insecure --peer-trusted-ca-file=/certs/ca.crt --peer-client-cert-auth --cert-file=/certs/server.crt --key-file=/certs/server.key.insecure --trusted-ca-file=/certs/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr

etcd6: ./etcd --name m6 --data-dir /tmp/m6.data --listen-client-urls https://127.0.0.1:33791 --advertise-client-urls https://m6.etcd.local:33791 --listen-peer-urls https://127.0.0.1:33880 --initial-advertise-peer-urls=https://m6.etcd.local:33880 --initial-cluster-token tkn --discovery-srv=etcd.local --discovery-srv-name=c1 --initial-cluster-state new --peer-cert-file=/certs/server.crt --peer-key-file=/certs/server.key.insecure --peer-trusted-ca-file=/certs/ca.crt --peer-client-cert-auth --cert-file=/certs/server.crt --key-file=/certs/server.key.insecure --trusted-ca-file=/certs/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr
30 changes: 15 additions & 15 deletions tests/docker-dns-srv/certs/ca.crt
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
-----BEGIN CERTIFICATE-----
MIIDsTCCApmgAwIBAgIUfPEaJnrBzeHM8echLjsPOsV1IzUwDQYJKoZIhvcNAQEL
MIIDrjCCApagAwIBAgIUb8ICEcp5me1o5zF4mh4GKnf57hUwDQYJKoZIhvcNAQEL
BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzExMjIxNzMzMDBaFw0yNzExMjAxNzMz
Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODExMDkxNzQ2MDBaFw0yODExMDYxNzQ2
MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDDU14WMuV1AC+6wDWRF6itx71EljW7Prw2drhuxOC3bE+QQx4LGcY2OP9N
9MC9u9M0s8waGDAbZvdLmCMfAAJoJ05rLcO7F2XEr7Ww7jUWl7+B/sW8ENQiqtUY
1JqLVjwducxmfHspAmSkhEpDBTiTFsya/i1Ic+ctfxDLtsNGgQuA9mCiBvuUhbWG
CkB0JpuL4s6LMuDukQHpZZCDnq0Y26M9sZnjmowbdRoQlhVId6Tl5b5b4Y3qLLbe
r1E+VChcPpOYrKhXBOW/dT5ph/fIQDuVKN6E5Z54AMm3fKsP3MLGBCMfFqIVg1+s
BZA5/Jau+US8Ll4bn8sy/HK1xoy/AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwIBBjAS
BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBSZZ+PEsPywCRKo/fxY2eSnI0wQ
IDANBgkqhkiG9w0BAQsFAAOCAQEAFU4QXMGx8zr8rKAp/IyGipDp/aQ49qYXPjIt
c92rzbYo11sJmBEXiYIOGuZdBBeawIzYsM8dW59LFO8ZcMq/gISBcS5ilqllw6SG
20UrFEKNzcPoRwXp3GSbSGr5PxTgWYWpwJaDa0j2qiM4PB9/IuTBqr6Vu1Olhx06
mXztYl4UL0HPkuB4Td+BIhjc+ZpxCfBOOBpiwAyeh4SpJ3cpZrbyz7JAsCTtywzy
lVO4lfcmxTWwruRyYAnexHdBvnqa8GZw1gufZoSbMTsN4Zz/j3j9T2LG1Q0Agi7o
MhqPqhG/9ISjA0G3bu2B/jHbmWMVbb+ueEYtAz5JHFik2snRtA==
AoIBAQDEBKTfgg0MFy62Sslp8nJPLknl+qTO8ohan80CealThTMuRoGMYpXha0sx
d+mv13sm+vRwEMaRU0FTmxtE9nrM/DNfRoeDd+ZW+Q/hNRuQ0mf0xvmY/h25M+It
uaDbAD3m+UhmOCC1nzdwyBOxm4DQONMwMGtfCOZ8OkIVsKkubx3/pgRB/LdJZRdL
1KWGucjMFxEaTGdwAIxdRyPS9pIX9g+B3zC7T3sYk7YbCGyvi1KLVR45Lm1MPcFY
Gy3hU+CVHiljT6+87N+c98lv8wjnTFJXDkouLm6CxyxGgfGop8fHzpMpGcNmcN5t
Yb3exRWn9u9BfNVH1YEOfiRVB+ylAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQe5E9CeqoDpGgJ1u++mp72Ajvt6DAN
BgkqhkiG9w0BAQsFAAOCAQEAUCj9oKV43RyjvcqKSs00mFKctHZih4Kf0HWGC47M
ny8c/FzCcC66q9TZx1vuf2PHkLsY8Z8f7Rjig2G6hbPKwU05JSFzKCwJhnRSxX4f
ELDqQXbidlQ6wOcj2zoLSVC6WIjVmLyXCu0Zrcp+YwHyGb5x7SQcA1wNmJKOba+h
ooXl5Ea4R1bxK+43lB2bsFovJVhS+6iyBih6oMlLycaSu6c5X38i0mcxQu6Ul/Ua
I8nW1cAXnQC53VzQGkhfxnvWsc98XU/NzF778EaLwLECE7R4zkHWKSUktge1x+co
bRXtQ/C7BoEVaTmQnl211O3rA8gnZ0cmmNBO1S0hIiZIBQ==
-----END CERTIFICATE-----
Loading

0 comments on commit 9454c4c

Please sign in to comment.