bug 1872802: revert of Bug 1868117: cleanup: remove legacy etcd bits

cmd/machine-config-operator/bootstrap.go

@@ -51,8 +51,8 @@ var (
 
 func init() {
 	rootCmd.AddCommand(bootstrapCmd)
-	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdCAFile, "etcd-ca", "", "path to etcd CA certificate")
-	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdMetricCAFile, "etcd-metric-ca", "", "path to etcd metric CA certificate")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdCAFile, "etcd-ca", "/etc/ssl/etcd/ca.crt", "path to etcd CA certificate")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdMetricCAFile, "etcd-metric-ca", "/assets/tls/etcd-metric-ca-bundle.crt", "path to etcd metric CA certificate")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.rootCAFile, "root-ca", "/etc/ssl/kubernetes/ca.crt", "path to root CA certificate")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeCAFile, "kube-ca", "", "path to kube-apiserver serving-ca bundle")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.pullSecretFile, "pull-secret", "/assets/manifests/pull.json", "path to secret manifest that contains pull secret.")
@@ -123,7 +123,7 @@ func runBootstrapCmd(cmd *cobra.Command, args []string) {
 		bootstrapOpts.networkConfigFile,
 		bootstrapOpts.cloudConfigFile,
 		bootstrapOpts.cloudProviderCAFile,
-		bootstrapOpts.rootCAFile, bootstrapOpts.kubeCAFile, bootstrapOpts.pullSecretFile,
+		bootstrapOpts.etcdCAFile, bootstrapOpts.etcdMetricCAFile, bootstrapOpts.rootCAFile, bootstrapOpts.kubeCAFile, bootstrapOpts.pullSecretFile,
 		&imgs,
 		bootstrapOpts.destinationDir,
 		bootstrapOpts.releaseImage,

cmd/machine-config-operator/start.go

@@ -6,6 +6,9 @@ import (
 	"os"
 
 	"github.com/golang/glog"
+	operatorclientset "github.com/openshift/client-go/operator/clientset/versioned"
+	operatorinformers "github.com/openshift/client-go/operator/informers/externalversions"
+	operatorv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
 	"github.com/openshift/machine-config-operator/cmd/common"
 	"github.com/openshift/machine-config-operator/internal/clients"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
@@ -52,6 +55,15 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 	}
 	run := func(ctx context.Context) {
 		ctrlctx := ctrlcommon.CreateControllerContext(cb, ctx.Done(), componentNamespace)
+		operatorClient := cb.OperatorClientOrDie("operator-shared-informer")
+
+		etcdInformer, err := getEtcdInformer(operatorClient, ctrlctx.OperatorInformerFactory)
+		if err != nil {
+			// MCO pod needs to restart for transient apiserver errors
+			glog.Errorf("unable to query discovery API %#v", err)
+			ctrlcommon.WriteTerminationError(err)
+		}
+
 		controller := operator.New(
 			componentNamespace, componentName,
 			startOpts.imagesFile,
@@ -74,6 +86,7 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 			ctrlctx.ClientBuilder.APIExtClientOrDie(componentName),
 			ctrlctx.ClientBuilder.ConfigClientOrDie(componentName),
 			ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
+			etcdInformer,
 			ctrlctx.KubeMAOSharedInformer.Core().V1().Secrets(),
 		)
 
@@ -106,3 +119,18 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 	})
 	panic("unreachable")
 }
+
+func getEtcdInformer(operatorClient operatorclientset.Interface, operatorSharedInformer operatorinformers.SharedInformerFactory) (operatorv1.EtcdInformer, error) {
+	operatorGroups, err := operatorClient.Discovery().ServerResourcesForGroupVersion("operator.openshift.io/v1")
+	if err != nil {
+		glog.Errorf("unable to get operatorGroups: %#v", err)
+		return nil, err
+	}
+
+	for _, o := range operatorGroups.APIResources {
+		if o.Kind == "Etcd" {
+			return operatorSharedInformer.Operator().V1().Etcds(), nil
+		}
+	}
+	return nil, nil
+}

install/0000_80_machine-config-operator_04_deployment.yaml

@@ -34,6 +34,8 @@ spec:
         volumeMounts:
         - name: root-ca
           mountPath: /etc/ssl/kubernetes/ca.crt
+        - name: etcd-ca
+          mountPath: /etc/ssl/etcd/ca.crt
         - name: images
           mountPath: /etc/mco/images
       nodeSelector:
@@ -59,6 +61,9 @@ spec:
       - name: images
         configMap:
           name: machine-config-operator-images
+      - name: etcd-ca
+        hostPath:
+          path: /etc/ssl/etcd/ca.crt
       - name: root-ca
         hostPath:
           path: /etc/kubernetes/ca.crt

install/image-references

@@ -3,11 +3,15 @@ apiVersion: image.openshift.io/v1
 spec:
   tags:
   # machine-config-operator is the new master mco image that contains all of the 
-  # component images:mco, mcc, mcs & mcd 
+  # component images:mco, mcc, mcs, mcd & setup etcd
   - name: machine-config-operator
     from:
       kind: DockerImage
       name: registry.svc.ci.openshift.org/openshift:machine-config-operator
+  - name: etcd
+    from:
+      kind: DockerImage
+      name: registry.svc.ci.openshift.org/openshift:etcd
   - name: pod
     from:
       kind: DockerImage
@@ -27,6 +31,10 @@ spec:
     from:
       kind: DockerImage
       name: registry.svc.ci.openshift.org/openshift:kube-client-agent
+  - name: cluster-etcd-operator
+    from:
+      kind: DockerImage
+      name: registry.svc.ci.openshift.org/openshift:cluster-etcd-operator
   - name: keepalived-ipfailover
     from:
       kind: DockerImage

lib/resourcemerge/machineconfig.go

@@ -74,6 +74,8 @@ func ensureControllerConfigSpec(modified *bool, existing *mcfgv1.ControllerConfi
 	setStringIfSet(modified, &existing.NetworkType, required.NetworkType)
 
 	setBytesIfSet(modified, &existing.AdditionalTrustBundle, required.AdditionalTrustBundle)
+	setBytesIfSet(modified, &existing.EtcdCAData, required.EtcdCAData)
+	setBytesIfSet(modified, &existing.EtcdMetricCAData, required.EtcdMetricCAData)
 	setBytesIfSet(modified, &existing.RootCAData, required.RootCAData)
 	setBytesIfSet(modified, &existing.KubeAPIServerServingCAData, required.KubeAPIServerServingCAData)
 	setBytesIfSet(modified, &existing.CloudProviderCAData, required.CloudProviderCAData)

manifests/controllerconfig.crd.yaml

@@ -68,6 +68,17 @@ spec:
             clusterDNSIP:
               description: clusterDNSIP is the cluster DNS IP address
               type: string
+            etcdCAData:
+              description: etcdCAData specifies the etcd CA data
+              type: string
+              format: byte
+            etcdDiscoveryDomain:
+              description: etcdDiscoveryDomain is deprecated, use infra.status.etcdDiscoveryDomain instead
+              type: string
+            etcdMetricCAData:
+              description: etcdMetricData specifies the etcd metric CA data
+              type: string
+              format: byte
             images:
               description: images is map of images that are used by the controller
                 to render templates under ./templates/

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -48,6 +48,12 @@ type ControllerConfigSpec struct {
 	// kubeAPIServerServingCAData managed Kubelet to API Server Cert... Rotated automatically
 	KubeAPIServerServingCAData []byte `json:"kubeAPIServerServingCAData"`
 
+	// etcdCAData specifies the etcd CA data
+	EtcdCAData []byte `json:"etcdCAData"`
+
+	// etcdMetricData specifies the etcd metric CA data
+	EtcdMetricCAData []byte `json:"etcdMetricCAData"`
+
 	// rootCAData specifies the root CA data
 	RootCAData []byte `json:"rootCAData"`
 

pkg/controller/bootstrap/testdata/bootstrap/machineconfigcontroller-controllerconfig.yaml

@@ -6,14 +6,18 @@ spec:
   additionalTrustBundle: null
   cloudProviderConfig: ""
   clusterDNSIP: 172.30.0.10
+  etcdCAData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCkVUQ0QgQ0EgREFUQQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+  etcdMetricCAData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCkVUQ0QgTUVUUklDIENBIERBVEEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
   images:
     baremetalRuntimeCfgImage: ""
     corednsImage: ""
+    etcdKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     haproxyImage: ""
     infraImageKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
     keepalivedImage: ""
     kubeClientAgentImageKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
     mdnsPublisherImage: ""
+    setupEtcdEnvKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd
   infra:
     apiVersion: config.openshift.io/v1
     kind: Infrastructure
@@ -26,6 +30,7 @@ spec:
     status:
       apiServerInternalURI: https://api-int.domain.example.com:6443
       apiServerURL: https://api.domain.example.com:6443
+      etcdDiscoveryDomain: domain.example.com
       infrastructureName: lab-0aaaa
       platformStatus:
         type: None

pkg/controller/template/constants.go

@@ -1,12 +1,24 @@
 package template
 
 const (
+	// EtcdImageKey is the key that references the etcd image in the controller
+	EtcdImageKey string = "etcdKey"
+
+	// SetupEtcdEnvKey is the key that references the setup-etcd-environment image in the controller
+	SetupEtcdEnvKey string = "setupEtcdEnvKey"
+
 	// GCPRoutesControllerKey is the key that references the gcp-routes-controller image in the controller
 	GCPRoutesControllerKey string = "gcpRoutesControllerKey"
 
 	// InfraImageKey is the key that references the infra image in the controller for crio.conf
 	InfraImageKey string = "infraImageKey"
 
+	// KubeClientAgentImageKey is the key that references the kube-client-agent image in the controller
+	KubeClientAgentImageKey string = "kubeClientAgentImageKey"
+
+	// ClusterEtcdOperatorImageKey is the key that references the cluster-etcd-operator image in the controller
+	ClusterEtcdOperatorImageKey string = "clusterEtcdOperatorImageKey"
+
 	// KeepalivedKey is the key that references the keepalived-ipfailover image in the controller
 	KeepalivedKey string = "keepalivedImage"
 

pkg/controller/template/render.go

@@ -281,6 +281,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir,
 func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) {
 	funcs := sprig.TxtFuncMap()
 	funcs["skip"] = skipMissing
+	funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames
+	funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames
+	funcs["etcdServerCertCommand"] = etcdServerCertCommand
+	funcs["etcdPeerCertCommand"] = etcdPeerCertCommand
+	funcs["etcdMetricCertCommand"] = etcdMetricCertCommand
 	funcs["cloudProvider"] = cloudProvider
 	funcs["cloudConfigFlag"] = cloudConfigFlag
 	tmpl, err := template.New(path).Funcs(funcs).Parse(string(b))
@@ -307,6 +312,115 @@ func skipMissing(key string) (interface{}, error) {
 	return fmt.Sprintf("{{.%s}}", key), nil
 }
 
+// Process the {{etcdPeerCertDNSNames}} and {{etcdServerCertDNSNames}}
+func etcdServerCertDNSNames(cfg RenderConfig) (interface{}, error) {
+	var dnsNames = []string{
+		"localhost",
+		"etcd.kube-system.svc",                  // sign for the local etcd service name that cluster-network apiservers use to communicate
+		"etcd.kube-system.svc.cluster.local",    // sign for the local etcd service name that cluster-network apiservers use to communicate
+		"etcd.openshift-etcd.svc",               // sign for the local etcd service name that cluster-network apiservers use to communicate
+		"etcd.openshift-etcd.svc.cluster.local", // sign for the local etcd service name that cluster-network apiservers use to communicate
+		"${ETCD_WILDCARD_DNS_NAME}",
+	}
+	return strings.Join(dnsNames, ","), nil
+}
+
+func etcdPeerCertDNSNames(cfg RenderConfig) (interface{}, error) {
+	if cfg.Infra.Status.EtcdDiscoveryDomain == "" {
+		return nil, fmt.Errorf("invalid configuration")
+	}
+
+	var dnsNames = []string{
+		"${ETCD_DNS_NAME}",
+		cfg.Infra.Status.EtcdDiscoveryDomain, // https://github.com/etcd-io/etcd/blob/583763261f1c843e07c1bf7fea5fb4cfb684fe87/Documentation/op-guide/clustering.md#dns-discovery
+	}
+	return strings.Join(dnsNames, ","), nil
+}
+
+func etcdServerCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		serverCertDNS, err := etcdServerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-servers \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", serverCertDNS),
+			"  --commonname=system:etcd-server:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS},${ETCD_LOCALHOST_IP} \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-server:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
+func etcdPeerCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		peerCertDNS, err := etcdPeerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-peers \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", peerCertDNS),
+			"  --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS} \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
+func etcdMetricCertCommand(cfg RenderConfig) (interface{}, error) {
+	commands := []string{}
+	if cfg.Images[ClusterEtcdOperatorImageKey] == "" {
+		metricCertDNS, err := etcdServerCertDNSNames(cfg)
+		if err != nil {
+			return nil, err
+		}
+		commands = append(commands, []string{
+			"kube-client-agent \\",
+			"  request \\",
+			"  --kubeconfig=/etc/kubernetes/kubeconfig \\",
+			"  --orgname=system:etcd-metrics \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			fmt.Sprintf("  --dnsnames=%s \\", metricCertDNS),
+			"  --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\",
+			"  --ipaddrs=${ETCD_IPV4_ADDRESS} \\",
+		}...)
+	} else {
+		commands = append(commands, []string{
+			"cluster-etcd-operator \\",
+			"  mount \\",
+			"  --assetsdir=/etc/ssl/etcd \\",
+			"  --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\",
+		}...)
+	}
+	return commands, nil
+}
+
 func cloudProvider(cfg RenderConfig) (interface{}, error) {
 	if cfg.Infra.Status.PlatformStatus != nil {
 		switch cfg.Infra.Status.PlatformStatus.Type {