diff --git a/cmd/machine-config-operator/bootstrap.go b/cmd/machine-config-operator/bootstrap.go index 1e99ca6871..8cdd4221e4 100644 --- a/cmd/machine-config-operator/bootstrap.go +++ b/cmd/machine-config-operator/bootstrap.go @@ -51,8 +51,8 @@ var ( func init() { rootCmd.AddCommand(bootstrapCmd) - bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdCAFile, "etcd-ca", "", "path to etcd CA certificate") - bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdMetricCAFile, "etcd-metric-ca", "", "path to etcd metric CA certificate") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdCAFile, "etcd-ca", "/etc/ssl/etcd/ca.crt", "path to etcd CA certificate") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.etcdMetricCAFile, "etcd-metric-ca", "/assets/tls/etcd-metric-ca-bundle.crt", "path to etcd metric CA certificate") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.rootCAFile, "root-ca", "/etc/ssl/kubernetes/ca.crt", "path to root CA certificate") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.kubeCAFile, "kube-ca", "", "path to kube-apiserver serving-ca bundle") bootstrapCmd.PersistentFlags().StringVar(&bootstrapOpts.pullSecretFile, "pull-secret", "/assets/manifests/pull.json", "path to secret manifest that contains pull secret.") @@ -123,7 +123,7 @@ func runBootstrapCmd(cmd *cobra.Command, args []string) { bootstrapOpts.networkConfigFile, bootstrapOpts.cloudConfigFile, bootstrapOpts.cloudProviderCAFile, - bootstrapOpts.rootCAFile, bootstrapOpts.kubeCAFile, bootstrapOpts.pullSecretFile, + bootstrapOpts.etcdCAFile, bootstrapOpts.etcdMetricCAFile, bootstrapOpts.rootCAFile, bootstrapOpts.kubeCAFile, bootstrapOpts.pullSecretFile, &imgs, bootstrapOpts.destinationDir, bootstrapOpts.releaseImage, diff --git a/cmd/machine-config-operator/start.go b/cmd/machine-config-operator/start.go index 86b81d0ff7..06d428d8d6 100644 --- a/cmd/machine-config-operator/start.go +++ b/cmd/machine-config-operator/start.go @@ -6,6 +6,9 @@ import ( "os" "github.com/golang/glog" + operatorclientset "github.com/openshift/client-go/operator/clientset/versioned" + operatorinformers "github.com/openshift/client-go/operator/informers/externalversions" + operatorv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" "github.com/openshift/machine-config-operator/cmd/common" "github.com/openshift/machine-config-operator/internal/clients" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" @@ -52,6 +55,15 @@ func runStartCmd(cmd *cobra.Command, args []string) { } run := func(ctx context.Context) { ctrlctx := ctrlcommon.CreateControllerContext(cb, ctx.Done(), componentNamespace) + operatorClient := cb.OperatorClientOrDie("operator-shared-informer") + + etcdInformer, err := getEtcdInformer(operatorClient, ctrlctx.OperatorInformerFactory) + if err != nil { + // MCO pod needs to restart for transient apiserver errors + glog.Errorf("unable to query discovery API %#v", err) + ctrlcommon.WriteTerminationError(err) + } + controller := operator.New( componentNamespace, componentName, startOpts.imagesFile, @@ -74,6 +86,7 @@ func runStartCmd(cmd *cobra.Command, args []string) { ctrlctx.ClientBuilder.APIExtClientOrDie(componentName), ctrlctx.ClientBuilder.ConfigClientOrDie(componentName), ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Core().V1().ConfigMaps(), + etcdInformer, ctrlctx.KubeMAOSharedInformer.Core().V1().Secrets(), ) @@ -106,3 +119,18 @@ func runStartCmd(cmd *cobra.Command, args []string) { }) panic("unreachable") } + +func getEtcdInformer(operatorClient operatorclientset.Interface, operatorSharedInformer operatorinformers.SharedInformerFactory) (operatorv1.EtcdInformer, error) { + operatorGroups, err := operatorClient.Discovery().ServerResourcesForGroupVersion("operator.openshift.io/v1") + if err != nil { + glog.Errorf("unable to get operatorGroups: %#v", err) + return nil, err + } + + for _, o := range operatorGroups.APIResources { + if o.Kind == "Etcd" { + return operatorSharedInformer.Operator().V1().Etcds(), nil + } + } + return nil, nil +} diff --git a/install/0000_80_machine-config-operator_04_deployment.yaml b/install/0000_80_machine-config-operator_04_deployment.yaml index 6332d0477b..06fec43e40 100644 --- a/install/0000_80_machine-config-operator_04_deployment.yaml +++ b/install/0000_80_machine-config-operator_04_deployment.yaml @@ -34,6 +34,8 @@ spec: volumeMounts: - name: root-ca mountPath: /etc/ssl/kubernetes/ca.crt + - name: etcd-ca + mountPath: /etc/ssl/etcd/ca.crt - name: images mountPath: /etc/mco/images nodeSelector: @@ -59,6 +61,9 @@ spec: - name: images configMap: name: machine-config-operator-images + - name: etcd-ca + hostPath: + path: /etc/ssl/etcd/ca.crt - name: root-ca hostPath: path: /etc/kubernetes/ca.crt diff --git a/install/image-references b/install/image-references index f207ff8245..ba9018288c 100644 --- a/install/image-references +++ b/install/image-references @@ -3,11 +3,15 @@ apiVersion: image.openshift.io/v1 spec: tags: # machine-config-operator is the new master mco image that contains all of the - # component images:mco, mcc, mcs & mcd + # component images:mco, mcc, mcs, mcd & setup etcd - name: machine-config-operator from: kind: DockerImage name: registry.svc.ci.openshift.org/openshift:machine-config-operator + - name: etcd + from: + kind: DockerImage + name: registry.svc.ci.openshift.org/openshift:etcd - name: pod from: kind: DockerImage @@ -27,6 +31,10 @@ spec: from: kind: DockerImage name: registry.svc.ci.openshift.org/openshift:kube-client-agent + - name: cluster-etcd-operator + from: + kind: DockerImage + name: registry.svc.ci.openshift.org/openshift:cluster-etcd-operator - name: keepalived-ipfailover from: kind: DockerImage diff --git a/lib/resourcemerge/machineconfig.go b/lib/resourcemerge/machineconfig.go index 955d11e322..2e3cd1e6d0 100644 --- a/lib/resourcemerge/machineconfig.go +++ b/lib/resourcemerge/machineconfig.go @@ -74,6 +74,8 @@ func ensureControllerConfigSpec(modified *bool, existing *mcfgv1.ControllerConfi setStringIfSet(modified, &existing.NetworkType, required.NetworkType) setBytesIfSet(modified, &existing.AdditionalTrustBundle, required.AdditionalTrustBundle) + setBytesIfSet(modified, &existing.EtcdCAData, required.EtcdCAData) + setBytesIfSet(modified, &existing.EtcdMetricCAData, required.EtcdMetricCAData) setBytesIfSet(modified, &existing.RootCAData, required.RootCAData) setBytesIfSet(modified, &existing.KubeAPIServerServingCAData, required.KubeAPIServerServingCAData) setBytesIfSet(modified, &existing.CloudProviderCAData, required.CloudProviderCAData) diff --git a/manifests/controllerconfig.crd.yaml b/manifests/controllerconfig.crd.yaml index 8c7d720660..dc347f7391 100644 --- a/manifests/controllerconfig.crd.yaml +++ b/manifests/controllerconfig.crd.yaml @@ -68,6 +68,17 @@ spec: clusterDNSIP: description: clusterDNSIP is the cluster DNS IP address type: string + etcdCAData: + description: etcdCAData specifies the etcd CA data + type: string + format: byte + etcdDiscoveryDomain: + description: etcdDiscoveryDomain is deprecated, use infra.status.etcdDiscoveryDomain instead + type: string + etcdMetricCAData: + description: etcdMetricData specifies the etcd metric CA data + type: string + format: byte images: description: images is map of images that are used by the controller to render templates under ./templates/ diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/types.go b/pkg/apis/machineconfiguration.openshift.io/v1/types.go index 9c5be0f5ad..7957743e3f 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/types.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/types.go @@ -48,6 +48,12 @@ type ControllerConfigSpec struct { // kubeAPIServerServingCAData managed Kubelet to API Server Cert... Rotated automatically KubeAPIServerServingCAData []byte `json:"kubeAPIServerServingCAData"` + // etcdCAData specifies the etcd CA data + EtcdCAData []byte `json:"etcdCAData"` + + // etcdMetricData specifies the etcd metric CA data + EtcdMetricCAData []byte `json:"etcdMetricCAData"` + // rootCAData specifies the root CA data RootCAData []byte `json:"rootCAData"` diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go b/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go index 19715ebeae..45ed5a88de 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go @@ -226,6 +226,16 @@ func (in *ControllerConfigSpec) DeepCopyInto(out *ControllerConfigSpec) { *out = make([]byte, len(*in)) copy(*out, *in) } + if in.EtcdCAData != nil { + in, out := &in.EtcdCAData, &out.EtcdCAData + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.EtcdMetricCAData != nil { + in, out := &in.EtcdMetricCAData, &out.EtcdMetricCAData + *out = make([]byte, len(*in)) + copy(*out, *in) + } if in.RootCAData != nil { in, out := &in.RootCAData, &out.RootCAData *out = make([]byte, len(*in)) diff --git a/pkg/controller/bootstrap/testdata/bootstrap/machineconfigcontroller-controllerconfig.yaml b/pkg/controller/bootstrap/testdata/bootstrap/machineconfigcontroller-controllerconfig.yaml index 110747b67e..26d92832e7 100644 --- a/pkg/controller/bootstrap/testdata/bootstrap/machineconfigcontroller-controllerconfig.yaml +++ b/pkg/controller/bootstrap/testdata/bootstrap/machineconfigcontroller-controllerconfig.yaml @@ -6,14 +6,18 @@ spec: additionalTrustBundle: null cloudProviderConfig: "" clusterDNSIP: 172.30.0.10 + etcdCAData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCkVUQ0QgQ0EgREFUQQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + etcdMetricCAData: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCkVUQ0QgTUVUUklDIENBIERBVEEKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= images: baremetalRuntimeCfgImage: "" corednsImage: "" + etcdKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa haproxyImage: "" infraImageKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb keepalivedImage: "" kubeClientAgentImageKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc mdnsPublisherImage: "" + setupEtcdEnvKey: registry.product.example.org/ocp/4.2-DATE-VERSION@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd infra: apiVersion: config.openshift.io/v1 kind: Infrastructure @@ -26,6 +30,7 @@ spec: status: apiServerInternalURI: https://api-int.domain.example.com:6443 apiServerURL: https://api.domain.example.com:6443 + etcdDiscoveryDomain: domain.example.com infrastructureName: lab-0aaaa platformStatus: type: None diff --git a/pkg/controller/template/constants.go b/pkg/controller/template/constants.go index 98a08a95df..7d98bb9d9b 100644 --- a/pkg/controller/template/constants.go +++ b/pkg/controller/template/constants.go @@ -1,12 +1,24 @@ package template const ( + // EtcdImageKey is the key that references the etcd image in the controller + EtcdImageKey string = "etcdKey" + + // SetupEtcdEnvKey is the key that references the setup-etcd-environment image in the controller + SetupEtcdEnvKey string = "setupEtcdEnvKey" + // GCPRoutesControllerKey is the key that references the gcp-routes-controller image in the controller GCPRoutesControllerKey string = "gcpRoutesControllerKey" // InfraImageKey is the key that references the infra image in the controller for crio.conf InfraImageKey string = "infraImageKey" + // KubeClientAgentImageKey is the key that references the kube-client-agent image in the controller + KubeClientAgentImageKey string = "kubeClientAgentImageKey" + + // ClusterEtcdOperatorImageKey is the key that references the cluster-etcd-operator image in the controller + ClusterEtcdOperatorImageKey string = "clusterEtcdOperatorImageKey" + // KeepalivedKey is the key that references the keepalived-ipfailover image in the controller KeepalivedKey string = "keepalivedImage" diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 231a519373..bcba1ee5b5 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -281,6 +281,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, templateDir, func renderTemplate(config RenderConfig, path string, b []byte) ([]byte, error) { funcs := sprig.TxtFuncMap() funcs["skip"] = skipMissing + funcs["etcdServerCertDNSNames"] = etcdServerCertDNSNames + funcs["etcdPeerCertDNSNames"] = etcdPeerCertDNSNames + funcs["etcdServerCertCommand"] = etcdServerCertCommand + funcs["etcdPeerCertCommand"] = etcdPeerCertCommand + funcs["etcdMetricCertCommand"] = etcdMetricCertCommand funcs["cloudProvider"] = cloudProvider funcs["cloudConfigFlag"] = cloudConfigFlag tmpl, err := template.New(path).Funcs(funcs).Parse(string(b)) @@ -307,6 +312,115 @@ func skipMissing(key string) (interface{}, error) { return fmt.Sprintf("{{.%s}}", key), nil } +// Process the {{etcdPeerCertDNSNames}} and {{etcdServerCertDNSNames}} +func etcdServerCertDNSNames(cfg RenderConfig) (interface{}, error) { + var dnsNames = []string{ + "localhost", + "etcd.kube-system.svc", // sign for the local etcd service name that cluster-network apiservers use to communicate + "etcd.kube-system.svc.cluster.local", // sign for the local etcd service name that cluster-network apiservers use to communicate + "etcd.openshift-etcd.svc", // sign for the local etcd service name that cluster-network apiservers use to communicate + "etcd.openshift-etcd.svc.cluster.local", // sign for the local etcd service name that cluster-network apiservers use to communicate + "${ETCD_WILDCARD_DNS_NAME}", + } + return strings.Join(dnsNames, ","), nil +} + +func etcdPeerCertDNSNames(cfg RenderConfig) (interface{}, error) { + if cfg.Infra.Status.EtcdDiscoveryDomain == "" { + return nil, fmt.Errorf("invalid configuration") + } + + var dnsNames = []string{ + "${ETCD_DNS_NAME}", + cfg.Infra.Status.EtcdDiscoveryDomain, // https://github.com/etcd-io/etcd/blob/583763261f1c843e07c1bf7fea5fb4cfb684fe87/Documentation/op-guide/clustering.md#dns-discovery + } + return strings.Join(dnsNames, ","), nil +} + +func etcdServerCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + serverCertDNS, err := etcdServerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-servers \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", serverCertDNS), + " --commonname=system:etcd-server:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS},${ETCD_LOCALHOST_IP} \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-server:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + +func etcdPeerCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + peerCertDNS, err := etcdPeerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-peers \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", peerCertDNS), + " --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS} \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-peer:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + +func etcdMetricCertCommand(cfg RenderConfig) (interface{}, error) { + commands := []string{} + if cfg.Images[ClusterEtcdOperatorImageKey] == "" { + metricCertDNS, err := etcdServerCertDNSNames(cfg) + if err != nil { + return nil, err + } + commands = append(commands, []string{ + "kube-client-agent \\", + " request \\", + " --kubeconfig=/etc/kubernetes/kubeconfig \\", + " --orgname=system:etcd-metrics \\", + " --assetsdir=/etc/ssl/etcd \\", + fmt.Sprintf(" --dnsnames=%s \\", metricCertDNS), + " --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\", + " --ipaddrs=${ETCD_IPV4_ADDRESS} \\", + }...) + } else { + commands = append(commands, []string{ + "cluster-etcd-operator \\", + " mount \\", + " --assetsdir=/etc/ssl/etcd \\", + " --commonname=system:etcd-metric:${ETCD_DNS_NAME} \\", + }...) + } + return commands, nil +} + func cloudProvider(cfg RenderConfig) (interface{}, error) { if cfg.Infra.Status.PlatformStatus != nil { switch cfg.Infra.Status.PlatformStatus.Type { diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index 192382f3c9..c7b9d001c6 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -144,6 +144,75 @@ func TestCloudConfigFlag(t *testing.T) { } } +func TestEtcdPeerCertDNSNames(t *testing.T) { + dummyTemplate := []byte(`{{etcdPeerCertDNSNames .}}`) + + cases := []struct { + etcdDiscoveryDomain string + + url string + err bool + }{{ + etcdDiscoveryDomain: "", + url: "", + err: true, + }, { + etcdDiscoveryDomain: "my-test-cluster.tt.testing", + url: "${ETCD_DNS_NAME},my-test-cluster.tt.testing", + err: false, + }} + for idx, c := range cases { + name := fmt.Sprintf("case #%d", idx) + t.Run(name, func(t *testing.T) { + config := &mcfgv1.ControllerConfig{ + Spec: mcfgv1.ControllerConfigSpec{ + Infra: &configv1.Infrastructure{ + Status: configv1.InfrastructureStatus{ + EtcdDiscoveryDomain: c.etcdDiscoveryDomain, + }, + }, + }, + } + got, err := renderTemplate(RenderConfig{&config.Spec, `{"dummy":"dummy"}`}, name, dummyTemplate) + if err != nil && !c.err { + t.Fatalf("expected nil error %v", err) + } + + if string(got) != c.url { + t.Fatalf("mismatch got: %s want: %s", got, c.url) + } + }) + } +} + +func TestEtcdServerCertDNSNames(t *testing.T) { + dummyTemplate := []byte(`{{etcdServerCertDNSNames .}}`) + + cases := []struct { + url string + err bool + }{{ + url: "localhost,etcd.kube-system.svc,etcd.kube-system.svc.cluster.local,etcd.openshift-etcd.svc,etcd.openshift-etcd.svc.cluster.local,${ETCD_WILDCARD_DNS_NAME}", + err: false, + }} + for idx, c := range cases { + name := fmt.Sprintf("case #%d", idx) + t.Run(name, func(t *testing.T) { + config := &mcfgv1.ControllerConfig{ + Spec: mcfgv1.ControllerConfigSpec{}, + } + got, err := renderTemplate(RenderConfig{&config.Spec, `{"dummy":"dummy"}`}, name, dummyTemplate) + if err != nil && !c.err { + t.Fatalf("expected nil error %v", err) + } + + if string(got) != c.url { + t.Fatalf("mismatch got: %s want: %s", got, c.url) + } + }) + } +} + func TestSkipMissing(t *testing.T) { dummyTemplate := `{{skip "%s"}}` @@ -167,8 +236,11 @@ func TestSkipMissing(t *testing.T) { key: "index", err: false, res: "{{.index}}", - }, - } + }, { + key: "etcd_index", + err: false, + res: "{{.etcd_index}}", + }} for idx, c := range cases { name := fmt.Sprintf("case #%d", idx) diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 8ad7def18a..cfbefdc802 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -615,6 +615,17 @@ spec: clusterDNSIP: description: clusterDNSIP is the cluster DNS IP address type: string + etcdCAData: + description: etcdCAData specifies the etcd CA data + type: string + format: byte + etcdDiscoveryDomain: + description: etcdDiscoveryDomain is deprecated, use infra.status.etcdDiscoveryDomain instead + type: string + etcdMetricCAData: + description: etcdMetricData specifies the etcd metric CA data + type: string + format: byte images: description: images is map of images that are used by the controller to render templates under ./templates/ diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 247191df7c..cd2aad5249 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -30,7 +30,7 @@ func RenderBootstrap( clusterConfigConfigMapFile, infraFile, networkFile, cloudConfigFile, cloudProviderCAFile, - rootCAFile, kubeAPIServerServingCA, pullSecretFile string, + etcdCAFile, etcdMetricCAFile, rootCAFile, kubeAPIServerServingCA, pullSecretFile string, imgs *Images, destinationDir, releaseImage string, ) error { @@ -41,6 +41,8 @@ func RenderBootstrap( infraFile, networkFile, rootCAFile, + etcdCAFile, + etcdMetricCAFile, pullSecretFile, } if kubeAPIServerServingCA != "" { @@ -127,18 +129,24 @@ func RenderBootstrap( spec.CloudProviderCAData = data } + spec.EtcdCAData = filesData[etcdCAFile] + spec.EtcdMetricCAData = filesData[etcdMetricCAFile] spec.RootCAData = bundle spec.PullSecret = nil spec.OSImageURL = imgs.MachineOSContent spec.ReleaseImage = releaseImage spec.Images = map[string]string{ - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, - templatectrl.InfraImageKey: imgs.InfraImage, - templatectrl.KeepalivedKey: imgs.Keepalived, - templatectrl.CorednsKey: imgs.Coredns, - templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, - templatectrl.HaproxyKey: imgs.Haproxy, - templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, + templatectrl.EtcdImageKey: imgs.Etcd, + templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, + templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.InfraImageKey: imgs.InfraImage, + templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, + templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator, + templatectrl.KeepalivedKey: imgs.Keepalived, + templatectrl.CorednsKey: imgs.Coredns, + templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, + templatectrl.HaproxyKey: imgs.Haproxy, + templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, } config := getRenderConfig("", string(filesData[kubeAPIServerServingCA]), spec, &imgs.RenderConfigImages, infra.Status.APIServerInternalURL, nil) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index ef2dd848b6..a8d9065182 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -9,6 +9,8 @@ import ( "github.com/golang/glog" configclientset "github.com/openshift/client-go/config/clientset/versioned" + operatorv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + operatorlisterv1 "github.com/openshift/client-go/operator/listers/operator/v1" corev1 "k8s.io/api/core/v1" apiextclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextinformersv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1beta1" @@ -81,6 +83,7 @@ type Operator struct { clusterCmLister corelisterv1.ConfigMapLister proxyLister configlistersv1.ProxyLister oseKubeAPILister corelisterv1.ConfigMapLister + etcdLister operatorlisterv1.EtcdLister crdListerSynced cache.InformerSynced deployListerSynced cache.InformerSynced @@ -97,6 +100,7 @@ type Operator struct { clusterRoleBindingInformerSynced cache.InformerSynced proxyListerSynced cache.InformerSynced oseKubeAPIListerSynced cache.InformerSynced + etcdSynced cache.InformerSynced maoSecretInformerSynced cache.InformerSynced // queue only ever has one item, but it has nice error handling backoff/retry semantics @@ -129,6 +133,7 @@ func New( apiExtClient apiextclientset.Interface, configClient configclientset.Interface, oseKubeAPIInformer coreinformersv1.ConfigMapInformer, + etcdInformer operatorv1.EtcdInformer, maoSecretInformer coreinformersv1.SecretInformer, ) *Operator { eventBroadcaster := record.NewBroadcaster() @@ -198,6 +203,17 @@ func New( optr.infraListerSynced = infraInformer.Informer().HasSynced optr.networkLister = networkInformer.Lister() optr.networkListerSynced = networkInformer.Informer().HasSynced + if etcdInformer != nil { + optr.etcdLister = etcdInformer.Lister() + optr.etcdSynced = etcdInformer.Informer().HasSynced + etcdInformer.Informer().AddEventHandler(optr.eventHandler()) + } else { + optr.etcdLister = nil + optr.etcdSynced = func() bool { + // if etcd is not part of CVO it will return true immediately + return true + } + } optr.vStore.Set("operator", os.Getenv("RELEASE_VERSION")) @@ -235,7 +251,8 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { optr.proxyListerSynced, optr.oseKubeAPIListerSynced, optr.mcpListerSynced, - optr.mcListerSynced) { + optr.mcListerSynced, + optr.etcdSynced) { glog.Error("failed to sync caches") return } diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 9b001fff90..9181170391 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/cache" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" libgoevents "github.com/openshift/library-go/pkg/operator/events" libgoresapply "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/machine-config-operator/lib/resourceapply" @@ -188,6 +189,14 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error { } // sync up CAs + etcdCA, err := optr.getCAsFromConfigMap("openshift-config", "etcd-serving-ca", "ca-bundle.crt") + if err != nil { + return err + } + etcdMetricCA, err := optr.getCAsFromConfigMap("openshift-config", "etcd-metric-serving-ca", "ca-bundle.crt") + if err != nil { + return err + } rootCA, err := optr.getCAsFromConfigMap("kube-system", "root-ca", "ca.crt") if err != nil { return err @@ -263,18 +272,29 @@ func (optr *Operator) syncRenderConfig(_ *renderConfig) error { return err } + //TODO: alaypatel07 remove after cluster-etcd-operator deployed via CVO as Managed + if err = optr.setEtcdOperatorImage(&imgs); err != nil { + glog.Errorf("error setting etcd operator images: %#v", err) + } + spec.KubeAPIServerServingCAData = kubeAPIServerServingCABytes + spec.EtcdCAData = etcdCA + spec.EtcdMetricCAData = etcdMetricCA spec.RootCAData = bundle spec.PullSecret = &corev1.ObjectReference{Namespace: "openshift-config", Name: "pull-secret"} spec.OSImageURL = imgs.MachineOSContent spec.Images = map[string]string{ - templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, - templatectrl.InfraImageKey: imgs.InfraImage, - templatectrl.KeepalivedKey: imgs.Keepalived, - templatectrl.CorednsKey: imgs.Coredns, - templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, - templatectrl.HaproxyKey: imgs.Haproxy, - templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, + templatectrl.EtcdImageKey: imgs.Etcd, + templatectrl.SetupEtcdEnvKey: imgs.MachineConfigOperator, + templatectrl.GCPRoutesControllerKey: imgs.MachineConfigOperator, + templatectrl.InfraImageKey: imgs.InfraImage, + templatectrl.KubeClientAgentImageKey: imgs.KubeClientAgent, + templatectrl.ClusterEtcdOperatorImageKey: imgs.ClusterEtcdOperator, + templatectrl.KeepalivedKey: imgs.Keepalived, + templatectrl.CorednsKey: imgs.Coredns, + templatectrl.MdnsPublisherKey: imgs.MdnsPublisher, + templatectrl.HaproxyKey: imgs.Haproxy, + templatectrl.BaremetalRuntimeCfgKey: imgs.BaremetalRuntimeCfg, } ignitionHost, err := getIgnitionHost(&infra.Status) @@ -856,6 +876,32 @@ func (optr *Operator) getGlobalConfig() (*configv1.Infrastructure, *configv1.Net return infra, network, proxy, nil } +func (optr *Operator) setEtcdOperatorImage(imgs *Images) error { + if optr.etcdLister == nil { + // if the resource is not found, i.e. it is not created by CVO + // which means cluster-etcd-operator images is not part of CVO + imgs.ControllerConfigImages.ClusterEtcdOperator = "" + glog.V(4).Info("etcd cr not found") + return nil + } + etcd, err := optr.etcdLister.Get("cluster") + if err != nil { + if apierrors.IsNotFound(err) { + imgs.ControllerConfigImages.ClusterEtcdOperator = "" + return nil + } + imgs.ControllerConfigImages.ClusterEtcdOperator = "" + return fmt.Errorf("error getting etcd CR: %#v", err) + } + + if etcd.Spec.ManagementState == operatorv1.Unmanaged { + glog.V(4).Info("etcd cluster in unmanaged") + imgs.ControllerConfigImages.ClusterEtcdOperator = "" + return nil + } + return nil +} + func getRenderConfig(tnamespace, kubeAPIServerServingCA string, ccSpec *mcfgv1.ControllerConfigSpec, imgs *RenderConfigImages, apiServerURL string, pointerConfigData []byte) *renderConfig { return &renderConfig{ TargetNamespace: tnamespace, diff --git a/templates/common/_base/units/machine-config-daemon-pull.service.yaml b/templates/common/_base/units/machine-config-daemon-pull.service.yaml index 8731f1f240..85a9faed0f 100644 --- a/templates/common/_base/units/machine-config-daemon-pull.service.yaml +++ b/templates/common/_base/units/machine-config-daemon-pull.service.yaml @@ -16,8 +16,8 @@ contents: | RemainAfterExit=yes # See https://github.com/coreos/fedora-coreos-tracker/issues/354 ExecStart=/bin/sh -c '/bin/mkdir -p /run/bin && chcon --reference=/usr/bin /run/bin' - ExecStart=/bin/sh -c "/usr/bin/podman pull --authfile=/var/lib/kubelet/config.json --quiet '{{ .Images.gcpRoutesControllerKey }}'" - ExecStart=/bin/sh -c "/usr/bin/podman run --rm --quiet --net=host --entrypoint=cat '{{ .Images.gcpRoutesControllerKey }}' /usr/bin/machine-config-daemon > /run/bin/machine-config-daemon.tmp" + ExecStart=/bin/sh -c "/usr/bin/podman pull --authfile=/var/lib/kubelet/config.json --quiet '{{ .Images.setupEtcdEnvKey }}'" + ExecStart=/bin/sh -c "/usr/bin/podman run --rm --quiet --net=host --entrypoint=cat '{{ .Images.setupEtcdEnvKey }}' /usr/bin/machine-config-daemon > /run/bin/machine-config-daemon.tmp" ExecStart=/bin/sh -c '/usr/bin/chmod a+x /run/bin/machine-config-daemon.tmp && mv /run/bin/machine-config-daemon.tmp /run/bin/machine-config-daemon' {{if .Proxy -}} diff --git a/templates/master/00-master/_base/files/etcd-ca.yaml b/templates/master/00-master/_base/files/etcd-ca.yaml new file mode 100644 index 0000000000..ae45e5d7db --- /dev/null +++ b/templates/master/00-master/_base/files/etcd-ca.yaml @@ -0,0 +1,5 @@ +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/etcd-member/ca.crt" +contents: + inline: | +{{.EtcdCAData | toString | indent 4}} diff --git a/templates/master/00-master/_base/files/etcd-metric-ca.yaml b/templates/master/00-master/_base/files/etcd-metric-ca.yaml new file mode 100644 index 0000000000..5199ac153b --- /dev/null +++ b/templates/master/00-master/_base/files/etcd-metric-ca.yaml @@ -0,0 +1,5 @@ +mode: 0644 +path: "/etc/kubernetes/static-pod-resources/etcd-member/metric-ca.crt" +contents: + inline: | +{{.EtcdMetricCAData | toString | indent 4}}