From b1960abeff747c8f4bb9b2ec7fc6766e73a1f28c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 28 Jun 2024 22:28:38 +0000 Subject: [PATCH] Switch to using kubelet config file for all supported flags Signed-off-by: Brad Davidson --- cmd/agent/main.go | 2 +- cmd/k3s/main.go | 2 +- cmd/server/main.go | 2 +- pkg/agent/config/config.go | 5 + pkg/agent/util/file.go | 21 ++++ pkg/daemons/agent/agent.go | 194 ++++++++++++++++++++++++++++- pkg/daemons/agent/agent_linux.go | 114 ++++++----------- pkg/daemons/agent/agent_windows.go | 94 ++++++-------- pkg/daemons/config/types.go | 1 + 9 files changed, 296 insertions(+), 139 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 1ed96710bbb0..97053f2ab0d4 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -19,6 +19,6 @@ func main() { } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index a3401158bea1..b32873e993cd 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -82,7 +82,7 @@ func main() { } if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/cmd/server/main.go b/cmd/server/main.go index ffa1f60ca952..d62130941026 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -80,6 +80,6 @@ func main() { } if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { - logrus.Fatal(err) + logrus.Fatalf("Error: %v", err) } } diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index b7e2cfc67b56..94c3798359ad 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -533,6 +533,10 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N return nil, err } + // Ensure kubelet config dir exists + kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d") + os.MkdirAll(kubeletConfigDir, 0700) + nodeConfig := &config.Node{ Docker: envInfo.Docker, SELinux: envInfo.EnableSELinux, @@ -561,6 +565,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.ClientCA = clientCAFile + nodeConfig.AgentConfig.KubeletConfigDir = kubeletConfigDir nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController diff --git a/pkg/agent/util/file.go b/pkg/agent/util/file.go index 2420acc8f4bc..54bc1b7d3704 100644 --- a/pkg/agent/util/file.go +++ b/pkg/agent/util/file.go @@ -1,8 +1,11 @@ package util import ( + "bytes" "os" + "os/exec" "path/filepath" + "strings" "github.com/pkg/errors" ) @@ -16,6 +19,8 @@ func WriteFile(name string, content string) error { return nil } +// CopyFile copies the contents of a file. +// If ignoreNotExist is true, no error is returned if the source file does not exist. func CopyFile(sourceFile string, destinationFile string, ignoreNotExist bool) error { os.MkdirAll(filepath.Dir(destinationFile), 0755) input, err := os.ReadFile(sourceFile) @@ -30,3 +35,19 @@ func CopyFile(sourceFile string, destinationFile string, ignoreNotExist bool) er } return nil } + +// kubeadm utility cribbed from: +// https://github.com/kubernetes/kubernetes/blob/v1.25.4/cmd/kubeadm/app/util/copy.go +// Copying this instead of importing from kubeadm saves about 4mb of binary size. + +// CopyDir copies the content of a folder +func CopyDir(src string, dst string) error { + stderr := &bytes.Buffer{} + cmd := exec.Command("cp", "-r", src, dst) + cmd.Stderr = stderr + err := cmd.Run() + if err != nil { + return errors.New(strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 4c426ad81d98..3519dd988baf 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -2,19 +2,33 @@ package agent import ( "context" + "fmt" "math/rand" + "net" "os" + "path/filepath" + "strings" "time" "github.com/k3s-io/k3s/pkg/agent/config" "github.com/k3s-io/k3s/pkg/agent/proxy" + "github.com/k3s-io/k3s/pkg/agent/util" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/executor" + "github.com/k3s-io/k3s/pkg/version" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/component-base/logs" logsapi "k8s.io/component-base/logs/api/v1" + logsv1 "k8s.io/component-base/logs/api/v1" _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + kubeletconfig "k8s.io/kubelet/config/v1beta1" + "k8s.io/kubernetes/pkg/util/taints" + utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" + "sigs.k8s.io/yaml" ) func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy) error { @@ -24,7 +38,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy defer logs.FlushLogs() if err := startKubelet(ctx, &nodeConfig.AgentConfig); err != nil { - return err + return errors.Wrap(err, "failed to start kubelet") } go func() { @@ -46,9 +60,21 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error { } func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error { - argsMap := kubeletArgs(cfg) + argsMap, defaultConfig, err := kubeletArgsAndConfig(cfg) + if err != nil { + return errors.Wrap(err, "prepare default configuration drop-in") + } + + extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs, defaultConfig) + if err != nil { + return errors.Wrap(err, "prepare user configuration drop-ins") + } + + if err := writeKubeletConfig(cfg.KubeletConfigDir, defaultConfig); err != nil { + return errors.Wrap(err, "generate default kubelet configuration drop-in") + } - args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeletArgs) + args := daemonconfig.GetArgs(argsMap, extraArgs) logrus.Infof("Running kubelet %s", daemonconfig.ArgString(args)) return executor.Kubelet(ctx, args) @@ -67,3 +93,165 @@ func ImageCredProvAvailable(cfg *daemonconfig.Agent) bool { } return true } + +// extractConfigArgs strips out any --config or --config-dir flags from the +// provided args list, and if set, copies the content of the file or dir into +// the target drop-in directory. +func extractConfigArgs(path string, extraArgs []string, config *kubeletconfig.KubeletConfiguration) ([]string, error) { + args := make([]string, 0, len(extraArgs)) + strippedArgs := map[string]string{} + var skipVal bool + for i := range extraArgs { + if skipVal { + skipVal = false + continue + } + + var val string + key := strings.TrimPrefix(extraArgs[i], "--") + if k, v, ok := strings.Cut(key, "="); ok { + // key=val pair + key = k + val = v + } else if len(extraArgs) > i+1 { + // key in this arg, value in next arg + val = extraArgs[i+1] + skipVal = true + } + + switch key { + case "config", "config-dir": + if val == "" { + return nil, fmt.Errorf("value required for kubelet-arg --%s", key) + } + strippedArgs[key] = val + default: + args = append(args, extraArgs[i]) + } + } + + if strippedArgs["config"] != "" { + if err := util.CopyFile(strippedArgs["config"], filepath.Join(path, "10-cli-config.conf"), false); err != nil { + return nil, errors.Wrap(err, "copy config into managed drop-in dir") + } + } + if strippedArgs["config-dir"] != "" && strippedArgs["config-dir"] != path { + if err := util.CopyDir(strippedArgs["config"], filepath.Join(path, "20-cli-config-dir")); err != nil { + return nil, errors.Wrap(err, "copy config-dir into managed drop-in dir") + } + } + return args, nil +} + +// writeKubeletConfig marshals the provided KubeletConfiguration object into a +// drop-in config file in the target drop-in directory. +func writeKubeletConfig(path string, config *kubeletconfig.KubeletConfiguration) error { + b, err := yaml.Marshal(config) + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(path, "00-"+version.Program+"-defaults.conf"), b, 0600) +} + +func defaultKubeletConfig(cfg *daemonconfig.Agent) (*kubeletconfig.KubeletConfiguration, error) { + bindAddress := "127.0.0.1" + isIPv6 := utilsnet.IsIPv6(net.ParseIP([]string{cfg.NodeIP}[0])) + if isIPv6 { + bindAddress = "::1" + } + + defaultConfig := &kubeletconfig.KubeletConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kubelet.config.k8s.io/v1beta1", + Kind: "KubeletConfiguration", + }, + CPUManagerReconcilePeriod: metav1.Duration{Duration: time.Second * 10}, + CgroupDriver: "cgroupfs", + ClusterDomain: cfg.ClusterDomain, + EvictionPressureTransitionPeriod: metav1.Duration{Duration: time.Minute * 5}, + FailSwapOn: utilsptr.To(false), + FileCheckFrequency: metav1.Duration{Duration: time.Second * 20}, + HTTPCheckFrequency: metav1.Duration{Duration: time.Second * 20}, + HealthzBindAddress: bindAddress, + ImageMinimumGCAge: metav1.Duration{Duration: time.Minute * 2}, + NodeStatusReportFrequency: metav1.Duration{Duration: time.Minute * 5}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: time.Second * 10}, + ProtectKernelDefaults: cfg.ProtectKernelDefaults, + ReadOnlyPort: 0, + RuntimeRequestTimeout: metav1.Duration{Duration: time.Minute * 2}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: time.Hour * 4}, + SyncFrequency: metav1.Duration{Duration: time.Minute}, + VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, + EvictionHard: map[string]string{ + "imagefs.available": "5%", + "nodefs.available": "5%", + }, + EvictionMinimumReclaim: map[string]string{ + "imagefs.available": "10%", + "nodefs.available": "10%", + }, + Authentication: kubeletconfig.KubeletAuthentication{ + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: utilsptr.To(false), + }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: utilsptr.To(true), + CacheTTL: metav1.Duration{Duration: time.Minute * 2}, + }, + }, + Authorization: kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + Webhook: kubeletconfig.KubeletWebhookAuthorization{ + CacheAuthorizedTTL: metav1.Duration{Duration: time.Minute * 5}, + CacheUnauthorizedTTL: metav1.Duration{Duration: time.Second * 30}, + }, + }, + Logging: logsv1.LoggingConfiguration{ + Format: "text", + Verbosity: logsv1.VerbosityLevel(cfg.VLevel), + FlushFrequency: logsv1.TimeOrMetaDuration{ + Duration: metav1.Duration{Duration: time.Second * 5}, + SerializeAsString: true, + }, + }, + } + + if cfg.ListenAddress != "" { + defaultConfig.Address = cfg.ListenAddress + } + + if cfg.ClientCA != "" { + defaultConfig.Authentication.X509.ClientCAFile = cfg.ClientCA + } + + if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { + defaultConfig.TLSCertFile = cfg.ServingKubeletCert + defaultConfig.TLSPrivateKeyFile = cfg.ServingKubeletKey + } + + for _, addr := range cfg.ClusterDNSs { + defaultConfig.ClusterDNS = append(defaultConfig.ClusterDNS, addr.String()) + } + + if cfg.ResolvConf != "" { + defaultConfig.ResolverConfig = utilsptr.To(cfg.ResolvConf) + } + + if cfg.PodManifests != "" && defaultConfig.StaticPodPath == "" { + defaultConfig.StaticPodPath = cfg.PodManifests + } + if err := os.MkdirAll(defaultConfig.StaticPodPath, 0750); err != nil { + return nil, errors.Wrapf(err, "failed to create static pod manifest dir %s", defaultConfig.StaticPodPath) + } + + if t, _, err := taints.ParseTaints(cfg.NodeTaints); err != nil { + return nil, errors.Wrap(err, "failed to parse node taints") + } else { + defaultConfig.RegisterWithTaints = t + } + + logsv1.VModuleConfigurationPflag(&defaultConfig.Logging.VModule).Set(cfg.VModule) + + return defaultConfig, nil +} diff --git a/pkg/daemons/agent/agent_linux.go b/pkg/daemons/agent/agent_linux.go index 5e22fbc085f7..7ba37a035338 100644 --- a/pkg/daemons/agent/agent_linux.go +++ b/pkg/daemons/agent/agent_linux.go @@ -5,7 +5,6 @@ package agent import ( "net" - "os" "path/filepath" "strconv" "strings" @@ -13,23 +12,25 @@ import ( "github.com/k3s-io/k3s/pkg/cgroups" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + kubeletconfig "k8s.io/kubelet/config/v1beta1" utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" ) const socketPrefix = "unix://" -func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) { +func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) error { argsMap["feature-gates=KubeletInUserNamespace"] = "true" // "/sys/fs/cgroup" is namespaced cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil if controllers["cpu"] && controllers["pids"] && cgroupfsWritable { logrus.Info("cgroup v2 controllers are delegated for rootless.") - return + return nil } - logrus.Fatal("delegated cgroup v2 controllers are required for rootless.") + return errors.New("delegated cgroup v2 controllers are required for rootless") } func kubeProxyArgs(cfg *config.Agent) map[string]string { @@ -65,74 +66,49 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { return argsMap } -func kubeletArgs(cfg *config.Agent) map[string]string { - bindAddress := "127.0.0.1" - isIPv6 := utilsnet.IsIPv6(net.ParseIP([]string{cfg.NodeIP}[0])) - if isIPv6 { - bindAddress = "::1" +// kubeletArgsAndConfig generates default kubelet args and configuration. +// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them, +// and a structured configuration file that upstream does not provide a convienent way to initailize with default values. +// The defaults and our desired config also vary by OS. +func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) { + defaultConfig, err := defaultKubeletConfig(cfg) + if err != nil { + return nil, nil, err } argsMap := map[string]string{ - "healthz-bind-address": bindAddress, - "read-only-port": "0", - "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfigKubelet, - "eviction-hard": "imagefs.available<5%,nodefs.available<5%", - "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", - "fail-swap-on": "false", - "cgroup-driver": "cgroupfs", - "authentication-token-webhook": "true", - "anonymous-auth": "false", - "authorization-mode": modes.ModeWebhook, - } - if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" { - argsMap["pod-manifest-path"] = cfg.PodManifests - } - if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil { - logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err) + "config-dir": cfg.KubeletConfigDir, + "kubeconfig": cfg.KubeConfigKubelet, } + if cfg.RootDir != "" { argsMap["root-dir"] = cfg.RootDir argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki") } - if len(cfg.ClusterDNS) > 0 { - argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs) - } - if cfg.ResolvConf != "" { - argsMap["resolv-conf"] = cfg.ResolvConf - } if cfg.RuntimeSocket != "" { - argsMap["serialize-image-pulls"] = "false" + defaultConfig.SerializeImagePulls = utilsptr.To(false) + // note: this is a legacy cadvisor flag that the kubelet still exposes, but + // it must be set in order for cadvisor to pull stats properly. if strings.Contains(cfg.RuntimeSocket, "containerd") { argsMap["containerd"] = cfg.RuntimeSocket } // cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) { - argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket } else { - argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket } } if cfg.ImageServiceSocket != "" { if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) { - argsMap["image-service-endpoint"] = cfg.ImageServiceSocket + defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket } else { - argsMap["image-service-endpoint"] = socketPrefix + cfg.ImageServiceSocket + defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket } } - if cfg.ListenAddress != "" { - argsMap["address"] = cfg.ListenAddress - } - if cfg.ClientCA != "" { - argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.ClientCA - } - if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { - argsMap["tls-cert-file"] = cfg.ServingKubeletCert - argsMap["tls-private-key-file"] = cfg.ServingKubeletKey - } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } + // If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe. // When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly. // This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet @@ -143,35 +119,30 @@ func kubeletArgs(cfg *config.Agent) map[string]string { argsMap["node-ip"] = cfg.NodeIP } } else { + argsMap["cloud-provider"] = "external" // Cluster is using the embedded CCM, we know that the feature-gate will be enabled there as well. argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "CloudDualStackNodeIPs=true") if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" { argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs) } } + kubeletRoot, runtimeRoot, controllers := cgroups.CheckCgroups() + if !controllers["pids"] { + return nil, nil, errors.New("pids cgroup controller not found") + } if !controllers["cpu"] { logrus.Warn("Disabling CPU quotas due to missing cpu controller or cpu.cfs_period_us") - argsMap["cpu-cfs-quota"] = "false" - } - if !controllers["pids"] { - logrus.Fatal("pids cgroup controller not found") + defaultConfig.CPUCFSQuota = utilsptr.To(false) } if kubeletRoot != "" { - argsMap["kubelet-cgroups"] = kubeletRoot + defaultConfig.KubeletCgroups = kubeletRoot } if runtimeRoot != "" { argsMap["runtime-cgroups"] = runtimeRoot } argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",") - if len(cfg.NodeTaints) > 0 { - argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",") - } - - if !cfg.DisableCCM { - argsMap["cloud-provider"] = "external" - } if ImageCredProvAvailable(cfg) { logrus.Infof("Kubelet image credential provider bin dir and configuration file found.") @@ -180,25 +151,18 @@ func kubeletArgs(cfg *config.Agent) map[string]string { } if cfg.Rootless { - createRootlessConfig(argsMap, controllers) + if err := createRootlessConfig(argsMap, controllers); err != nil { + return nil, nil, err + } } if cfg.Systemd { - argsMap["cgroup-driver"] = "systemd" - } - - if cfg.ProtectKernelDefaults { - argsMap["protect-kernel-defaults"] = "true" + defaultConfig.CgroupDriver = "systemd" } if !cfg.DisableServiceLB { - argsMap["allowed-unsafe-sysctls"] = "net.ipv4.ip_forward,net.ipv6.conf.all.forwarding" + defaultConfig.AllowedUnsafeSysctls = []string{"net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding"} } - if cfg.VLevel != 0 { - argsMap["v"] = strconv.Itoa(cfg.VLevel) - } - if cfg.VModule != "" { - argsMap["vmodule"] = cfg.VModule - } - return argsMap + + return argsMap, defaultConfig, nil } diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 69995f5e9f7e..483f0f0c5588 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -4,15 +4,15 @@ package agent import ( - "os" "path/filepath" "strings" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/net" - "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" + kubeletconfig "k8s.io/kubelet/config/v1beta1" + utilsnet "k8s.io/utils/net" + utilsptr "k8s.io/utils/ptr" ) const ( @@ -38,86 +38,64 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { return argsMap } -func kubeletArgs(cfg *config.Agent) map[string]string { - bindAddress := "127.0.0.1" - _, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP}) - if IPv6only { - bindAddress = "::1" +// kubeletArgsAndConfig generates default kubelet args and configuration. +// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them, +// and a structured configuration file that upstream does not provide a convienent way to initailize with default values. +// The defaults and our desired config also vary by OS. +func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) { + defaultConfig, err := defaultKubeletConfig(cfg) + if err != nil { + return nil, nil, err } argsMap := map[string]string{ - "healthz-bind-address": bindAddress, - "read-only-port": "0", - "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfigKubelet, - "eviction-hard": "imagefs.available<5%,nodefs.available<5%", - "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", - "fail-swap-on": "false", - "authentication-token-webhook": "true", - "anonymous-auth": "false", - "authorization-mode": modes.ModeWebhook, - } - if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" { - argsMap["pod-manifest-path"] = cfg.PodManifests - } - if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil { - logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err) + "config-dir": cfg.KubeletConfigDir, + "kubeconfig": cfg.KubeConfigKubelet, } if cfg.RootDir != "" { argsMap["root-dir"] = cfg.RootDir argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki") } - if len(cfg.ClusterDNS) > 0 { - argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs) - } - if cfg.ResolvConf != "" { - argsMap["resolv-conf"] = cfg.ResolvConf - } if cfg.RuntimeSocket != "" { - argsMap["serialize-image-pulls"] = "false" + defaultConfig.SerializeImagePulls = utilsptr.To(false) // cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) { - argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket } else { - argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket + defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket } } - if cfg.ListenAddress != "" { - argsMap["address"] = cfg.ListenAddress - } - if cfg.ClientCA != "" { - argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.ClientCA - } - if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { - argsMap["tls-cert-file"] = cfg.ServingKubeletCert - argsMap["tls-private-key-file"] = cfg.ServingKubeletKey + if cfg.ImageServiceSocket != "" { + if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) { + defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket + } else { + defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket + } } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } - defaultIP, err := net.ChooseHostInterface() - if err != nil || defaultIP.String() != cfg.NodeIP { - argsMap["node-ip"] = cfg.NodeIP - } - - argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",") - if len(cfg.NodeTaints) > 0 { - argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",") - } - if !cfg.DisableCCM { + if cfg.DisableCCM { + dualStack, err := utilsnet.IsDualStackIPs(cfg.NodeIPs) + if err == nil && !dualStack { + argsMap["node-ip"] = cfg.NodeIP + } + } else { argsMap["cloud-provider"] = "external" + // Cluster is using the embedded CCM, we know that the feature-gate will be enabled there as well. + argsMap["feature-gates"] = util.AddFeatureGate(argsMap["feature-gates"], "CloudDualStackNodeIPs=true") + if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" { + argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs) + } } + argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",") + if ImageCredProvAvailable(cfg) { logrus.Infof("Kubelet image credential provider bin dir and configuration file found.") argsMap["image-credential-provider-bin-dir"] = cfg.ImageCredProvBinDir argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig } - if cfg.ProtectKernelDefaults { - argsMap["protect-kernel-defaults"] = "true" - } - - return argsMap + return argsMap, defaultConfig, nil } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 33db100222a2..721539162a00 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -101,6 +101,7 @@ type Agent struct { ClusterDomain string ResolvConf string RootDir string + KubeletConfigDir string KubeConfigKubelet string KubeConfigKubeProxy string KubeConfigK3sController string