set net.ipv4.tcp_keepalive_time and net.core.somaxconn for tidb and t…

…ikv in init container (#1107)

* set net.ipv4.tcp_keepalive_time and net.core.somaxconn for tidb and tikv in init container

* add new function to get image for init container

* add comments

* fix gofmt error

charts/tidb-cluster/values.yaml

@@ -177,10 +177,6 @@ pd:
     # # when the kubelet is configured to allow unsafe sysctls
     # - name: net.core.somaxconn
     #   value: "32768"
-    # - name: net.ipv4.tcp_syncookies
-    #   value: "0"
-    # - name: net.ipv4.tcp_tw_recycle
-    #   value: "0"
 
   # Specify the priorityClassName for PD Pod.
   # refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#how-to-use-priority-and-preemption
@@ -275,10 +271,6 @@ tikv:
     # # when the kubelet is configured to allow unsafe sysctls
     # - name: net.core.somaxconn
     #   value: "32768"
-    # - name: net.ipv4.tcp_syncookies
-    #   value: "0"
-    # - name: net.ipv4.tcp_tw_recycle
-    #   value: "0"
 
   # Specify the priorityClassName for TiKV Pod.
   # refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#how-to-use-priority-and-preemption
@@ -362,10 +354,6 @@ tidb:
     # # when the kubelet is configured to allow unsafe sysctls
     # - name: net.core.somaxconn
     #   value: "32768"
-    # - name: net.ipv4.tcp_syncookies
-    #   value: "0"
-    # - name: net.ipv4.tcp_tw_recycle
-    #   value: "0"
 
     # # Load balancers usually have an idle timeout (eg. AWS NLB idle timeout is 350),
     # # the tcp_keepalive_time must be set to lower than LB idle timeout.

deploy/modules/aliyun/tidb-cluster/values/default.yaml

@@ -8,10 +8,26 @@ pd:
       storage: 20Gi
   storageClassName: alicloud-disk
 tikv:
+  annotations:
+    tidb.pingcap.com/sysctl-init: "true"
+  podSecurityContext:
+    sysctls:
+    - name: net.core.somaxconn
+      value: "32768"
   logLevel: info
   storageClassName: local-volume
   syncLog: true
 tidb:
+  annotations:
+    tidb.pingcap.com/sysctl-init: "true"
+  podSecurityContext:
+    sysctls:
+    - name: net.core.somaxconn
+      value: "32768"
+    - name: net.ipv4.tcp_keepalive_intvl
+      value: "75"
+    - name: net.ipv4.tcp_keepalive_time
+      value: "300"
   logLevel: info
   service:
     type: LoadBalancer

deploy/modules/gcp/tidb-cluster/values/default.yaml

@@ -4,8 +4,24 @@ timezone: UTC
 pd:
   storageClassName: pd-ssd
 tikv:
+  annotations:
+    tidb.pingcap.com/sysctl-init: "true"
+  podSecurityContext:
+    sysctls:
+    - name: net.core.somaxconn
+      value: "32768"
   storageClassName: local-storage
 tidb:
+  annotations:
+    tidb.pingcap.com/sysctl-init: "true"
+  podSecurityContext:
+    sysctls:
+    - name: net.core.somaxconn
+      value: "32768"
+    - name: net.ipv4.tcp_keepalive_intvl
+      value: "75"
+    - name: net.ipv4.tcp_keepalive_time
+      value: "300"
   service:
     type: LoadBalancer
     externalTrafficPolicy: Local

pkg/controller/controller_utils.go

@@ -201,6 +201,14 @@ func TiKVCapacity(limits *v1alpha1.ResourceRequirement) string {
 	return fmt.Sprintf("%dMB", i/humanize.MiByte)
 }
 
+// Reuse the SlowLogTailer image for TiDB
+func GetUtilImage(cluster *v1alpha1.TidbCluster) string {
+	if img := cluster.Spec.TiDB.SlowLogTailer.Image; img != "" {
+		return img
+	}
+	return defaultTiDBLogTailerImage
+}
+
 func GetSlowLogTailerImage(cluster *v1alpha1.TidbCluster) string {
 	if img := cluster.Spec.TiDB.SlowLogTailer.Image; img != "" {
 		return img

pkg/label/label.go

@@ -75,9 +75,13 @@ const (
 	AnnForceUpgradeKey = "tidb.pingcap.com/force-upgrade"
 	// AnnPDDeferDeleting is pd pod annotation key  in pod for defer for deleting pod
 	AnnPDDeferDeleting = "tidb.pingcap.com/pd-defer-deleting"
+	// AnnSysctlInit is pod annotation key to indicate whether configuring sysctls with init container
+	AnnSysctlInit = "tidb.pingcap.com/sysctl-init"
 
 	// AnnForceUpgradeVal is tc annotation value to indicate whether force upgrade should be done
 	AnnForceUpgradeVal = "true"
+	// AnnSysctlInitVal is pod annotation value to indicate whether configuring sysctls with init container
+	AnnSysctlInitVal = "true"
 
 	// PDLabelVal is PD label value
 	PDLabelVal string = "pd"

pkg/manager/member/tidb_member_manager.go

@@ -266,6 +266,39 @@ func getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster) *apps.StatefulSet {
 		})
 	}
 
+	sysctls := "sysctl -w"
+	var initContainers []corev1.Container
+	if tc.Spec.TiDB.Annotations != nil {
+		init, ok := tc.Spec.TiDB.Annotations[label.AnnSysctlInit]
+		if ok && (init == label.AnnSysctlInitVal) {
+			if tc.Spec.TiDB.PodSecurityContext != nil && len(tc.Spec.TiDB.PodSecurityContext.Sysctls) > 0 {
+				for _, sysctl := range tc.Spec.TiDB.PodSecurityContext.Sysctls {
+					sysctls = sysctls + fmt.Sprintf(" %s=%s", sysctl.Name, sysctl.Value)
+				}
+				privileged := true
+				initContainers = append(initContainers, corev1.Container{
+					Name:  "init",
+					Image: controller.GetUtilImage(tc),
+					Command: []string{
+						"sh",
+						"-c",
+						sysctls,
+					},
+					SecurityContext: &corev1.SecurityContext{
+						Privileged: &privileged,
+					},
+				})
+			}
+		}
+	}
+	// Init container is only used for the case where allowed-unsafe-sysctls
+	// cannot be enabled for kubelet, so clean the sysctl in statefulset
+	// SecurityContext if init container is enabled
+	podSecurityContext := tc.Spec.TiDB.PodSecurityContext.DeepCopy()
+	if len(initContainers) > 0 {
+		podSecurityContext.Sysctls = []corev1.Sysctl{}
+	}
+
 	var containers []corev1.Container
 	if tc.Spec.TiDB.SeparateSlowLog {
 		// mount a shared volume and tail the slow log to STDOUT using a sidecar.
@@ -383,8 +416,9 @@ func getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbCluster) *apps.StatefulSet {
 					RestartPolicy:     corev1.RestartPolicyAlways,
 					Tolerations:       tc.Spec.TiDB.Tolerations,
 					Volumes:           vols,
-					SecurityContext:   tc.Spec.TiDB.PodSecurityContext,
+					SecurityContext:   podSecurityContext,
 					PriorityClassName: tc.Spec.TiDB.PriorityClassName,
+					InitContainers:    initContainers,
 				},
 			},
 			ServiceName:         controller.TiDBPeerMemberName(tcName),