user-scheduler: fallback to modern kube-scheduler version and config

jupyterhub/files/userscheduler-defaultpolicy.yaml

@@ -0,0 +1,76 @@
+{
+  "kind": "Policy",
+  "apiVersion": "v1",
+  "predicates": [
+    { "name": "PodFitsResources" },
+    { "name": "HostName" },
+    { "name": "PodFitsHostPorts" },
+    { "name": "MatchNodeSelector" },
+    { "name": "NoDiskConflict" },
+    { "name": "PodToleratesNodeTaints" },
+    { "name": "MaxEBSVolumeCount" },
+    { "name": "MaxGCEPDVolumeCount" },
+    { "name": "MaxAzureDiskVolumeCount" },
+    { "name": "MaxCSIVolumeCountPred" },
+    { "name": "CheckVolumeBinding" },
+    { "name": "NoVolumeZoneConflict" },
+    { "name": "MatchInterPodAffinity" }
+  ],
+  "priorities": [
+    { "name": "NodePreferAvoidPodsPriority",  "weight": 161051 },
+    { "name": "NodeAffinityPriority",         "weight": 14641 },
+    { "name": "InterPodAffinityPriority",     "weight": 1331 },
+    { "name": "MostRequestedPriority",        "weight": 121 },
+    { "name": "ImageLocalityPriority",        "weight": 11}
+  ],
+  "hardPodAffinitySymmetricWeight" : 100,
+  "alwaysCheckAllPredicates" : false
+}
+
+# # Notes about ranges
+# ImageLocalityPriority       - ranges from 0-10 * 11
+# MostRequestedPriority       - ranges from 0-10 * 11^2
+# InterPodAffinityPriority    - ranges from 0-1  * 11^3 (i guess)
+# NodeAffinityPriority        - ranges from 0-1  * 11^4 (i guess)
+# NodePreferAvoidPodsPriority - ranges from 0-1  * 11^5 (i guess)
+
+# # Notes about the GeneralPredicates
+# The following predicates was not found by kube-scheduler 1.11.1-beta.0
+# { "name": "CheckNodePIDPressure" },
+# { "name": "CheckNodeUnschedulable" },
+# { "name": "CheckNodeCondition" },
+# { "name": "General" },
+# { "name": "PodToleratesNodeNoExecuteTaints" },
+# { "name": "CheckNodeMemoryPressure" },
+# { "name": "CheckNodeDiskPressure" },
+
+# # Notes about the priorities
+# NodePreferAvoidPodsPriority: What does this really mean?
+# HardPodAffinitySymmetricWeight: "It represents the weight of implicit
+# PreferredDuringScheduling affinity rule." - preferred node affinity or preferred
+# pod/anti-pod affinity or those affinities in general? How does this relate to
+# the InterPodAffinityPriority and NodeAffinityPriority?
+
+# AlwaysCheckAllPredicates: scheduler checks all the configured predicates even
+# after one or more of them fails.
+
+# GeneralPredicates checks whether noncriticalPredicates and EssentialPredicates
+# pass. noncriticalPredicates are the predicates that only non-critical pods need
+# and EssentialPredicates are the predicates that all pods, including critical
+# pods, need
+
+# MostRequestedPriority: Is using the default MostRequestedPriorityMap that is a
+# priority function that favors nodes with most requested resources. It calculates
+# the percentage of memory and CPU requested by pods scheduled on the node, and
+# prioritizes based on the maximum of the average of the fraction of requested to
+# capacity.
+
+# Details: (cpu(10 * sum(requested) / capacity) + memory(10 * sum(requested) /
+# capacity)) / 2
+
+# ImageLocalityPriorityMap is a priority function that favors nodes that already
+# have requested pod container's images. It will detect whether the requested
+# images are present on a node, and then calculate a score ranging from 0 to 10
+# based on the total size of those images. - If none of the images are present,
+# this node will be given the lowest priority. - If some of the images are present
+# on a node, the larger their sizes' sum, the higher the node's priority.

jupyterhub/templates/scheduling/user-scheduler/configmap.yaml

@@ -27,4 +27,7 @@ data:
             enabled:
               - name: NodeResourcesMostAllocated
           {{- end }}
+
+  {{- $defaultPolicy := .Files.Get "files/userscheduler-defaultpolicy.yaml" | fromYaml }}
+  policy.cfg: {{ .Values.scheduling.userScheduler.policy | default $defaultPolicy | toJson | quote }}
 {{- end }}

jupyterhub/templates/scheduling/user-scheduler/deployment.yaml

@@ -31,16 +31,31 @@ spec:
             name: user-scheduler
       containers:
         - name: user-scheduler
-          image: {{ include "jupyterhub.scheduler.image" . }}
+          # NOTE: When kube-scheduler 1.17+ fail to find CSINode resource in the
+          #       cluster, they fail to startup. Due to this, we fallback to the
+          #       latest functional version with its legacy configuration.
+          {{- if .Capabilities.APIVersions.Has "storage.k8s.io/v1/CSINode" }}
+          image: {{ .Values.scheduling.userScheduler.image.name }}:{{ .Values.scheduling.userScheduler.image.tag }}
+          {{- else }}
+          image: {{ .Values.scheduling.userScheduler.image.name }}:v1.16.11
+          {{- end }}
           command:
-            # NOTE: --leader-elect- flags are silently ignored in favor of whats
-            #       defined in the passed KubeSchedulerConfiguration.
+            # NOTE: --leader-elect-... (new) and --lock-object-... (deprecated)
+            #       flags are silently ignored in favor of whats defined in the
+            #       passed KubeSchedulerConfiguration whenever --config is
+            #       passed.
             #
             # ref: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/
             # ref: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/
             - /usr/local/bin/kube-scheduler
-            - --scheduler-name={{ .Release.Name }}-user-scheduler
+            {{- if .Capabilities.APIVersions.Has "storage.k8s.io/v1/CSINode" }}
             - --config=/etc/user-scheduler/config.yaml
+            {{- else }}
+            - --scheduler-name={{ .Release.Name }}-user-scheduler
+            - --policy-config-file=/etc/user-scheduler/policy.cfg
+            - --lock-object-name=user-scheduler-lock
+            - --lock-object-namespace={{ .Release.Namespace }}
+            {{- end }}
             - --v={{ .Values.scheduling.userScheduler.logLevel | default 4 }}
           volumeMounts:
             - mountPath: /etc/user-scheduler