Updates to user-scheduler's coupling to the kube-scheduler binary

jupyterhub · Sep 12, 2020 · c581968 · c581968
1 parent cd1eff7
commit c581968
Show file tree

Hide file tree

Showing 3 changed files with 182 additions and 36 deletions.
diff --git a/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml b/jupyterhub/templates/scheduling/user-scheduler/deployment.yaml
@@ -13,10 +13,8 @@ spec:
   template:
     metadata:
       labels:
-        {{- /* Changes here will cause the Deployment to restart the pods. */}}
         {{- include "jupyterhub.matchLabels" . | nindent 8 }}
       annotations:
-        # This lets us autorestart when the configmap changes!
         checksum/config-map: {{ include (print $.Template.BasePath "/scheduling/user-scheduler/configmap.yaml") . | sha256sum }}
     spec:
       {{- if .Values.rbac.enabled }}
@@ -31,6 +29,7 @@ spec:
         - name: user-scheduler
           image: {{ include "jupyterhub.scheduler.image" . }}
           command:
+            # ref: https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/
             - /usr/local/bin/kube-scheduler
             - --scheduler-name={{ .Release.Name }}-user-scheduler
             - --policy-configmap=user-scheduler

diff --git a/jupyterhub/templates/scheduling/user-scheduler/rbac.yaml b/jupyterhub/templates/scheduling/user-scheduler/rbac.yaml
@@ -7,58 +7,205 @@ metadata:
   labels:
     {{- include "jupyterhub.labels" . | nindent 4 }}
 ---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: {{ .Release.Name }}-user-scheduler-base
-  labels:
-    {{- $_ := merge (dict "componentSuffix" "-base") . }}
-    {{- include "jupyterhub.labels" $_ | nindent 4 }}
-subjects:
-  - kind: ServiceAccount
-    name: user-scheduler
-    namespace: {{ .Release.Namespace }}
-roleRef:
-  kind: ClusterRole
-  name: system:kube-scheduler
-  apiGroup: rbac.authorization.k8s.io
----
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: {{ .Release.Name }}-user-scheduler-complementary
+  name: {{ .Release.Name }}-user-scheduler
   labels:
-    {{- $_ := merge (dict "componentSuffix" "-complementary") . }}
-    {{- include "jupyterhub.labels" $_ | nindent 4 }}
+    {{- include "jupyterhub.labels" . | nindent 4 }}
 rules:
-  # Support leader elections
+  # Added for configmap based leader election
   - apiGroups: [""]
     resourceNames: ["user-scheduler"]
     resources: ["configmaps"]
     verbs: ["get", "update"]
-  # Workaround for missing permission in system:kube-scheduler as of k8s 1.10.4
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  # Workaround for missing permission with rancher local-path-provisioner
-  - apiGroups: [""]
-    resources: ["persistentvolume", "persistentvolumeclaims"]
-    verbs: ["update"]
+
+  # Copied from the system:kube-scheduler ClusterRole of the k8s version
+  # matching the kube-scheduler binary we use. These have been unchanged between
+  # 1.12 and 1.15, but changed in 1.16 and in 1.17, but not 1.18 and 1.19.
+  #
+  # ref: https://github.com/kubernetes/kubernetes/blob/v1.19.0/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml#L696-L829
+  rules:
+  - apiGroups:
+    - ""
+    - events.k8s.io
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+    - update
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - create
+  - apiGroups:
+    - coordination.k8s.io
+    resourceNames:
+    - kube-scheduler
+    resources:
+    - leases
+    verbs:
+    - get
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - endpoints
+    verbs:
+    - create
+  - apiGroups:
+    - ""
+    resourceNames:
+    - kube-scheduler
+    resources:
+    - endpoints
+    verbs:
+    - get
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - pods
+    verbs:
+    - delete
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - bindings
+    - pods/binding
+    verbs:
+    - create
+  - apiGroups:
+    - ""
+    resources:
+    - pods/status
+    verbs:
+    - patch
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - replicationcontrollers
+    - services
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - apps
+    - extensions
+    resources:
+    - replicasets
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - apps
+    resources:
+    - statefulsets
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - policy
+    resources:
+    - poddisruptionbudgets
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - persistentvolumeclaims
+    - persistentvolumes
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - authentication.k8s.io
+    resources:
+    - tokenreviews
+    verbs:
+    - create
+  - apiGroups:
+    - authorization.k8s.io
+    resources:
+    - subjectaccessreviews
+    verbs:
+    - create
+  - apiGroups:
+    - storage.k8s.io
+    resources:
+    - csinodes
+    verbs:
+    - get
+    - list
+    - watch
+  # Copied from the system:volume-scheduler ClusterRole of the k8s version
+  # matching the kube-scheduler binary we use. These have not changed between
+  # 1.12 and 1.19.
+  #
+  # ref: https://github.com/kubernetes/kubernetes/blob/v1.19.0/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml#L1213-L1240
+  - apiGroups:
+    - ""
+    resources:
+    - persistentvolumes
+    verbs:
+    - get
+    - list
+    - patch
+    - update
+    - watch
+  - apiGroups:
+    - storage.k8s.io
+    resources:
+    - storageclasses
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - persistentvolumeclaims
+    verbs:
+    - get
+    - list
+    - patch
+    - update
+    - watch
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: {{ .Release.Name }}-user-scheduler-complementary
+  name: {{ .Release.Name }}-user-scheduler
   labels:
-    {{- $_ := merge (dict "componentSuffix" "-complementary") . }}
-    {{- include "jupyterhub.labels" $_ | nindent 4 }}
+    {{- include "jupyterhub.labels" . | nindent 4 }}
 subjects:
   - kind: ServiceAccount
     name: user-scheduler
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
-  name: {{ .Release.Name }}-user-scheduler-complementary
+  name: {{ .Release.Name }}-user-scheduler
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
 {{- end }}
diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml
@@ -326,8 +326,8 @@ scheduling:
     ## to breaking changes in the kube-scheduler binary.
     policy: {}
     image:
-      name: gcr.io/google_containers/kube-scheduler-amd64
-      tag: v1.16.11
+      name: k8s.gcr.io/kube-scheduler
+      tag: v1.19.1
     nodeSelector: {}
     pdb:
       enabled: true