Tighten and flesh out networkpolicies

dev-config.yaml

@@ -12,6 +12,7 @@ proxy:
         cpu: 0
   networkPolicy:
     enabled: true
+    egress: []  # overrides allowance of 0.0.0.0/0
 
 hub:
   cookieSecret: 1470700e01f77171c2c67b12130c25081dfbdf2697af8c2f2bd05621b31100bf
@@ -27,7 +28,11 @@ hub:
       apiToken: 0cc05feaefeeb29179e924ffc6d3886ffacf0d1a28ab225f5c210436ffc5cfd5
   networkPolicy:
     enabled: true
-
+    egress:  # overrides allowance of 0.0.0.0/0
+      # In kind/k3s clusters the Kubernetes API server is exposing this port
+      - ports:
+        - protocol: TCP
+          port: 6443
 
 singleuser:
   storage:
@@ -36,21 +41,13 @@ singleuser:
     guarantee: null
   networkPolicy:
     enabled: true
-    # Block all egress apart from DNS and jupyter.org
-    # CIDR must match the allowed URL in test_singleuser_netpol
+    # For testing purposes in test_singleuser_netpol
     egress:
-      - ports:
-        ## port 53 is the default port for DNS queries
-        - port: 53
-          protocol: UDP
       - to:
-        ## nslookup jupyter.org
-        ## - 104.28.9.110
-        ## - 104.28.8.110
         - ipBlock:
-            cidr: 104.28.9.110/32
+            cidr: 104.28.9.110/32       # jupyter.org 1
         - ipBlock:
-            cidr: 104.28.8.110/32
+            cidr: 104.28.8.110/32       # jupyter.org 2
 
 prePuller:
   hook:

images/singleuser-sample/Dockerfile

@@ -14,7 +14,9 @@ ARG JUPYTERHUB_VERSION=1.1.*
 # NOTE: git is already available in the jupyter/minimal-notebook image.
 USER root
 RUN apt-get update && apt-get install --yes --no-install-recommends \
+    dnsutils \
     git \
+    iputils-ping \
  && rm -rf /var/lib/apt/lists/*
 USER $NB_USER
 

jupyterhub/templates/hub/netpol.yaml

@@ -12,26 +12,48 @@ spec:
   policyTypes:
     - Ingress
     - Egress
+
   ingress:
-    - from:
+    # allowed pods (hub.jupyter.org/network-access-hub) --> hub
+    - ports:
+        - port: http
+      from:
         - podSelector:
             matchLabels:
               hub.jupyter.org/network-access-hub: "true"
-      ports:
-        - port: http
-    {{- /* Useful if you want to give hub access to pods from other namespaces */}}
-    {{- if .Values.hub.networkPolicy.ingress}}
-    {{- .Values.hub.networkPolicy.ingress| toYaml | trimSuffix "\n" | nindent 4 }}
+
+    {{- with .Values.hub.networkPolicy.ingress }}
+    # default: nothing --> hub
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
+
   egress:
-    {{- /*
-    The default is to allow all egress for hub If you want to restrict it the
-    following egress is required
-      -  proxy:8001
-      -  singleuser:8888
-      -  Kubernetes api-server
-    */}}
-    {{- if .Values.hub.networkPolicy.egress }}
-    {{- .Values.hub.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }}
+    # hub --> proxy
+    - ports:
+        - port: 8001
+      to:
+        - podSelector:
+            matchLabels:
+              {{- $_ := merge (dict "componentLabel" "proxy") . }}
+              {{- include "jupyterhub.matchLabels" $_ | nindent 14 }}
+    # hub --> singleuser-server
+    - ports:
+        - port: 8888
+      to:
+        - podSelector:
+            matchLabels:
+              {{- $_ := merge (dict "componentLabel" "singleuser-server") . }}
+              {{- include "jupyterhub.matchLabels" $_ | nindent 14 }}
+
+    # hub -> Kubernetes internal DNS
+    - ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+
+    {{- with .Values.hub.networkPolicy.egress }}
+    # hub --> default: everything
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
 {{- end }}

jupyterhub/templates/proxy/netpol.yaml

@@ -16,37 +16,60 @@ spec:
   policyTypes:
     - Ingress
     - Egress
+
   ingress:
+    # allowed pods (hub.jupyter.org/network-access-proxy-http) --> proxy (http/https port)
     - ports:
-      - port: http
-      {{- if or $manualHTTPS $manualHTTPSwithsecret}}
-      - port: https
-      {{- end }}
-    - from:
+        - port: http
+        {{- if or $manualHTTPS $manualHTTPSwithsecret }}
+        - port: https
+        {{- end }}
+      from:
         - podSelector:
             matchLabels:
               hub.jupyter.org/network-access-proxy-http: "true"
-      ports:
-        - port: http
-    - from:
+
+    # allowed pods (hub.jupyter.org/network-access-proxy-api) --> proxy (api port)
+    - ports:
+        - port: api
+      from:
         - podSelector:
             matchLabels:
               hub.jupyter.org/network-access-proxy-api: "true"
-      ports:
-        - port: api
-    {{- /* Useful if you want to give proxy access to pods from other namespaces */}}
-    {{- if .Values.proxy.networkPolicy.ingress}}
-    {{- .Values.proxy.networkPolicy.ingress | toYaml | trimSuffix "\n" | nindent 4 }}
+
+    {{- with .Values.proxy.networkPolicy.ingress}}
+    # default: nothing --> proxy
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
+
   egress:
-    {{- /*
-    The default is to allow all egress for proxy If you want to restrict it the
-    following egress is required
-      -  hub:8081
-      -  singleuser:8888
-      -  Kubernetes api-server
-    */}}
-    {{- if .Values.proxy.networkPolicy.egress }}
-    {{- .Values.proxy.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }}
+    # proxy --> hub
+    - ports:
+        - port: 8081
+      to:
+        - podSelector:
+            matchLabels:
+              {{- $_ := merge (dict "componentLabel" "hub") . }}
+              {{- include "jupyterhub.matchLabels" $_ | nindent 14 }}
+
+    # proxy --> singleuser-server
+    - ports:
+        - port: 8888
+      to:
+        - podSelector:
+            matchLabels:
+              {{- $_ := merge (dict "componentLabel" "singleuser-server") . }}
+              {{- include "jupyterhub.matchLabels" $_ | nindent 14 }}
+
+    # proxy -> Kubernetes internal DNS
+    - ports:
+      - protocol: UDP
+        port: 53
+      - protocol: TCP
+        port: 53
+
+    {{- with .Values.proxy.networkPolicy.egress }}
+    # proxy --> default: everything
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
 {{- end }}

jupyterhub/templates/singleuser/netpol.yaml

@@ -13,30 +13,40 @@ spec:
   policyTypes:
     - Ingress
     - Egress
+
   ingress:
-    - from:
+    # allowed pods (hub.jupyter.org/network-access-singleuser) --> singleuser-server
+    - ports:
+        - port: notebook-port
+      from:
         - podSelector:
             matchLabels:
               hub.jupyter.org/network-access-singleuser: "true"
-      ports:
-        - port: 8888
-    {{- /* Useful if you want to give user server access to pods from other namespaces */}}
-    {{- if .Values.singleuser.networkPolicy.ingress }}
-    {{- .Values.singleuser.networkPolicy.ingress | toYaml | trimSuffix "\n" | nindent 4 }}
+
+    {{- with .Values.singleuser.networkPolicy.ingress }}
+    # default: nothing --> singleuser-server
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
+
   egress:
-    - to:
+    # singleuser-server --> hub
+    - ports:
+        - port: 8081
+      to:
         - podSelector:
             matchLabels:
-              {{- /*
-              Override componentLabel because we need the label of the
-              destination, not the source
-              */}}
               {{- $_ := merge (dict "componentLabel" "hub") . }}
               {{- include "jupyterhub.matchLabels" $_ | nindent 14 }}
-      ports:
-        - port: 8081
-    {{- if .Values.singleuser.networkPolicy.egress }}
-    {{- .Values.singleuser.networkPolicy.egress | toYaml | trimSuffix "\n" | nindent 4 }}
+
+    # singleuser-server -> Kubernetes internal DNS
+    - ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+
+    {{- with .Values.singleuser.networkPolicy.egress }}
+    # singleuser-server --> default: everything
+    {{- . | toYaml | trimSuffix "\n" | nindent 4 }}
     {{- end }}
 {{- end }}

jupyterhub/values.yaml

@@ -68,6 +68,12 @@ hub:
   networkPolicy:
     enabled: false
     ingress: []
+    ## egress for JupyterHub already includes Kubernetes internal DNS and
+    ## access to the proxy, but can be restricted further, but ensure to allow
+    ## access to the Kubernetes API server that couldn't be pinned ahead of
+    ## time.
+    ##
+    ## ref: https://stackoverflow.com/a/59016417/2220152
     egress:
       - to:
           - ipBlock:

tests/test_spawn.py

@@ -162,6 +162,24 @@ def test_singleuser_netpol(api_request, jupyter_user, request_data):
         print(server_model)
         pod_name = server_model["state"]["pod_name"]
 
+        c = subprocess.run([
+            "kubectl", "exec", pod_name,
+            "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"],
+            "--context", os.environ["Z2JH_KUBE_CONTEXT"],
+            "--",
+            "nslookup", "hub",
+        ])
+        assert c.returncode == 0, "DNS issue: failed to resolve 'hub' from a singleuser-server"
+
+        c = subprocess.run([
+            "kubectl", "exec", pod_name,
+            "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"],
+            "--context", os.environ["Z2JH_KUBE_CONTEXT"],
+            "--",
+            "nslookup", "jupyter.org",
+        ])
+        assert c.returncode == 0, "DNS issue: failed to resolve 'jupyter.org' from a singleuser-server"
+
         # Must match CIDR in singleuser.networkPolicy.egress.
         allowed_url = "http://jupyter.org"
         blocked_url = "http://mybinder.org"
@@ -173,14 +191,14 @@ def test_singleuser_netpol(api_request, jupyter_user, request_data):
             "--",
             "wget", "--quiet", "--tries=1", "--timeout=3", allowed_url,
         ])
-        assert c.returncode == 0, "Unable to get allowed domain (or failed to resolve the domain name)"
+        assert c.returncode == 0, "Unable to get allowed domain"
 
         c = subprocess.run([
             "kubectl", "exec", pod_name,
             "--namespace", os.environ["Z2JH_KUBE_NAMESPACE"],
             "--context", os.environ["Z2JH_KUBE_CONTEXT"],
             "--",
-            "wget", "--quiet", "--tries=1", "--timeout=3", blocked_url,
+            "wget", "--quiet", "--server-response", "-O-", "--tries=1", "--timeout=3", blocked_url,
         ])
         assert c.returncode > 0, "Blocked domain was allowed"