Merge pull request #1837 from consideRatio/pr/user-scheduler-image-lo…

…cality

user-scheduler: let image locality etc matter again

.travis.yml

@@ -58,7 +58,7 @@ script:
   - await_autohttps_tls_cert_acquisition
 
   # Run tests
-  - pytest --verbose --exitfirst ./tests
+  - pytest --verbose --exitfirst ./tests || (full_namespace_report && exit 1)
 
 env:
   ## Global environment variables will act as defaults, jobs will override them
@@ -124,7 +124,7 @@ jobs:
         - await_autohttps_tls_cert_acquisition
 
         # Run tests
-        - pytest --verbose --exitfirst ./tests
+        - pytest --verbose --exitfirst ./tests || (full_namespace_report && exit 1)
 
     ## k3s versions:  https://github.com/rancher/k3s/releases
     - stage: test

ci/common

@@ -156,3 +156,35 @@ setup_kubeval () {
     curl -sfL https://github.com/instrumenta/kubeval/releases/download/0.15.0/kubeval-linux-amd64.tar.gz | tar xz kubeval
     sudo mv kubeval /usr/local/bin/
 }
+
+full_namespace_report () {
+    # list config (secret,configmap)
+    kubectl get secret,cm
+    # list networking (service,ingress)
+    kubectl get svc,ing
+    # list workloads (deployment,statefulset,daemonset,pod)
+    kubectl get deploy,sts,ds,pod
+
+    # if any pod has any non-ready -> show its containers' logs
+    kubectl get pods -o json \
+    | jq '
+        .items[]
+        | select(
+            any(.status.containerStatuses[]?; .ready == false)
+        )
+        | .metadata.name' \
+    | xargs --max-args 1 --no-run-if-empty \
+    sh -c 'printf "\nPod with non-ready container detected\n - Logs of $0:\n"; kubectl logs --all-containers $0'
+
+    # if any pods that should be scheduled by the user-scheduler are pending ->
+    # show user-scheduler's logs
+    (
+        kubectl get pods -l "component in (user-placeholder,singleuser-server)" -o json \
+        | jq -r '
+            .items[]
+            | select(.status.phase == "Pending")
+            | .metadata.name
+        '
+    ) | xargs --max-args 1 --no-run-if-empty --max-lines \
+    sh -c 'printf "\nPending user pod detected ($0)\n - Logs of deploy/user-scheduler:\n"; kubectl logs --all-containers deploy/user-scheduler'
+}

dev-config.yaml

@@ -75,5 +75,13 @@ prePuller:
   hook:
     enabled: false
 
+scheduling:
+  userScheduler:
+    enabled: true
+    # replicas lowered to make it easier to get the relevant logs if a pod get
+    # stuck in pending mode
+    replicas: 1
+    logLevel: 10
+
 debug:
   enabled: true

jupyterhub/values.yaml

@@ -357,15 +357,32 @@ scheduling:
     enabled: true
     replicas: 2
     logLevel: 4
+    # plugins ref: https://kubernetes.io/docs/reference/scheduling/config/#scheduling-plugins-1
     plugins:
       score:
         disabled:
-          - name: ImageLocality
-          - name: NodeResourcesLeastAllocated
-          - name: NodeResourcesBalancedAllocation
           - name: SelectorSpread
+          - name: TaintToleration
+          - name: PodTopologySpread
+          - name: NodeResourcesBalancedAllocation
+          - name: NodeResourcesLeastAllocated
+          # Disable plugins to be allowed to enable them again with a different
+          # weight and avoid an error.
+          - name: NodePreferAvoidPods
+          - name: NodeAffinity
+          - name: InterPodAffinity
+          - name: ImageLocality
         enabled:
+          - name: NodePreferAvoidPods
+            weight: 161051
+          - name: NodeAffinity
+            weight: 14631
+          - name: InterPodAffinity
+            weight: 1331
           - name: NodeResourcesMostAllocated
+            weight: 121
+          - name: ImageLocality
+            weight: 11
     containerSecurityContext:
       runAsUser: 65534  # nobody user
       runAsGroup: 65534 # nobody group

tests/conftest.py

@@ -37,8 +37,8 @@ def request_data():
         "token": token,
         "hub_url": f'{hub_url.rstrip("/")}/hub/api',
         "headers": {"Authorization": f"token {token}"},
-        "test_timeout": 300,
-        "request_timeout": 60,
+        "test_timeout": 30,
+        "request_timeout": 25,
     }