Merge pull request #495 from pangeo-data/staging

staging -> prod (#477, #478, #480, #481, #482, #484, #486, #493, #494)
pangeo-data · Jan 10, 2020 · 74d01ab · 74d01ab
2 parents 8c824b6 + b52b559
commit 74d01ab
Show file tree

Hide file tree

Showing 20 changed files with 68 additions and 58 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -55,6 +55,7 @@ jobs:
       - run:
           name: Test building ocean.pangeo.io image if needed
           when: always
+          no_output_timeout: 60m
           command: |
             hubploy build ocean --commit-range ${COMMIT_RANGE}
 
@@ -79,7 +80,7 @@ jobs:
       #       hubploy build icesat2 --commit-range ${COMMIT_RANGE}
       #     environment:
       #       PYTHONIOENCODING: utf-8
-      
+
       # - run:
       #     name: Test building esip.pangeo.io image if needed
       #     when: always
@@ -173,6 +174,7 @@ jobs:
       - run:
           name: Build ocean.pangeo.io image if needed
           when: always
+          no_output_timeout: 60m
           command: |
             hubploy build ocean --check-registry --push
 
@@ -197,7 +199,7 @@ jobs:
       #       hubploy build icesat2 --check-registry --push
       #     environment:
       #       PYTHONIOENCODING: utf-8
-      
+
       # - run:
       #     name: Build esip.pangeo.io image if needed
       #     when: always
@@ -226,6 +228,7 @@ jobs:
             helm init --client-only
             helm repo add pangeo https://pangeo-data.github.io/helm-chart/
             helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
+            helm repo add dask-gateway https://dask.org/dask-gateway-helm-repo/
             helm repo update
 
       - run:
@@ -257,7 +260,7 @@ jobs:
       #     when: always
       #     command: |
       #       hubploy deploy icesat2 pangeo-deploy ${CIRCLE_BRANCH}
-      
+
       # - run:
       #     name: Deploy esip.pangeo.io
       #     when: always
@@ -274,7 +277,7 @@ jobs:
             BODY="Deployment to ooi-${CIRCLE_BRANCH} failed on `date -u +'%D at %T UTC.'`"
             errormsg() { ./.circleci/send-email "${EMAIL}" "${SUBJECT}" "${BODY}"; }
             trap errormsg ERR
-            hubploy deploy ooi pangeo-deploy ${CIRCLE_BRANCH}
+            hubploy deploy ooi pangeo-deploy ${CIRCLE_BRANCH} --timeout 1200
 
 workflows:
   version: 2

diff --git a/dask-values.yaml b/dask-values.yaml
diff --git a/deployments/dev/config/common.yaml b/deployments/dev/config/common.yaml
@@ -32,6 +32,10 @@ pangeo:
         limit: 14G
         guarantee: 4G
     hub:
+      services:
+        dask-gateway:
+          # This makes the gateway available at ${HUB_URL}/services/dask-gateway
+          url: http://web-public-dev-staging-dask-gateway
       resources:
         requests:
           cpu: "0.25"

diff --git a/deployments/dev/image/binder/Dockerfile b/deployments/dev/image/binder/Dockerfile
@@ -1,2 +1,2 @@
 # Note that there must be a tag
-FROM pangeo/pangeo-notebook-onbuild:2019.09.23
+FROM pangeo/pangeo-notebook-onbuild:2019.12.06
diff --git a/deployments/dev/image/binder/dask_config.yaml b/deployments/dev/image/binder/dask_config.yaml
@@ -43,3 +43,9 @@ labextension:
     class: KubeCluster
     args: []
     kwargs: {}
+
+gateway:
+  address: http://34.68.195.134
+  proxy-address: tls://35.225.202.35:8786
+  auth:
+    type: jupyterhub
diff --git a/deployments/dev/secrets/prod.yaml b/deployments/dev/secrets/prod.yaml
diff --git a/deployments/dev/secrets/staging.yaml b/deployments/dev/secrets/staging.yaml
diff --git a/deployments/hydro/secrets/prod.yaml b/deployments/hydro/secrets/prod.yaml
diff --git a/deployments/hydro/secrets/staging.yaml b/deployments/hydro/secrets/staging.yaml
diff --git a/deployments/ocean/config/common.yaml b/deployments/ocean/config/common.yaml
@@ -39,18 +39,17 @@ pangeo:
         profile_list: |
           c.KubeSpawner.profile_list = [
             {
-                'display_name': 'standard ** Research Computing Students **',
+                'display_name': 'Small (1 GB - 4 GB)',
                 'default': 'True',
                 'kubespawner_override': {
-                    'image': 'us.gcr.io/pangeo-181919/ocean-pangeo-io-notebook:634578d',
                     'cpu_limit': 2,
                     'cpu_guarantee': 0.3,
                     'mem_limit': '4G',
                     'mem_guarantee': '1G',
                 }
             },
             {
-                'display_name': '** CMIP6 Hackathon Participants ** (up to 2 cores, 8GB)',
+                'display_name': 'Medium (4 GB - 8G GB)',
                 'kubespawner_override': {
                     'cpu_limit': 2,
                     'cpu_guarantee': 1,
@@ -59,12 +58,12 @@ pangeo:
                 }
             },
             {
-                'display_name': 'large (up to 4 cores, 10GB)',
+                'display_name': 'Large (12 GB - 16 GB)',
                 'kubespawner_override': {
                     'cpu_limit': 4,
                     'cpu_guarantee': 1,
-                    'mem_limit': '10G',
-                    'mem_guarantee': '4G',
+                    'mem_limit': '16G',
+                    'mem_guarantee': '12G',
                 }
             },
           ]

diff --git a/deployments/ocean/image/binder/Dockerfile b/deployments/ocean/image/binder/Dockerfile
@@ -1,2 +1,2 @@
 # Note that there must be a tag
-FROM pangeo/pangeo-notebook-onbuild:2019.11.15
+FROM pangeo/pangeo-notebook-onbuild:2019.12.08
diff --git a/deployments/ocean/secrets/prod.yaml b/deployments/ocean/secrets/prod.yaml
diff --git a/deployments/ocean/secrets/staging.yaml b/deployments/ocean/secrets/staging.yaml
diff --git a/deployments/ooi/config/prod.yaml b/deployments/ooi/config/prod.yaml
@@ -4,6 +4,13 @@ pangeo:
       userPlaceholder:
         enabled: true
         replicas: 4
+        resources:
+          requests:
+            cpu: 1.6
+            memory: 5G
+          limits:
+            cpu: 3.2
+            memory: 10G
     proxy:
       https:
         enabled: true

diff --git a/deployments/ooi/image/binder/Dockerfile b/deployments/ooi/image/binder/Dockerfile
@@ -1,5 +1,5 @@
 # Note that there must be a tag
-FROM pangeo/pangeo-notebook-onbuild:2019.10.15
+FROM pangeo/pangeo-notebook-onbuild:latest
 
 # Add NB_USER to sudo
 #USER root

diff --git a/deployments/ooi/image/binder/environment.yml b/deployments/ooi/image/binder/environment.yml
@@ -9,4 +9,3 @@ dependencies:
     - azure
     - git+https://github.com/ooicloud/ooi-opendata.git@bad61bf649ae5b189ff14fae06b567b25a8b2e47
     - pycamhd
-    - git+https://github.com/pydata/xarray.git@0ef9aa3abae55833e4431d690bc55c5b5a44911b
diff --git a/deployments/ooi/secrets/staging.yaml b/deployments/ooi/secrets/staging.yaml
diff --git a/pangeo-deploy/requirements.yaml b/pangeo-deploy/requirements.yaml
@@ -1,6 +1,6 @@
 dependencies:
 - name: pangeo
-  version: "19.09.27-86dd66c"
+  version: "19.12.04-ead1e6a"
   repository: https://pangeo-data.github.io/helm-chart/
   import-values:
     - child: rbac

diff --git a/pangeo-deploy/values.yaml b/pangeo-deploy/values.yaml
@@ -37,7 +37,41 @@ pangeo:
         enabled: true
       userPlaceholder:
         enabled: false
+  gateway:
+    clusterManager:
+      clusterStartTimeout: 600
+      workerStartTimeout: 600
+      worker:
+        extraPodConfig:
+          tolerations:
+            - key: "k8s.dask.org_dedicated"
+              operator: "Equal"
+              value: "worker"
+      scheduler:
+        extraPodConfig:
+          tolerations:
+            - key: "k8s.dask.org_dedicated"
+              operator: "Equal"
+              value: "worker"
+    extraConfig: |
+      from dask_gateway_server.options import Options, Integer, Float, String
+      def option_handler(options):
+          if ":" not in options.image:
+              raise ValueError("When specifying an image you must also provide a tag")
+          return {
+              "worker_cores_limit": options.worker_cores,
+              "worker_cores": min(options.worker_cores / 2, 1),
+              "worker_memory": "%fG" % options.worker_memory,
+              "image": options.image,
+          }
+      c.DaskGateway.cluster_manager_options = Options(
+          Integer("worker_cores", 2, min=1, max=4, label="Worker Cores"),
+          Float("worker_memory", 4, min=1, max=8, label="Worker Memory (GiB)"),
+          String("image", default="daskgateway/dask-gateway:0.4.1", label="Image"),
+          handler=option_handler,
+      )
 
 homeDirectories:
   nfs:
     enabled: false
+
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
-hubploy==0.1.0
+git+https://github.com/yuvipanda/hubploy
 jupyter-repo2docker>=0.10.0
 pygithub
 awscli