Add Google Cloud support with GCPCluster (#131)

* Add VMCluster class and Digital Ocean implementation * Fix worker startup command * spec work on gcp provider * updates * checkpoint * Working state * Refactoring * working setup * Use correct images * Add more config options, docs and a RAPIDS example * Use correct images * Add more config options, docs and a RAPIDS example * clean up * Add Packer support * fix importerror * credential handling * better messaging * Add autoshutdown to EC2Cluster * more updates * add worker args * more updates * Fix init order and test sync cluster * Skip tests with missing deps * Flake8 * Start on docs refactor * More docs refactor * More documentation * Add flake8 to precommit hooks * Fix linting * Black * update yaml spec and add defaults * move default values to config * Change worker command to module, add worker/scheduler options and refactor * Refactor mixins and siplify code * Refactor EC2Cluster * Change black version * Run black 20.8b1 * Print black version * Force black to 20.8b1 * Shuffle AzureML docs from merge to new location * Add GPU docs * swtich to worker_class * Add package import * fix imports and update google dependecy requirements * Migrate to GitHub Actions (#155) * Migrate to GitHub Actions * Fix environment file name * Quiet conda * Success message * Exclude versioneer from flake8 * add pytest-asycnio/timeout to ci env * Skip sync test without credentials * Test imports * Skip all auth failures and fix linting * Slight refactors, start docs and add non RAPIDS tests * Black * Fix pytest-asyncio versions * Add custom exception for missing credentials * Add missing skip without credentials call * Add docs and fix tests * Remove fixture * Fix get cloud init test Co-authored-by: Jacob Tomlinson <jtomlinson@nvidia.com> Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
dask · Nov 6, 2020 · ac35f9e · ac35f9e
1 parent 9f3fc1d
commit ac35f9e
Show file tree

Hide file tree

Showing 14 changed files with 791 additions and 26 deletions.
diff --git a/ci/environment-3.7.yml b/ci/environment-3.7.yml
@@ -27,6 +27,9 @@ dependencies:
   - tblib
   - tornado >=5
   - zict >=0.1.3
+  # `event_loop_policy` change See https://github.com/dask/distributed/pull/4212
+  - pytest-asyncio >=0.14.0
+  - pytest-timeout
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/distributed
diff --git a/ci/environment-3.8.yml b/ci/environment-3.8.yml
@@ -27,6 +27,9 @@ dependencies:
   - tblib
   - tornado >=5
   - zict >=0.1.3
+  # `event_loop_policy` change See https://github.com/dask/distributed/pull/4212
+  - pytest-asyncio >=0.14.0
+  - pytest-timeout
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/distributed
diff --git a/ci/scripts/test_imports.sh b/ci/scripts/test_imports.sh
@@ -18,3 +18,4 @@ test_import () {
 test_import "aws"               "import dask_cloudprovider.aws"
 test_import "azure"             "import dask_cloudprovider.azure"
 test_import "digitalocean"      "import dask_cloudprovider.digitalocean"
+test_import "gcp"               "import dask_cloudprovider.gcp"
diff --git a/dask_cloudprovider/aws/ec2.py b/dask_cloudprovider/aws/ec2.py
@@ -51,7 +51,7 @@ def __init__(
         self.region = region
         self.bootstrap = bootstrap
         self.ami = ami
-        self.docker_image = docker_image
+        self.docker_image = docker_image or self.config.get("docker_image")
         self.instance_type = instance_type
         self.gpu_instance = gpu_instance
         self.vpc = vpc
@@ -311,6 +311,7 @@ def __init__(
         subnet_id=None,
         security_groups=None,
         filesystem_size=None,
+        docker_image=None,
         **kwargs,
     ):
         self.boto_session = aiobotocore.get_session()
@@ -354,7 +355,7 @@ def __init__(
             "region": self.region,
             "bootstrap": self.bootstrap,
             "ami": self.ami,
-            "docker_image": self.docker_image,
+            "docker_image": docker_image or self.config.get("docker_image"),
             "instance_type": self.instance_type,
             "gpu_instance": self.gpu_instance,
             "vpc": self.vpc,

diff --git a/dask_cloudprovider/cloudprovider.yaml b/dask_cloudprovider/cloudprovider.yaml
@@ -41,6 +41,7 @@ cloudprovider:
     # worker_command: "dask-worker" # The command for workers to run. If the instance_type is a GPU instance dask-cuda-worker will be used.
     # ami: "" # AMI ID to use for all instances. Defaults to latest Ubuntu 20.04 image.
     instance_type: "t2.micro" # Instance type for all workers
+    docker_image: "daskdev/dask:latest" # docker image to use
     # vpc: "" # VPC id for instances to join. Defaults to default VPC.
     # subnet_id: "" # Subnet ID for instances to. Defaults to all subnets in default VPC.
     # security_groups: [] # Security groups for instances. Will create a minimal Dask security group by default.
@@ -66,3 +67,14 @@ cloudprovider:
     region: "nyc3" # Region to launch Droplets in
     size: "s-1vcpu-1gb" # Droplet size to launch, default is 1GB RAM, 1 vCPU
     image: "ubuntu-20-04-x64" # Operating System image to use
+
+  gcp:
+    source_image: "projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20201014" # the gcp image to use for all instances
+    zone: "us-east1-c" # the zone of of where to launch the instances
+    projectid: "" # name of the google cloud project
+    machine_type: "n1-standard-1" # size of the machine type to use
+    filesystem_size: 50 # amount in GBs of hard drive space to allocate
+    ngpus: "" # number of GPUs to use
+    gpu_type: "" # type of gpus to use: nvidia-tesla-k80, nvidia-tesla-p100, nvidia-tesla-t4
+    docker_image: "daskdev/dask:latest" # docker image to use
+    auto_shutdown: true # Shutdown instances automatically if the scheduler or worker services time out.
diff --git a/dask_cloudprovider/gcp/__init__.py b/dask_cloudprovider/gcp/__init__.py
@@ -0,0 +1 @@
+from .instances import GCPCluster