GCS network project id for shared VPCs (#317)

dask_cloudprovider/cloudprovider.yaml

@@ -83,8 +83,9 @@ cloudprovider:
 
   gcp:
     source_image: "projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20201014" # the gcp image to use for all instances
-    zone: "us-east1-c" # the zone of of where to launch the instances
+    zone: "us-east1-c" # the zone of where to launch the instances
     network: "default" # the network/subnetwork in GCP to use
+    network_projectid: null # GCP project id where the network exists
     projectid: "" # name of the google cloud project
     on_host_maintenance: "TERMINATE"
     machine_type: "n1-standard-1" # size of the machine type to use

dask_cloudprovider/gcp/instances.py

@@ -57,6 +57,7 @@ def __init__(
         source_image=None,
         docker_image=None,
         network=None,
+        network_projectid=None,
         env_vars=None,
         ngpus=None,
         gpu_type=None,
@@ -87,6 +88,9 @@ def __init__(
         self.disk_type = disk_type or self.config.get("disk_type")
         self.ngpus = ngpus or self.config.get("ngpus")
         self.network = network or self.config.get("network")
+        self.network_projectid = (
+            network_projectid if network_projectid is not None else projectid
+        )
         self.gpu_type = gpu_type or self.config.get("gpu_type")
         self.gpu_instance = gpu_instance
         self.bootstrap = bootstrap
@@ -96,6 +100,7 @@ def __init__(
         self.general_zone = "-".join(self.zone.split("-")[:2])  # us-east1-c -> us-east1
 
     def create_gcp_config(self):
+        subnetwork = f"projects/{self.network_projectid}/regions/{self.general_zone}/subnetworks/{self.network}"
         config = {
             "name": self.name,
             "machineType": f"zones/{self.zone}/machineTypes/{self.machine_type}",
@@ -124,7 +129,7 @@ def create_gcp_config(self):
             "networkInterfaces": [
                 {
                     "kind": "compute#networkInterface",
-                    "subnetwork": f"projects/{self.projectid}/regions/{self.general_zone}/subnetworks/{self.network}",
+                    "subnetwork": subnetwork,
                     "aliasIpRanges": [],
                 }
             ],
@@ -289,7 +294,7 @@ async def start_scheduler(self):
             f"\n  Source Image: {self.source_image} "
             f"\n  Docker Image: {self.docker_image} "
             f"\n  Machine Type: {self.machine_type} "
-            f"\n  Filesytsem Size: {self.filesystem_size} "
+            f"\n  Filesystem Size: {self.filesystem_size} "
             f"\n  Disk Type: {self.disk_type} "
             f"\n  N-GPU Type: {self.ngpus} {self.gpu_type}"
             f"\n  Zone: {self.zone} "
@@ -397,6 +402,9 @@ class GCPCluster(VMCluster):
             - ingress 10.0.0.0/8 on all ports for internal communication of workers
             - ingress 0.0.0.0/0 on 8786-8787 for external accessibility of the dashboard/scheduler
             - (optional) ingress 0.0.0.0./0 on 22 for ssh access
+    network_projectid: str
+        The project id of the GCP network. This defaults to the projectid. There may
+        be cases (i.e. Shared VPC) when network configurations from a different GCP project are used.
     machine_type: str
         The VM machine_type. You can get a full list with ``gcloud compute machine-types list``.
         The default is ``n1-standard-1`` which is 3.75GB RAM and 1 vCPU
@@ -544,6 +552,7 @@ def __init__(
         projectid=None,
         zone=None,
         network=None,
+        network_projectid=None,
         machine_type=None,
         on_host_maintenance=None,
         source_image=None,
@@ -589,6 +598,8 @@ def __init__(
             "machine_type": self.machine_type,
             "ngpus": ngpus or self.config.get("ngpus"),
             "network": network or self.config.get("network"),
+            "network_projectid": network_projectid
+            or self.config.get("network_projectid"),
             "gpu_type": gpu_type or self.config.get("gpu_type"),
             "gpu_instance": self.gpu_instance,
             "bootstrap": self.bootstrap,

doc/source/security.rst

@@ -12,7 +12,12 @@ This makes things quick and easy for new users to get up and running, but may po
 
 Many organisations have policies which do not allow users to assign public IP addresses or open ports. Our best practices
 advice is to use Dask Cloudprovider from within a cloud platform, either from a VM or a managed environment. Then disable public
-networking.
+networking. For example: 
+
+.. code-block:: python
+
+    >>> import dask.config, dask_cloudprovider
+    >>> dask.config.set({"cloudprovider.gcp.public_ingress": False})
 
 See each cluster manager for configuration options.
 
@@ -44,4 +49,4 @@ You can also specify your own certificates using the :class:`distributed.securit
     <Client: 'tls://10.142.0.29:8786' processes=0 threads=0, memory=0 B>
 
 You can disable secure connections by setting the ``security`` keyword argument to ``False``. This may be desirable when troubleshooting or
-when running on a trusted network (entirely inside a VPC for example).
+when running on a trusted network (entirely inside a VPC for example).