Merge remote-tracking branch 'upstream/master'

* upstream/master: (43 commits)
  Add unknown pytest markers (dask#2764)
  Delay lookup of allowed failures. (dask#2761)
  Change address -> worker in ColumnDataSource for nbytes plot (dask#2755)
  Remove module state in Prometheus Handlers (dask#2760)
  Add stress test for UCX (dask#2759)
  Add nanny logs (dask#2744)
  Move some of the adaptive logic into the scheduler (dask#2735)
  Add SpecCluster.new_worker_spec method (dask#2751)
  Worker dashboard fixes (dask#2747)
  Add async context managers to scheduler/worker classes (dask#2745)
  Fix the resource key representation before sending graphs (dask#2716) (dask#2733)
  Allow user to configure whether workers are daemon. (dask#2739)
  Pin pytest >=4 with pip in appveyor and python 3.5 (dask#2737)
  Add Experimental UCX Comm (dask#2591)
  Close nannies gracefully (dask#2731)
  add kwargs to progressbars (dask#2638)
  Add back LocalCluster.__repr__. (dask#2732)
  Move bokeh module to dashboard (dask#2724)
  Close clusters at exit (dask#2730)
  Add SchedulerPlugin TaskState example (dask#2622)
  ...

.gitignore

@@ -13,3 +13,6 @@ continuous_integration/hdfs-initialized
 .pytest_cache/
 dask-worker-space/
 .vscode/
+*.swp
+.ycm_extra_conf.py
+tags

README.rst

@@ -3,4 +3,5 @@ Distributed
 
 A library for distributed computation.  See documentation_ for more details.
 
+
 .. _documentation: https://distributed.readthedocs.io/en/latest

continuous_integration/setup_conda_environment.cmd

@@ -50,7 +50,7 @@ call activate %CONDA_ENV%
 %PIP_INSTALL% git+https://github.com/joblib/joblib.git --upgrade
 %PIP_INSTALL% git+https://github.com/dask/zict --upgrade
 
-%PIP_INSTALL% pytest-repeat pytest-timeout pytest-faulthandler sortedcollections
+%PIP_INSTALL% "pytest>=4" pytest-repeat pytest-timeout pytest-faulthandler sortedcollections pytest-asyncio
 
 @rem Display final environment (for reproducing)
 %CONDA% list

continuous_integration/travis/install.sh

@@ -44,7 +44,7 @@ conda install -q \
     paramiko \
     prometheus_client \
     psutil \
-    pytest \
+    pytest>=4 \
     pytest-timeout \
     python=$PYTHON \
     requests \
@@ -59,7 +59,7 @@ conda install -q \
 conda install -c defaults -c conda-forge libunwind
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
-pip install -q pytest-repeat pytest-faulthandler
+pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
 
 pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps

dev-requirements.txt

@@ -10,3 +10,4 @@ jupyter_client >= 4.4.0
 ipykernel >= 4.5.2
 pytest >= 3.0.5
 prometheus_client >= 0.6.0
+jupyter-server-proxy >= 1.1.0

distributed/__init__.py

@@ -4,7 +4,7 @@
 from dask.config import config
 from .actor import Actor, ActorFuture
 from .core import connect, rpc
-from .deploy import LocalCluster, Adaptive
+from .deploy import LocalCluster, Adaptive, SpecCluster
 from .diagnostics import progress
 from .client import (
     Client,

distributed/bokeh.py

@@ -0,0 +1 @@
+raise ImportError("The distributed.bokeh module has moved to distributed.dashboard")

distributed/cli/dask_mpi.py

@@ -7,7 +7,7 @@
 from warnings import warn
 
 from distributed import Scheduler, Nanny, Worker
-from distributed.bokeh.worker import BokehWorker
+from distributed.dashboard import BokehWorker
 from distributed.cli.utils import check_python_3
 from distributed.comm.addressing import uri_from_host_port
 from distributed.utils import get_ip_interface
@@ -82,12 +82,12 @@ def main(
 
     if rank == 0 and scheduler:
         try:
-            from distributed.bokeh.scheduler import BokehScheduler
+            from distributed.dashboard import BokehScheduler
         except ImportError:
             services = {}
         else:
             services = {
-                ("bokeh", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
+                ("dashboard", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
             }
         scheduler = Scheduler(
             scheduler_file=scheduler_file, loop=loop, services=services
@@ -107,7 +107,7 @@ def main(
             name=rank if scheduler else None,
             ncores=nthreads,
             local_dir=local_directory,
-            services={("bokeh", bokeh_worker_port): BokehWorker},
+            services={("dashboard", bokeh_worker_port): BokehWorker},
             memory_limit=memory_limit,
         )
         addr = uri_from_host_port(host, None, 0)

distributed/cli/dask_scheduler.py

@@ -39,6 +39,9 @@
     default=None,
     help="Preferred network interface like 'eth0' or 'ib0'",
 )
+@click.option(
+    "--protocol", type=str, default=None, help="Protocol like tcp, tls, or ucx"
+)
 @click.option(
     "--tls-ca-file",
     type=pem_file_option_type,
@@ -68,27 +71,23 @@
     help="Address on which to listen for diagnostics dashboard",
 )
 @click.option(
-    "--bokeh/--no-bokeh",
-    "_bokeh",
+    "--dashboard/--no-dashboard",
+    "dashboard",
     default=True,
     show_default=True,
     required=False,
-    help="Launch Bokeh Web UI",
+    help="Launch the Dashboard",
 )
 @click.option("--show/--no-show", default=False, help="Show web UI")
 @click.option(
-    "--bokeh-whitelist",
-    default=None,
-    multiple=True,
-    help="IP addresses to whitelist for bokeh.",
+    "--dashboard-prefix", type=str, default=None, help="Prefix for the dashboard app"
 )
-@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
 @click.option(
     "--use-xheaders",
     type=bool,
     default=False,
     show_default=True,
-    help="User xheaders in bokeh app for ssl termination in header",
+    help="User xheaders in dashboard app for ssl termination in header",
 )
 @click.option("--pid-file", type=str, default="", help="File to write the process PID")
 @click.option(
@@ -119,13 +118,13 @@ def main(
     port,
     bokeh_port,
     show,
-    _bokeh,
-    bokeh_whitelist,
-    bokeh_prefix,
+    dashboard,
+    dashboard_prefix,
     use_xheaders,
     pid_file,
     scheduler_file,
     interface,
+    protocol,
     local_directory,
     preload,
     preload_argv,
@@ -195,8 +194,9 @@ def del_pid_file():
         host=host,
         port=port,
         interface=interface,
-        dashboard_address=dashboard_address if _bokeh else None,
-        service_kwargs={"bokeh": {"prefix": bokeh_prefix}},
+        protocol=protocol,
+        dashboard_address=dashboard_address if dashboard else None,
+        service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
     )
     scheduler.start()
     if not preload:

distributed/cli/dask_worker.py

@@ -70,12 +70,12 @@
     help="Address on which to listen for diagnostics dashboard",
 )
 @click.option(
-    "--bokeh/--no-bokeh",
-    "bokeh",
+    "--dashboard/--no-dashboard",
+    "dashboard",
     default=True,
     show_default=True,
     required=False,
-    help="Launch Bokeh Web UI",
+    help="Launch the Dashboard",
 )
 @click.option(
     "--listen-address",
@@ -104,6 +104,9 @@
 @click.option(
     "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
 )
+@click.option(
+    "--protocol", type=str, default=None, help="Protocol like tcp, tls, or ucx"
+)
 @click.option("--nthreads", type=int, default=0, help="Number of threads per process.")
 @click.option(
     "--nprocs",
@@ -114,7 +117,7 @@
 @click.option(
     "--name",
     type=str,
-    default="",
+    default=None,
     help="A unique name for this worker like 'worker-1'. "
     "If used with --nprocs then the process number "
     "will be appended like name-0, name-1, name-2, ...",
@@ -163,7 +166,9 @@
     default=None,
     help="Seconds to wait for a scheduler before closing",
 )
-@click.option("--bokeh-prefix", type=str, default="", help="Prefix for the bokeh app")
+@click.option(
+    "--dashboard-prefix", type=str, default="", help="Prefix for the dashboard"
+)
 @click.option(
     "--preload",
     type=str,
@@ -190,15 +195,16 @@ def main(
     pid_file,
     reconnect,
     resources,
-    bokeh,
+    dashboard,
     bokeh_port,
     local_directory,
     scheduler_file,
     interface,
+    protocol,
     death_timeout,
     preload,
     preload_argv,
-    bokeh_prefix,
+    dashboard_prefix,
     tls_ca_file,
     tls_cert,
     tls_key,
@@ -336,10 +342,11 @@ def del_pid_file():
             security=sec,
             contact_address=contact_address,
             interface=interface,
+            protocol=protocol,
             host=host,
             port=port,
-            dashboard_address=dashboard_address if bokeh else None,
-            service_kwargs={"bokhe": {"prefix": bokeh_prefix}},
+            dashboard_address=dashboard_address if dashboard else None,
+            service_kwargs={"bokhe": {"prefix": dashboard_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
             **kwargs
         )