Add performance reports

conftest.py

@@ -10,7 +10,7 @@
 import dask
 import pytest
 import s3fs
-from dask.distributed import Client
+from dask.distributed import Client, performance_report
 from toolz import merge
 
 try:
@@ -43,6 +43,9 @@ def pytest_collection_modifyitems(config, items):
             item.add_marker(skip_latest)
 
 
+UNIQUE_ID = uuid.uuid4().hex[:8]
+
+
 def get_software():
     try:
         return os.environ["COILED_SOFTWARE_NAME"]
@@ -79,7 +82,7 @@ def small_cluster(request):
     )
     module = os.path.basename(request.fspath).split(".")[0]
     with Cluster(
-        name=f"{module}-{uuid.uuid4().hex[:8]}",
+        name=f"{module}-{UNIQUE_ID}",
         n_workers=10,
         worker_memory="8 GiB",
         worker_vm_types=["m5.large"],
@@ -90,12 +93,15 @@ def small_cluster(request):
 
 
 @pytest.fixture
-def small_client(small_cluster):
+def small_client(small_cluster, s3, s3_report_url, request, tmp_path):
     with Client(small_cluster) as client:
         small_cluster.scale(10)
         client.wait_for_workers(10)
         client.restart()
-        yield client
+        local_file = str(tmp_path / "preformance-report.html")
+        with performance_report(local_file):
+            yield client
+        s3.put(local_file, s3_report_url + f"/{request.node.originalname}.html")
 
 
 S3_REGION = "us-east-2"
@@ -126,9 +132,19 @@ def s3_scratch(s3):
     return scratch_url
 
 
+@pytest.fixture(scope="session")
+def s3_report_url(s3, s3_scratch):
+    # Ensure that the test-scratch directory exists,
+    # but do NOT remove it as multiple test runs could be
+    # accessing it at the same time
+    report_url = f"{s3_scratch}/performance-reports/{UNIQUE_ID}"
+    s3.mkdirs(report_url, exist_ok=True)
+    return report_url
+
+
 @pytest.fixture(scope="function")
 def s3_url(s3, s3_scratch, request):
-    url = f"{s3_scratch}/{request.node.originalname}-{uuid.uuid4().hex}"
+    url = f"{s3_scratch}/{request.node.originalname}-{UNIQUE_ID}"
     s3.mkdirs(url, exist_ok=False)
     yield url
     s3.rm(url, recursive=True)