Enable capture of test failure screenshots.

beeware · Jul 16, 2023 · d15ed48 · d15ed48
1 parent de81c39
commit d15ed48
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 12 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -200,6 +200,7 @@ jobs:
 
           - backend: macOS
             runs-on: macos-12
+            app-user-data-path: /Users/runner/Library/Application Support/org.beeware.toga.testbed
 
           # We use a fixed Ubuntu version rather than `-latest` because at some point,
           # `-latest` will be updated, but it will be a soft changeover, which would cause
@@ -224,13 +225,15 @@ jobs:
 
             briefcase-run-prefix: 'DISPLAY=:99'
             setup-python: false  # Use the system Python packages.
+            app-user-data-path: /home/runner/.local/share/testbed
 
           - backend: windows
             runs-on: windows-latest
 
           - backend: iOS
             runs-on: macos-12
             briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
+            # app data path is determined at runtime, as the path contains the Simulator and app ID.
 
           - backend: android
             runs-on: macos-12
@@ -263,3 +266,19 @@ jobs:
       with:
         name: testbed-failure-logs-${{ matrix.backend }}
         path: testbed/logs/*
+    - name: Copy app generated user data
+      if: failure()
+      run: |
+        mkdir testbed/app_data
+        if [ "${{ matrix.backend }}" = "iOS" ]; then
+          APP_DATA_PATH="$(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents"
+        else
+          APP_DATA_PATH="${{ matrix.app-user-data-path }}"
+        fi
+        cp -r "$APP_DATA_PATH" testbed/app_data/testbed-app_data-${{ matrix.backend }}
+    - name: Upload app data
+      uses: actions/upload-artifact@v3.1.2
+      if: failure()
+      with:
+        name: testbed-failure-app-data-${{ matrix.backend }}
+        path: testbed/app_data/*
diff --git a/testbed/tests/widgets/test_canvas.py b/testbed/tests/widgets/test_canvas.py
@@ -246,6 +246,12 @@ def assert_reference(probe, reference, threshold=0.0):
     if not path.exists():
         path = toga.App.app.paths.app / "resources" / "canvas" / f"{reference}.png"
 
+    save_path = (
+        toga.App.app.paths.data
+        / "canvas"
+        / f"{reference}-{toga.platform.current_platform}.png"
+    )
+
     # If a reference image exists, scale the image to the same size as the reference,
     # and do an MSE comparison on every pixel in 0-1 RGBA colorspace.
     if path.exists():
@@ -264,20 +270,14 @@ def assert_reference(probe, reference, threshold=0.0):
         rmse = math.sqrt(total / (reference_image.size[0] * reference_image.size[1]))
         # If the delta exceeds threshold, save the test image and fail the test.
         if rmse > threshold:
-            scaled_image.save(
-                toga.App.app.paths.app
-                / "resources"
-                / "canvas"
-                / f"test-{reference}-{toga.platform.current_platform}.png"
-            )
+            print(f"Saving {save_path}")
+            save_path.parent.mkdir(parents=True, exist_ok=True)
+            scaled_image.save(save_path)
             assert pytest.fail(f"Rendered image doesn't match reference (RMSE=={rmse})")
     else:
-        scaled_image.save(
-            toga.App.app.paths.app
-            / "resources"
-            / "canvas"
-            / f"test-{reference}-{toga.platform.current_platform}.png"
-        )
+        print(f"Saving {save_path}")
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        scaled_image.save(save_path)
         assert pytest.fail(f"Couldn't find {reference!r} reference image")