[DO NOT MERGE] Screenshot test

.github/workflows/ci.yml

@@ -24,174 +24,175 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pre-commit:
-    name: Pre-commit checks
-    uses: beeware/.github/.github/workflows/pre-commit-run.yml@main
-    with:
-      pre-commit-source: "./core[dev]"
+  # pre-commit:
+  #   name: Pre-commit checks
+  #   uses: beeware/.github/.github/workflows/pre-commit-run.yml@main
+  #   with:
+  #     pre-commit-source: "./core[dev]"
 
-  towncrier:
-    name: Check towncrier
-    uses: beeware/.github/.github/workflows/towncrier-run.yml@main
-    with:
-      tox-source: "./core[dev]"
+  # towncrier:
+  #   name: Check towncrier
+  #   uses: beeware/.github/.github/workflows/towncrier-run.yml@main
+  #   with:
+  #     tox-source: "./core[dev]"
 
-  package:
-    name: Python Package
-    uses: beeware/.github/.github/workflows/python-package-create.yml@main
-    with:
-      tox-source: "./core[dev]"
-      build-subdirectory: ${{ matrix.subdir }}
-      distribution-path: "*/dist/*"
-    strategy:
-      matrix:
-        subdir:
-        - "android"
-        - "cocoa"
-        - "core"
-        - "demo"
-        - "dummy"
-        - "gtk"
-        - "iOS"
-        - "toga"
-        - "web"
-        - "winforms"
+  # package:
+  #   name: Python Package
+  #   uses: beeware/.github/.github/workflows/python-package-create.yml@main
+  #   with:
+  #     tox-source: "./core[dev]"
+  #     build-subdirectory: ${{ matrix.subdir }}
+  #     distribution-path: "*/dist/*"
+  #   strategy:
+  #     matrix:
+  #       subdir:
+  #       - "android"
+  #       - "cocoa"
+  #       - "core"
+  #       - "demo"
+  #       - "dummy"
+  #       - "gtk"
+  #       - "iOS"
+  #       - "toga"
+  #       - "web"
+  #       - "winforms"
 
-  core:
-    runs-on: ${{ matrix.platform }}-latest
-    needs: [pre-commit, towncrier, package]
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      matrix:
-        platform: [ "macos", "ubuntu", "windows" ]
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12-dev" ]
-        include:
-          - experimental: false
+  # core:
+  #   runs-on: ${{ matrix.platform }}-latest
+  #   needs: [pre-commit, towncrier, package]
+  #   continue-on-error: ${{ matrix.experimental }}
+  #   strategy:
+  #     matrix:
+  #       platform: [ "macos", "ubuntu", "windows" ]
+  #       python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12-dev" ]
+  #       include:
+  #         - experimental: false
 
-          - python-version: "3.12-dev"
-            experimental: true
-    steps:
-    - uses: actions/checkout@v3.5.3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.6.1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dev dependencies
-      run: |
-        # We don't actually want to install toga-core;
-        # we just want the dev extras so we have a known version of tox and coverage
-        python -m pip install ./core[dev]
-    - name: Get packages
-      uses: actions/download-artifact@v3.0.2
-      with:
-        name: ${{ needs.package.outputs.artifact-name }}
-    - name: Test
-      run: |
-        # The $(ls ...) shell expansion is done in the Github environment;
-        # the value of TOGA_INSTALL_COMMAND will be a literal string,
-        # without any shell expansions to perform
-        TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" tox -e py-core
-        cd core
-        mv .coverage .coverage.${{ matrix.platform }}.${{ matrix.python-version }}
-    - name: Store coverage data
-      uses: actions/upload-artifact@v3.1.2
-      with:
-        name: core-coverage-data
-        path: "core/.coverage.*"
-        if-no-files-found: error
+  #         - python-version: "3.12-dev"
+  #           experimental: true
+  #   steps:
+  #   - uses: actions/checkout@v3.5.3
+  #   - name: Set up Python ${{ matrix.python-version }}
+  #     uses: actions/setup-python@v4.6.1
+  #     with:
+  #       python-version: ${{ matrix.python-version }}
+  #   - name: Install dev dependencies
+  #     run: |
+  #       # We don't actually want to install toga-core;
+  #       # we just want the dev extras so we have a known version of tox and coverage
+  #       python -m pip install ./core[dev]
+  #   - name: Get packages
+  #     uses: actions/download-artifact@v3.0.2
+  #     with:
+  #       name: ${{ needs.package.outputs.artifact-name }}
+  #   - name: Test
+  #     run: |
+  #       # The $(ls ...) shell expansion is done in the Github environment;
+  #       # the value of TOGA_INSTALL_COMMAND will be a literal string,
+  #       # without any shell expansions to perform
+  #       TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl)" tox -e py-core
+  #       cd core
+  #       mv .coverage .coverage.${{ matrix.platform }}.${{ matrix.python-version }}
+  #   - name: Store coverage data
+  #     uses: actions/upload-artifact@v3.1.2
+  #     with:
+  #       name: core-coverage-data
+  #       path: "core/.coverage.*"
+  #       if-no-files-found: error
 
-  core-coverage:
-    name: Combine & check core coverage.
-    runs-on: ubuntu-latest
-    needs: core
-    steps:
-    - uses: actions/checkout@v3.5.3
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@v4.6.1
-      with:
-        # Use latest, so it understands all syntax.
-        python-version: ${{ env.max_python_version }}
-    - name: Install dev dependencies
-      run: |
-        # We don't actually want to install toga-core;
-        # we just want the dev extras so we have a known version of coverage
-        python -m pip install ./core[dev]
-    - name: Retrieve coverage data
-      uses: actions/download-artifact@v3.0.2
-      with:
-        name: core-coverage-data
-        path: core
-    - name: Generate coverage report
-      run: |
-        cd core
-        python -m coverage combine
-        python -m coverage html --skip-covered --skip-empty
-        python -m coverage report --rcfile ../pyproject.toml # --fail-under=100
-    - name: Upload HTML report if check failed.
-      uses: actions/upload-artifact@v3.1.2
-      with:
-        name: html-coverage-report
-        path: core/htmlcov
-      if: ${{ failure() }}
+  # core-coverage:
+  #   name: Combine & check core coverage.
+  #   runs-on: ubuntu-latest
+  #   needs: core
+  #   steps:
+  #   - uses: actions/checkout@v3.5.3
+  #     with:
+  #       fetch-depth: 0
+  #   - uses: actions/setup-python@v4.6.1
+  #     with:
+  #       # Use latest, so it understands all syntax.
+  #       python-version: ${{ env.max_python_version }}
+  #   - name: Install dev dependencies
+  #     run: |
+  #       # We don't actually want to install toga-core;
+  #       # we just want the dev extras so we have a known version of coverage
+  #       python -m pip install ./core[dev]
+  #   - name: Retrieve coverage data
+  #     uses: actions/download-artifact@v3.0.2
+  #     with:
+  #       name: core-coverage-data
+  #       path: core
+  #   - name: Generate coverage report
+  #     run: |
+  #       cd core
+  #       python -m coverage combine
+  #       python -m coverage html --skip-covered --skip-empty
+  #       python -m coverage report --rcfile ../pyproject.toml # --fail-under=100
+  #   - name: Upload HTML report if check failed.
+  #     uses: actions/upload-artifact@v3.1.2
+  #     with:
+  #       name: html-coverage-report
+  #       path: core/htmlcov
+  #     if: ${{ failure() }}
 
-  backend:
-    runs-on: ${{ matrix.runs-on }}
-    needs: [package, core]
-    strategy:
-      matrix:
-        backend: [ "android", "cocoa", "gtk", "iOS", "web", "winforms" ]
-        include:
-          - runs-on: ubuntu-latest
-          - python-version: "3.8"  # Should be env.min_python_version (https://github.com/actions/runner/issues/480)
-          - pre-command:
+  # backend:
+  #   runs-on: ${{ matrix.runs-on }}
+  #   needs: [package, core]
+  #   strategy:
+  #     matrix:
+  #       backend: [ "android", "cocoa", "gtk", "iOS", "web", "winforms" ]
+  #       include:
+  #         - runs-on: ubuntu-latest
+  #         - python-version: "3.8"  # Should be env.min_python_version (https://github.com/actions/runner/issues/480)
+  #         - pre-command:
 
-          - backend: cocoa
-            runs-on: macos-latest
+  #         - backend: cocoa
+  #           runs-on: macos-latest
 
-          - backend: gtk
-            pre-command: |
-              sudo apt update -y
-              sudo apt install -y pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-gtk-3.0
+  #         - backend: gtk
+  #           pre-command: |
+  #             sudo apt update -y
+  #             sudo apt install -y pkg-config python3-dev libgirepository1.0-dev libcairo2-dev gir1.2-gtk-3.0
 
-          - backend: iOS
-            runs-on: macos-latest
+  #         - backend: iOS
+  #           runs-on: macos-latest
 
-          - backend: winforms
-            runs-on: windows-latest
-            # Py3.9 is the first Python version for which
-            # a wheel of pythonnet isn't available on PyPI.
-            python-version: "3.9"
-    steps:
-    - uses: actions/checkout@v3.5.3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.6.1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Get packages
-      uses: actions/download-artifact@v3.0.2
-      with:
-        name: ${{ needs.package.outputs.artifact-name }}
-    - name: Install dev dependencies
-      run: |
-        ${{ matrix.pre-command }}
-        # We don't actually want to install toga-core;
-        # we just want the dev extras so we have a known version of tox
-        python -m pip install ./core[dev]
-    - name: Test
-      run: |
-        # The $(ls ...) shell expansion is done in the Github environment;
-        # the value of TOGA_INSTALL_COMMAND will be a literal string,
-        # without any shell expansions to perform
-        TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl) ../$(ls ${{ matrix.backend }}/dist/toga_${{ matrix.backend }}-*.whl)" tox -e py-${{ matrix.backend }}
+  #         - backend: winforms
+  #           runs-on: windows-latest
+  #           # Py3.9 is the first Python version for which
+  #           # a wheel of pythonnet isn't available on PyPI.
+  #           python-version: "3.9"
+  #   steps:
+  #   - uses: actions/checkout@v3.5.3
+  #   - name: Set up Python ${{ matrix.python-version }}
+  #     uses: actions/setup-python@v4.6.1
+  #     with:
+  #       python-version: ${{ matrix.python-version }}
+  #   - name: Get packages
+  #     uses: actions/download-artifact@v3.0.2
+  #     with:
+  #       name: ${{ needs.package.outputs.artifact-name }}
+  #   - name: Install dev dependencies
+  #     run: |
+  #       ${{ matrix.pre-command }}
+  #       # We don't actually want to install toga-core;
+  #       # we just want the dev extras so we have a known version of tox
+  #       python -m pip install ./core[dev]
+  #   - name: Test
+  #     run: |
+  #       # The $(ls ...) shell expansion is done in the Github environment;
+  #       # the value of TOGA_INSTALL_COMMAND will be a literal string,
+  #       # without any shell expansions to perform
+  #       TOGA_INSTALL_COMMAND="python -m pip install ../$(ls core/dist/toga_core-*.whl)[dev] ../$(ls dummy/dist/toga_dummy-*.whl) ../$(ls ${{ matrix.backend }}/dist/toga_${{ matrix.backend }}-*.whl)" tox -e py-${{ matrix.backend }}
 
   testbed:
     runs-on: ${{ matrix.runs-on }}
-    needs: core
+    # needs: core
     strategy:
       fail-fast: false
       matrix:
-        backend: [ "macOS", "windows", "linux", "android", "iOS" ]
+        # backend: [ "macOS", "windows", "linux", "android", "iOS" ]
+        backend: [ "macOS", "iOS", "linux" ]
         include:
           - pre-command:
             briefcase-run-prefix:
@@ -200,6 +201,7 @@ jobs:
 
           - backend: macOS
             runs-on: macos-12
+            app-user-data-path: /Users/runner/Library/Application Support/org.beeware.toga.testbed
 
           # We use a fixed Ubuntu version rather than `-latest` because at some point,
           # `-latest` will be updated, but it will be a soft changeover, which would cause
@@ -224,17 +226,19 @@ jobs:
 
             briefcase-run-prefix: 'DISPLAY=:99'
             setup-python: false  # Use the system Python packages.
+            app-user-data-path: /home/runner/.local/share/testbed
 
-          - backend: windows
-            runs-on: windows-latest
+          # - backend: windows
+          #   runs-on: windows-latest
 
           - backend: iOS
             runs-on: macos-12
             briefcase-run-args: ' -d "iPhone SE (3rd generation)"'
+            # app data path is determined at runtime, as the path contains the Simulator and app ID.
 
-          - backend: android
-            runs-on: macos-12
-            briefcase-run-args: " -d '{\"avd\":\"beePhone\"}' --Xemulator=-no-window --Xemulator=-no-snapshot --Xemulator=-no-audio --Xemulator=-no-boot-anim --shutdown-on-exit"
+          # - backend: android
+          #   runs-on: macos-12
+          #   briefcase-run-args: " -d '{\"avd\":\"beePhone\"}' --Xemulator=-no-window --Xemulator=-no-snapshot --Xemulator=-no-audio --Xemulator=-no-boot-anim --shutdown-on-exit"
     steps:
     - uses: actions/checkout@v3.5.3
       with:
@@ -263,3 +267,19 @@ jobs:
       with:
         name: testbed-failure-logs-${{ matrix.backend }}
         path: testbed/logs/*
+    - name: Copy app generated user data
+      if: failure()
+      run: |
+        mkdir testbed/app_data
+        if [ "${{ matrix.backend }}" = "iOS" ]; then
+          APP_DATA_PATH="$(xcrun simctl get_app_container booted org.beeware.toga.testbed data)/Documents"
+        else
+          APP_DATA_PATH="${{ matrix.app-user-data-path }}"
+        fi
+        cp -r "$APP_DATA_PATH" testbed/app_data/testbed-app_data-${{ matrix.backend }}
+    - name: Upload app data
+      uses: actions/upload-artifact@v3.1.2
+      if: failure()
+      with:
+        name: testbed-failure-app-data-${{ matrix.backend }}
+        path: testbed/app_data/*