Add windows to test workflows (#1809)

* add windows to test runners * disable fail-fast for debugging * re-enable login shell for conda workflow also fix expression to exclude windows from run tests * enable fail-fast again * fix condition, pin runner verisons * remove feature branch from push trigger since already being triggered now via PR * use gfpgan from pypi for windows curious if this would fix the installation here as well since worked for #1802 * unpin basicsr for windows * for curiosity enabling testing for windows as well * disable pip cache since windows failed with a memory error now but was working before it had a cache * use matrix.github-env * set platform specific root and outdir * disable tests for windows since memory error I guess the windows installation uses more space than linux and for this they have less swap memory
invoke-ai · Dec 9, 2022 · ed9186b · ed9186b
1 parent d2026d0
commit ed9186b
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 47 deletions.
diff --git a/.github/workflows/test-invoke-conda.yml b/.github/workflows/test-invoke-conda.yml
@@ -4,7 +4,6 @@ on:
     branches:
       - 'main'
       - 'development'
-      - 'fix-gh-actions-fork'
   pull_request:
     branches:
       - 'main'
@@ -20,16 +19,28 @@ jobs:
           - environment-lin-amd.yml
           - environment-lin-cuda.yml
           - environment-mac.yml
+          - environment-win-cuda.yml
         include:
           - environment-yaml: environment-lin-amd.yml
-            os: ubuntu-latest
+            os: ubuntu-22.04
+            curl-command: curl
+            github-env: $GITHUB_ENV
             default-shell: bash -l {0}
           - environment-yaml: environment-lin-cuda.yml
-            os: ubuntu-latest
+            os: ubuntu-22.04
+            curl-command: curl
+            github-env: $GITHUB_ENV
             default-shell: bash -l {0}
           - environment-yaml: environment-mac.yml
             os: macos-12
+            curl-command: curl
+            github-env: $GITHUB_ENV
             default-shell: bash -l {0}
+          - environment-yaml: environment-win-cuda.yml
+            os: windows-2022
+            curl-command: curl.exe
+            github-env: $env:GITHUB_ENV
+            default-shell: pwsh
           - stable-diffusion-model: stable-diffusion-1.5
             stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
             stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
@@ -72,15 +83,15 @@ jobs:
 
       - name: set test prompt to main branch validation
         if: ${{ github.ref == 'refs/heads/main' }}
-        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> ${{ matrix.github-env }}
 
       - name: set test prompt to development branch validation
         if: ${{ github.ref == 'refs/heads/development' }}
-        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> ${{ matrix.github-env }}
 
       - name: set test prompt to Pull Request validation
         if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
-        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
 
       - name: Use Cached Stable Diffusion Model
         id: cache-sd-model
@@ -96,10 +107,7 @@ jobs:
         if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
         run: |
           mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          curl \
-            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
-            -L ${{ matrix.stable-diffusion-model-url }}
+          ${{ matrix.curl-command }} -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" -L ${{ matrix.stable-diffusion-model-url }}
 
       - name: run configure_invokeai.py
         id: run-preload-models
@@ -112,6 +120,7 @@ jobs:
 
       - name: Run the tests
         id: run-tests
+        if: matrix.os != 'windows-2022'
         run: |
           time python scripts/invoke.py \
             --no-patchmatch \
@@ -123,11 +132,13 @@ jobs:
 
       - name: export conda env
         id: export-conda-env
+        if: matrix.os != 'windows-2022'
         run: |
           mkdir -p outputs/img-samples
-          conda env export --name ${{ env.CONDA_ENV_NAME }} > outputs/img-samples/environment-${{ runner.os }}-${{ runner.arch }}.yml
+          conda env export --name ${{ env.CONDA_ENV_NAME }} > ${{ env.INVOKEAI_ROOT }}/outputs/environment-${{ runner.os }}-${{ runner.arch }}.yml
 
       - name: Archive results
+        if: matrix.os != 'windows-2022'
         id: archive-results
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/test-invoke-pip.yml b/.github/workflows/test-invoke-pip.yml
@@ -19,51 +19,66 @@ jobs:
           - requirements-lin-cuda.txt
           - requirements-lin-amd.txt
           - requirements-mac-mps-cpu.txt
+          - requirements-win-colab-cuda.txt
         python-version:
           # - '3.9'
           - '3.10'
         include:
           - requirements-file: requirements-lin-cuda.txt
-            os: ubuntu-latest
-            default-shell: bash -l {0}
+            os: ubuntu-22.04
+            curl-command: curl
+            github-env: $GITHUB_ENV
           - requirements-file: requirements-lin-amd.txt
-            os: ubuntu-latest
-            default-shell: bash -l {0}
+            os: ubuntu-22.04
+            curl-command: curl
+            github-env: $GITHUB_ENV
           - requirements-file: requirements-mac-mps-cpu.txt
             os: macOS-12
-            default-shell: bash -l {0}
+            curl-command: curl
+            github-env: $GITHUB_ENV
+          - requirements-file: requirements-win-colab-cuda.txt
+            os: windows-2022
+            curl-command: curl.exe
+            github-env: $env:GITHUB_ENV
           - stable-diffusion-model: stable-diffusion-1.5
             stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
             stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
             stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
     name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: ${{ matrix.default-shell }}
-    env:
-      INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
     steps:
       - name: Checkout sources
         id: checkout-sources
         uses: actions/checkout@v3
 
+      - name: set INVOKEAI_ROOT Windows
+        if: matrix.os == 'windows-2022'
+        run: |
+          echo "INVOKEAI_ROOT=${{ github.workspace }}\invokeai" >> ${{ matrix.github-env }}
+          echo "INVOKEAI_OUTDIR=${{ github.workspace }}\invokeai\outputs" >> ${{ matrix.github-env }}
+
+      - name: set INVOKEAI_ROOT others
+        if: matrix.os != 'windows-2022'
+        run: |
+          echo "INVOKEAI_ROOT=${{ github.workspace }}/invokeai" >> ${{ matrix.github-env }}
+          echo "INVOKEAI_OUTDIR=${{ github.workspace }}/invokeai/outputs" >> ${{ matrix.github-env }}
+
       - name: create models.yaml from example
         run: |
           mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
           cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
 
       - name: set test prompt to main branch validation
         if: ${{ github.ref == 'refs/heads/main' }}
-        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> ${{ matrix.github-env }}
 
       - name: set test prompt to development branch validation
         if: ${{ github.ref == 'refs/heads/development' }}
-        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/dev_prompts.txt" >> ${{ matrix.github-env }}
 
       - name: set test prompt to Pull Request validation
         if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
-        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV
+        run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }}
 
       - name: create requirements.txt
         run: cp 'environments-and-requirements/${{ matrix.requirements-file }}' '${{ matrix.requirements-file }}'
@@ -72,14 +87,14 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: ${{ matrix.requirements-file }}
+          # cache: 'pip'
+          # cache-dependency-path: ${{ matrix.requirements-file }}
 
       # - name: install dependencies
       #   run: ${{ env.pythonLocation }}/bin/pip install --upgrade pip setuptools wheel
 
       - name: install requirements
-        run: ${{ env.pythonLocation }}/bin/pip install -r '${{ matrix.requirements-file }}'
+        run: pip3 install -r '${{ matrix.requirements-file }}'
 
       - name: Use Cached Stable Diffusion Model
         id: cache-sd-model
@@ -95,33 +110,20 @@ jobs:
         if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
         run: |
           mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          curl \
-            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
-            -L ${{ matrix.stable-diffusion-model-url }}
+          ${{ matrix.curl-command }} -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" -L ${{ matrix.stable-diffusion-model-url }}
 
       - name: run configure_invokeai.py
         id: run-preload-models
-        run: |
-          ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py --no-interactive --yes
-
-      - name: cat ~/.invokeai
-        id: cat-invokeai
-        run: cat ~/.invokeai
+        run: python3 scripts/configure_invokeai.py --no-interactive --yes
 
       - name: Run the tests
         id: run-tests
-        run: |
-          time ${{ env.pythonLocation }}/bin/python scripts/invoke.py \
-            --no-patchmatch \
-            --no-nsfw_checker \
-            --model ${{ matrix.stable-diffusion-model }} \
-            --from_file ${{ env.TEST_PROMPTS }} \
-            --root="${{ env.INVOKEAI_ROOT }}" \
-            --outdir="${{ env.INVOKEAI_ROOT }}/outputs"
+        if: matrix.os != 'windows-2022'
+        run: python3 scripts/invoke.py --no-patchmatch --no-nsfw_checker --model ${{ matrix.stable-diffusion-model }} --from_file ${{ env.TEST_PROMPTS }} --root="${{ env.INVOKEAI_ROOT }}" --outdir="${{ env.INVOKEAI_OUTDIR }}"
 
       - name: Archive results
         id: archive-results
+        if: matrix.os != 'windows-2022'
         uses: actions/upload-artifact@v3
         with:
           name: results_${{ matrix.requirements-file }}_${{ matrix.python-version }}

diff --git a/environments-and-requirements/requirements-base.txt b/environments-and-requirements/requirements-base.txt
@@ -33,7 +33,7 @@ torch-fidelity
 torchmetrics
 transformers==4.21.*
 picklescan
-git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.1#egg=gfpgan ; platform_system == 'Windows'
+# git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.1#egg=gfpgan ; platform_system == 'Windows'
 git+https://github.com/invoke-ai/GFPGAN@basicsr-1.4.2#egg=gfpgan ; platform_system != 'Windows'
 git+https://github.com/openai/CLIP.git@main#egg=clip
 git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion

diff --git a/environments-and-requirements/requirements-win-colab-cuda.txt b/environments-and-requirements/requirements-win-colab-cuda.txt
@@ -1,7 +1,8 @@
 -r environments-and-requirements/requirements-base.txt
 # Get hardware-appropriate torch/torchvision
 --extra-index-url https://download.pytorch.org/whl/cu116 --trusted-host https://download.pytorch.org
-basicsr==1.4.1
+gfpgan
+basicsr
 torch==1.12.1
 torchvision==0.13.1
 -e .