From fc2670b4d6da37690d2984e393552c253c4901fb Mon Sep 17 00:00:00 2001 From: mauwii Date: Sun, 5 Feb 2023 06:15:34 +0100 Subject: [PATCH 1/2] update `test-invoke-pip.yml` - add workflow_dispatch trigger - fix indentation in concurrency - set env `PIP_USE_PEP517: '1'` - cache python dependencies - remove models cache (since currently 183.59 GB of 10 GB are Used) - add step to set `INVOKEAI_OUTDIR` - add outdir arg to invokeai - fix path in archive results --- .github/workflows/test-invoke-pip.yml | 57 +++++++++++---------------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test-invoke-pip.yml b/.github/workflows/test-invoke-pip.yml index d696406ac8d..974d8dcee53 100644 --- a/.github/workflows/test-invoke-pip.yml +++ b/.github/workflows/test-invoke-pip.yml @@ -8,10 +8,11 @@ on: - 'ready_for_review' - 'opened' - 'synchronize' + workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: matrix: @@ -62,28 +63,13 @@ jobs: # github-env: $env:GITHUB_ENV name: ${{ matrix.pytorch }} on ${{ matrix.python-version }} runs-on: ${{ matrix.os }} + env: + PIP_USE_PEP517: '1' steps: - name: Checkout sources id: checkout-sources uses: actions/checkout@v3 - - name: setup python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Set Cache-Directory Windows - if: runner.os == 'Windows' - id: set-cache-dir-windows - run: | - echo "CACHE_DIR=$HOME\invokeai\models" >> ${{ matrix.github-env }} - echo "PIP_NO_CACHE_DIR=1" >> ${{ matrix.github-env }} - - - name: Set Cache-Directory others - if: runner.os != 'Windows' - id: set-cache-dir-others - run: echo "CACHE_DIR=$HOME/invokeai/models" >> ${{ matrix.github-env }} - - name: set test prompt to main branch validation if: ${{ github.ref == 'refs/heads/main' }} run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> ${{ matrix.github-env }} @@ -92,26 +78,29 @@ jobs: if: ${{ github.ref != 'refs/heads/main' }} run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> ${{ matrix.github-env }} + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: pyproject.toml + - name: install invokeai env: PIP_EXTRA_INDEX_URL: ${{ matrix.extra-index-url }} run: > pip3 install - --use-pep517 --editable=".[test]" - name: run pytest + id: run-pytest run: pytest - - name: Use Cached models - id: cache-sd-model - uses: actions/cache@v3 - env: - cache-name: huggingface-models - with: - path: ${{ env.CACHE_DIR }} - key: ${{ env.cache-name }} - enableCrossOsArchive: true + - name: set INVOKEAI_OUTDIR + run: > + python -c + "import os;from ldm.invoke.globals import Globals;OUTDIR=os.path.join(Globals.root,str('outputs'));print(f'INVOKEAI_OUTDIR={OUTDIR}')" + >> ${{ matrix.github-env }} - name: run invokeai-configure id: run-preload-models @@ -124,9 +113,8 @@ jobs: --full-precision # can't use fp16 weights without a GPU - - name: Run the tests - if: runner.os != 'Windows' - id: run-tests + - name: run invokeai + id: run-invokeai env: # Set offline mode to make sure configure preloaded successfully. HF_HUB_OFFLINE: 1 @@ -137,10 +125,11 @@ jobs: --no-patchmatch --no-nsfw_checker --from_file ${{ env.TEST_PROMPTS }} + --outdir ${{ env.INVOKEAI_OUTDIR }}/${{ matrix.python-version }}/${{ matrix.pytorch }} - name: Archive results id: archive-results uses: actions/upload-artifact@v3 with: - name: results_${{ matrix.pytorch }}_${{ matrix.python-version }} - path: ${{ env.INVOKEAI_ROOT }}/outputs + name: results + path: ${{ env.INVOKEAI_OUTDIR }} From a40bdef29f5803057b49b85c6b2e8ec582038a6f Mon Sep 17 00:00:00 2001 From: mauwii Date: Sun, 5 Feb 2023 07:13:42 +0100 Subject: [PATCH 2/2] update model_manager.py - read files in chunks when calculating sha - windows runner is crashing without --- ldm/invoke/model_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ldm/invoke/model_manager.py b/ldm/invoke/model_manager.py index 43854d7938f..69bfbd587d4 100644 --- a/ldm/invoke/model_manager.py +++ b/ldm/invoke/model_manager.py @@ -753,7 +753,7 @@ def search_models(self, search_folder): return search_folder, found_models def _choose_diffusers_vae(self, model_name:str, vae:str=None)->Union[dict,str]: - + # In the event that the original entry is using a custom ckpt VAE, we try to # map that VAE onto a diffuser VAE using a hard-coded dictionary. # I would prefer to do this differently: We load the ckpt model into memory, swap the @@ -954,7 +954,7 @@ def _push_newest_model(self,model_name:str) -> None: def _has_cuda(self) -> bool: return self.device.type == 'cuda' - def _diffuser_sha256(self,name_or_path:Union[str, Path])->Union[str,bytes]: + def _diffuser_sha256(self,name_or_path:Union[str, Path],chunksize=4096)->Union[str,bytes]: path = None if isinstance(name_or_path,Path): path = name_or_path @@ -976,7 +976,8 @@ def _diffuser_sha256(self,name_or_path:Union[str, Path])->Union[str,bytes]: for name in files: count += 1 with open(os.path.join(root,name),'rb') as f: - sha.update(f.read()) + while chunk := f.read(chunksize): + sha.update(chunk) hash = sha.hexdigest() toc = time.time() print(f' | sha256 = {hash} ({count} files hashed in','%4.2fs)' % (toc - tic))