From 6f31639adfbc403a59285293f9ac556a06ae0415 Mon Sep 17 00:00:00 2001 From: bruvio Date: Thu, 17 Feb 2022 13:23:18 +0000 Subject: [PATCH 01/19] update ci to run on windows --- .github/workflows/pyDataPipeline.yaml | 66 +++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 140ffad5..27f9bbf2 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -6,8 +6,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-latest] #, windows-latest] - python: ["3.7","3.8", "3.9"] + os: [windows-latest,ubuntu-20.04,macos-latest] + python: ["3.7"] #,"3.8", "3.9"] env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: @@ -17,14 +17,47 @@ jobs: python-version: ${{ matrix.python }} architecture: "x64" - name: Install graphviz - run: | - if [ ${{ runner.os }} == "Linux" ]; then - sudo apt-get install graphviz - else - brew install graphviz - fi + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install graphviz + elif [ "$RUNNER_OS" == "Windows" ]; then + choco install graphviz + elif [ "$RUNNER_OS" == "Darwin" ]; then + brew install graphviz + fi + shell: bash - name: Install local registry - run: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main + run: | + if [ "$RUNNER_OS" != "Windows" ]; then + curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main + fi + shell: bash + - name: Checkout data-registry + if: matrix.os == 'windows-latest' + uses: actions/checkout@v2 + with: + repository: FAIRDataPipeline/data-registry + path: data-registry + - name: Move registry + if: matrix.os == 'windows-latest' + run: | + mkdir $HOME\.fair\registry + Move-Item -Path "data-registry\*" -Destination "$HOME/.fair/registry" + - name: install local registry on windows + if: matrix.os == 'windows-latest' + run: | + python3 -m venv $HOME\.fair\registry\venv + python -m pip install --upgrade pip wheel + python -m pip install -r $HOME\.fair\registry\local-requirements.txt + $env:DJANGO_SETTINGS_MODULE="drams.local-settings" + $env:DJANGO_SUPERUSER_USERNAME="admin" + $env:DJANGO_SUPERUSER_PASSWORD="admin" + cd $HOME\.fair\registry + python3 manage.py makemigrations custom_user + python3 manage.py makemigrations data_management + python3 manage.py migrate + python3 manage.py collectstatic --noinput + python3 manage.py createsuperuser --noinput - name: Install Required run: brew install memcached if: matrix.os == 'macos-latest' @@ -40,7 +73,20 @@ jobs: path: pySimpleModel - name: Move pySimpleModel run: mv pySimpleModel ../pySimpleModel - - name: Run Tests + - name: Run Tests on Windows + run: | + poetry install + poetry shell + pip install fair-cli + pip install ../pySimpleModel + fair init --ci + fair registry start + fair pull data_pipeline_api/ext/SEIRSconfig.yaml + fair run data_pipeline_api/ext/SEIRSconfig.yaml + + python3 -m poetry run pytest --cov=data_pipeline_api --cov-report=xml --cov-report=term -s tests/ + if: matrix.os == 'windows-latest' + - name: Run Tests on other OS run: | poetry install source .venv/bin/activate From fccfa9b3a11f6628b92533cc5a4cc18bcc774d74 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 14:16:06 +0000 Subject: [PATCH 02/19] Update pyDataPipeline.yaml Use fair-cli to install registry on Windows --- .github/workflows/pyDataPipeline.yaml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 27f9bbf2..b61011f7 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -43,21 +43,6 @@ jobs: run: | mkdir $HOME\.fair\registry Move-Item -Path "data-registry\*" -Destination "$HOME/.fair/registry" - - name: install local registry on windows - if: matrix.os == 'windows-latest' - run: | - python3 -m venv $HOME\.fair\registry\venv - python -m pip install --upgrade pip wheel - python -m pip install -r $HOME\.fair\registry\local-requirements.txt - $env:DJANGO_SETTINGS_MODULE="drams.local-settings" - $env:DJANGO_SUPERUSER_USERNAME="admin" - $env:DJANGO_SUPERUSER_PASSWORD="admin" - cd $HOME\.fair\registry - python3 manage.py makemigrations custom_user - python3 manage.py makemigrations data_management - python3 manage.py migrate - python3 manage.py collectstatic --noinput - python3 manage.py createsuperuser --noinput - name: Install Required run: brew install memcached if: matrix.os == 'macos-latest' @@ -78,6 +63,7 @@ jobs: poetry install poetry shell pip install fair-cli + fair registry install pip install ../pySimpleModel fair init --ci fair registry start From be8308a1e72ef04258940b32f805c2d0ea220447 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 15:10:30 +0000 Subject: [PATCH 03/19] Update pyDataPipeline.yaml Try using windows installation script --- .github/workflows/pyDataPipeline.yaml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index b61011f7..25523857 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -32,17 +32,12 @@ jobs: curl -fsSL https://data.scrc.uk/static/localregistry.sh | /bin/bash -s -- -b main fi shell: bash - - name: Checkout data-registry - if: matrix.os == 'windows-latest' - uses: actions/checkout@v2 - with: - repository: FAIRDataPipeline/data-registry - path: data-registry - - name: Move registry + - name: install local registry on windows if: matrix.os == 'windows-latest' run: | - mkdir $HOME\.fair\registry - Move-Item -Path "data-registry\*" -Destination "$HOME/.fair/registry" + choco install wget + wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat + install_registry.bat - name: Install Required run: brew install memcached if: matrix.os == 'macos-latest' From 58013f5dd52791fe362ab492f13e4f0605a064cf Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 15:23:05 +0000 Subject: [PATCH 04/19] Update pyDataPipeline.yaml Use command prompt to run batch script --- .github/workflows/pyDataPipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 25523857..24b78ed7 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -33,6 +33,7 @@ jobs: fi shell: bash - name: install local registry on windows + shell: cmd if: matrix.os == 'windows-latest' run: | choco install wget From 3436be4b714296978f53c4d1ce12375c2ecc197b Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 15:35:00 +0000 Subject: [PATCH 05/19] Update pyDataPipeline.yaml Try change working directory --- .github/workflows/pyDataPipeline.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 24b78ed7..a4f99707 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -37,6 +37,8 @@ jobs: if: matrix.os == 'windows-latest' run: | choco install wget + mkdir ~/.fair/registry + cd ~/.fair/registry wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat install_registry.bat - name: Install Required From 00de9e118b8b93d916f6abc34ffed67a6ef229ad Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 15:49:52 +0000 Subject: [PATCH 06/19] Update pyDataPipeline.yaml Use userprofile variable --- .github/workflows/pyDataPipeline.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index a4f99707..bf46e9e7 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -37,8 +37,9 @@ jobs: if: matrix.os == 'windows-latest' run: | choco install wget - mkdir ~/.fair/registry - cd ~/.fair/registry + mkdir %userprofile%/.fair/registry + cd %userprofile%/.fair/registry + echo %CD% wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat install_registry.bat - name: Install Required From 28e162fdcf33ac965c9756f580b19ba195161f44 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 15:55:57 +0000 Subject: [PATCH 07/19] Update pyDataPipeline.yaml Use md and backslash --- .github/workflows/pyDataPipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index bf46e9e7..6e78d7c3 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -37,8 +37,8 @@ jobs: if: matrix.os == 'windows-latest' run: | choco install wget - mkdir %userprofile%/.fair/registry - cd %userprofile%/.fair/registry + md %userprofile%\.fair\registry + cd %userprofile%\.fair\registry echo %CD% wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat install_registry.bat From a4ef1f94fd79cbd79a85a7e187a0b2d12ef11738 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:00:20 +0000 Subject: [PATCH 08/19] Update pyDataPipeline.yaml Issue with CD on different drive --- .github/workflows/pyDataPipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 6e78d7c3..401ba34b 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -38,7 +38,7 @@ jobs: run: | choco install wget md %userprofile%\.fair\registry - cd %userprofile%\.fair\registry + cd /d %userprofile%\.fair\registry echo %CD% wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat install_registry.bat From 379b5633dec8c10bc0dae1ab60a9ab483d93e944 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:08:24 +0000 Subject: [PATCH 09/19] Update pyDataPipeline.yaml Use .fair directory --- .github/workflows/pyDataPipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 401ba34b..5e44b611 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -37,8 +37,8 @@ jobs: if: matrix.os == 'windows-latest' run: | choco install wget - md %userprofile%\.fair\registry - cd /d %userprofile%\.fair\registry + md %userprofile%\.fair + cd /d %userprofile%\.fair echo %CD% wget -O install_registry.bat https://data.scrc.uk/static/localregistry.bat install_registry.bat From 487a1c832215c37a0ab90de126cbb5873a0a028c Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:22:56 +0000 Subject: [PATCH 10/19] Update pyDataPipeline.yaml try using venv on windows --- .github/workflows/pyDataPipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 5e44b611..c70c782e 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -59,10 +59,10 @@ jobs: run: mv pySimpleModel ../pySimpleModel - name: Run Tests on Windows run: | - poetry install - poetry shell + python3 -m venv venv + venv\Scripts\activate.ps1 + pip install . pip install fair-cli - fair registry install pip install ../pySimpleModel fair init --ci fair registry start From c7d487860d5561b08ba22ed2a0228ba982c56290 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:31:17 +0000 Subject: [PATCH 11/19] Update pyDataPipeline.yaml Use cmd as shell --- .github/workflows/pyDataPipeline.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index c70c782e..270f8d0d 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -58,9 +58,10 @@ jobs: - name: Move pySimpleModel run: mv pySimpleModel ../pySimpleModel - name: Run Tests on Windows + shell: cmd run: | python3 -m venv venv - venv\Scripts\activate.ps1 + venv\Scripts\activate.bat pip install . pip install fair-cli pip install ../pySimpleModel From bef2133bb29ebb8f123343e59e99603e837edea2 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 16:43:01 +0000 Subject: [PATCH 12/19] Update pyDataPipeline.yaml Use same python version --- .github/workflows/pyDataPipeline.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 270f8d0d..d1aa31f2 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -60,11 +60,13 @@ jobs: - name: Run Tests on Windows shell: cmd run: | + pip3 install venv + pip3 install poetry python3 -m venv venv venv\Scripts\activate.bat - pip install . - pip install fair-cli - pip install ../pySimpleModel + pip3 install . + pip3 install fair-cli + pip3 install ../pySimpleModel fair init --ci fair registry start fair pull data_pipeline_api/ext/SEIRSconfig.yaml From bcd075b1ce018019f9f341e3956232d588ca892a Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 19:52:08 +0000 Subject: [PATCH 13/19] Update pyDataPipeline.yaml Use poetry venv --- .github/workflows/pyDataPipeline.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index d1aa31f2..7fd6bf5b 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -58,13 +58,9 @@ jobs: - name: Move pySimpleModel run: mv pySimpleModel ../pySimpleModel - name: Run Tests on Windows - shell: cmd run: | - pip3 install venv - pip3 install poetry - python3 -m venv venv - venv\Scripts\activate.bat - pip3 install . + poetry install + source $VENV pip3 install fair-cli pip3 install ../pySimpleModel fair init --ci From f26cf444702375924e1bdb9311e5748de9583e3d Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 20:00:53 +0000 Subject: [PATCH 14/19] Update pyDataPipeline.yaml Use activate.bat --- .github/workflows/pyDataPipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 7fd6bf5b..14556836 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -60,7 +60,7 @@ jobs: - name: Run Tests on Windows run: | poetry install - source $VENV + .venv\Scripts\activate.bat pip3 install fair-cli pip3 install ../pySimpleModel fair init --ci From e0886954164bf7e19ff31458f904cc2970e6805a Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 20:18:02 +0000 Subject: [PATCH 15/19] Update pyDataPipeline.yaml use ps1 activation --- .github/workflows/pyDataPipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 14556836..ee9fd248 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -60,7 +60,7 @@ jobs: - name: Run Tests on Windows run: | poetry install - .venv\Scripts\activate.bat + .venv\Scripts\activate.ps1 pip3 install fair-cli pip3 install ../pySimpleModel fair init --ci From 1bb287d7cf5bc1189f6ee855a8e1e2e8a65492aa Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 20:23:56 +0000 Subject: [PATCH 16/19] Update pyDataPipeline.yaml --- .github/workflows/pyDataPipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index ee9fd248..08005585 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -59,8 +59,8 @@ jobs: run: mv pySimpleModel ../pySimpleModel - name: Run Tests on Windows run: | - poetry install .venv\Scripts\activate.ps1 + poetry install pip3 install fair-cli pip3 install ../pySimpleModel fair init --ci From 40b34b3eb36b9634dd312996d7e1a303adcbb85f Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 21:19:36 +0000 Subject: [PATCH 17/19] Update pyDataPipeline.yaml --- .github/workflows/pyDataPipeline.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index 08005585..e314bfa0 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -59,7 +59,6 @@ jobs: run: mv pySimpleModel ../pySimpleModel - name: Run Tests on Windows run: | - .venv\Scripts\activate.ps1 poetry install pip3 install fair-cli pip3 install ../pySimpleModel @@ -68,7 +67,7 @@ jobs: fair pull data_pipeline_api/ext/SEIRSconfig.yaml fair run data_pipeline_api/ext/SEIRSconfig.yaml - python3 -m poetry run pytest --cov=data_pipeline_api --cov-report=xml --cov-report=term -s tests/ + poetry run pytest --cov=data_pipeline_api --cov-report=xml --cov-report=term -s tests/ if: matrix.os == 'windows-latest' - name: Run Tests on other OS run: | From 8289fa9c65a7dbac3261c3c1243bf7121e344fbe Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Thu, 17 Feb 2022 21:30:38 +0000 Subject: [PATCH 18/19] Update pyDataPipeline.yaml --- .github/workflows/pyDataPipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pyDataPipeline.yaml b/.github/workflows/pyDataPipeline.yaml index e314bfa0..575a6a88 100644 --- a/.github/workflows/pyDataPipeline.yaml +++ b/.github/workflows/pyDataPipeline.yaml @@ -70,6 +70,7 @@ jobs: poetry run pytest --cov=data_pipeline_api --cov-report=xml --cov-report=term -s tests/ if: matrix.os == 'windows-latest' - name: Run Tests on other OS + if: matrix.os != 'windows-latest' run: | poetry install source .venv/bin/activate From 650d1122263061573e2c4770b990f5d6c0ae1288 Mon Sep 17 00:00:00 2001 From: Ryan J Field <57794045+RyanJField@users.noreply.github.com> Date: Fri, 18 Feb 2022 02:53:22 +0000 Subject: [PATCH 19/19] Update Code Coverage in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 316d5562..2fff01b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ line_length = 79 line-length = 79 [tool.pytest.ini_options] -addopts = '-s -v --cov=fairdatapipeline --cov-report=html --cov-report=term' +addopts = '-s -v --cov=data_pipeline_api --cov-report=html --cov-report=term' markers = [ "pipeline: tests for 'pipeline' module ", "issue: tests for raising issues ",