From a0eef91540a9256f816d304119a9d05acfcd1570 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 30 Nov 2020 16:18:43 +0100 Subject: [PATCH 01/17] GitHub Action to lint Python code Because Travis CI no longer freely supports open source projects. --- .github/workflows/lint_python.yml | 24 +++++++++++++++++++++ .github/workflows/npm_run_test.yml.disabled | 16 ++++++++++++++ .travis.yml => was.travis.yml | 0 3 files changed, 40 insertions(+) create mode 100644 .github/workflows/lint_python.yml create mode 100644 .github/workflows/npm_run_test.yml.disabled rename .travis.yml => was.travis.yml (100%) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 00000000000..82a7aea5e1f --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,24 @@ +name: lint_python +on: [push] # pull_request, +jobs: + lint_python: + strategy: + fail-fast: false + matrix: + python-version: [2.7, 3.8, 3.9] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - run: sudo apt-get install libxml2-dev libxslt-dev + - run: pip install --upgrade pip setuptools wheel + - run: pip install -r requirements_test.txt + - run: pip list --outdated + - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master + - run: rm infogami # Remove symlink in favour of PYTHONPATH + - run: make lint + - run: make i18n + - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" make test-py + - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" source scripts/run_doctests.sh diff --git a/.github/workflows/npm_run_test.yml.disabled b/.github/workflows/npm_run_test.yml.disabled new file mode 100644 index 00000000000..87b8737fa8c --- /dev/null +++ b/.github/workflows/npm_run_test.yml.disabled @@ -0,0 +1,16 @@ +name: npm_run_test +on: [push] # pull_request, +jobs: + npm_run_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: '12' # # Should match what's in our Dockerfile + - run: npm install + - run: npm run lint + - run: make js + - run: make css + - run: make components + - run: npm run test diff --git a/.travis.yml b/was.travis.yml similarity index 100% rename from .travis.yml rename to was.travis.yml From f0174ccf3666f2cf8fa92fef0c3cc3c99bf22588 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 7 Dec 2020 14:01:22 +0100 Subject: [PATCH 02/17] Run mypy in allow_failures mode --- .github/workflows/lint_python.yml | 5 +++-- .github/workflows/npm_run_test.yml.disabled | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 82a7aea5e1f..efe85ed890c 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -1,5 +1,5 @@ name: lint_python -on: [push] # pull_request, +on: [push] # pull_request, jobs: lint_python: strategy: @@ -14,7 +14,7 @@ jobs: python-version: ${{ matrix.python-version }} - run: sudo apt-get install libxml2-dev libxslt-dev - run: pip install --upgrade pip setuptools wheel - - run: pip install -r requirements_test.txt + - run: pip install mypy -r requirements_test.txt - run: pip list --outdated - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master - run: rm infogami # Remove symlink in favour of PYTHONPATH @@ -22,3 +22,4 @@ jobs: - run: make i18n - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" make test-py - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" source scripts/run_doctests.sh + - run: mypy . || true diff --git a/.github/workflows/npm_run_test.yml.disabled b/.github/workflows/npm_run_test.yml.disabled index 87b8737fa8c..df8bf71cece 100644 --- a/.github/workflows/npm_run_test.yml.disabled +++ b/.github/workflows/npm_run_test.yml.disabled @@ -1,5 +1,5 @@ name: npm_run_test -on: [push] # pull_request, +on: [push] # pull_request, jobs: npm_run_test: runs-on: ubuntu-latest From 61ec1e09aca0534958a8d575d67e9e355a56829b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 8 Dec 2020 15:40:58 +0100 Subject: [PATCH 03/17] Update and rename lint_python.yml to python_tests.yml --- .github/workflows/{lint_python.yml => python_tests.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{lint_python.yml => python_tests.yml} (96%) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/python_tests.yml similarity index 96% rename from .github/workflows/lint_python.yml rename to .github/workflows/python_tests.yml index efe85ed890c..2b5ce900f36 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/python_tests.yml @@ -1,7 +1,7 @@ -name: lint_python +name: python_tests on: [push] # pull_request, jobs: - lint_python: + python_tests: strategy: fail-fast: false matrix: From 4c2ca0c204d4e3e0f5663cc480810973c624586e Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 8 Dec 2020 15:42:30 +0100 Subject: [PATCH 04/17] Delete npm_run_test.yml.disabled --- .github/workflows/npm_run_test.yml.disabled | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/npm_run_test.yml.disabled diff --git a/.github/workflows/npm_run_test.yml.disabled b/.github/workflows/npm_run_test.yml.disabled deleted file mode 100644 index df8bf71cece..00000000000 --- a/.github/workflows/npm_run_test.yml.disabled +++ /dev/null @@ -1,16 +0,0 @@ -name: npm_run_test -on: [push] # pull_request, -jobs: - npm_run_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: '12' # # Should match what's in our Dockerfile - - run: npm install - - run: npm run lint - - run: make js - - run: make css - - run: make components - - run: npm run test From 9540059b0568953fcd9e44a0890e0d13725582e2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 8 Dec 2020 15:42:53 +0100 Subject: [PATCH 05/17] Rename was.travis.yml to .travis.yml --- was.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename was.travis.yml => .travis.yml (100%) diff --git a/was.travis.yml b/.travis.yml similarity index 100% rename from was.travis.yml rename to .travis.yml From bb4d8da421554d5ef25d26bf5bd5d2f8afabeb71 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 8 Dec 2020 15:48:45 +0100 Subject: [PATCH 06/17] mypy only if: matrix.python-version >= 3.6 --- .github/workflows/python_tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 2b5ce900f36..39e9f3e94ce 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -14,7 +14,7 @@ jobs: python-version: ${{ matrix.python-version }} - run: sudo apt-get install libxml2-dev libxslt-dev - run: pip install --upgrade pip setuptools wheel - - run: pip install mypy -r requirements_test.txt + - run: pip install -r requirements_test.txt - run: pip list --outdated - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master - run: rm infogami # Remove symlink in favour of PYTHONPATH @@ -22,4 +22,7 @@ jobs: - run: make i18n - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" make test-py - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" source scripts/run_doctests.sh - - run: mypy . || true + - if: matrix.python-version >= 3.6 + run: | + pip install mypy + mypy . || true From fc5a48040ad63de519303f507638cc8c47238228 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 9 Dec 2020 16:15:29 +0100 Subject: [PATCH 07/17] DIsable Python 3.9 to save resources --- .github/workflows/python_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 39e9f3e94ce..2a8a149f729 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -5,8 +5,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: [2.7, 3.8, 3.9] - runs-on: ubuntu-latest + python-version: [2.7, 3.8] # , 3.9] + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From 25895c13c6c7b2b272a42d0ec1ec152816c3c6f2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 09:42:18 +0100 Subject: [PATCH 08/17] make git --- .github/workflows/python_tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 2a8a149f729..b7dfde1a5a5 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -16,12 +16,13 @@ jobs: - run: pip install --upgrade pip setuptools wheel - run: pip install -r requirements_test.txt - run: pip list --outdated - - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master - - run: rm infogami # Remove symlink in favour of PYTHONPATH + - run: make git + # - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master + # - run: rm infogami # Remove symlink in favour of PYTHONPATH - run: make lint - run: make i18n - - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" make test-py - - run: PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" source scripts/run_doctests.sh + - run: make test-py # PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" + - run: source scripts/run_doctests.sh # PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" - if: matrix.python-version >= 3.6 run: | pip install mypy From 615d2344f0d0dfb1b11d84c13e56445cba58c233 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 09:53:57 +0100 Subject: [PATCH 09/17] pushd vendor/infogami && git pull origin master && popd --- .github/workflows/python_tests.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index b7dfde1a5a5..62b52da6305 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -17,13 +17,11 @@ jobs: - run: pip install -r requirements_test.txt - run: pip list --outdated - run: make git - # - run: git clone https://github.com/internetarchive/infogami.git vendor/infogami-master - # - run: rm infogami # Remove symlink in favour of PYTHONPATH + - if: matrix.python-version >= 3.0 + run: pushd vendor/infogami && git pull origin master && popd - run: make lint - run: make i18n - - run: make test-py # PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" - - run: source scripts/run_doctests.sh # PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" + - run: make test-py + - run: source scripts/run_doctests.sh - if: matrix.python-version >= 3.6 - run: | - pip install mypy - mypy . || true + run: pip install mypy && mypy . || true From f3d3fec06437f104393fdef3a40119203b27de54 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 10:02:56 +0100 Subject: [PATCH 10/17] We are not yet ready for mypy --- .github/workflows/python_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 62b52da6305..63a9d00dd6b 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -23,5 +23,3 @@ jobs: - run: make i18n - run: make test-py - run: source scripts/run_doctests.sh - - if: matrix.python-version >= 3.6 - run: pip install mypy && mypy . || true From baf9f61989ad0571497b91350b1edd44e9bf8ea4 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 19:37:02 +0100 Subject: [PATCH 11/17] Update .github/workflows/python_tests.yml Co-authored-by: Drini Cami --- .github/workflows/python_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 63a9d00dd6b..15c75bcbbd5 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -6,7 +6,7 @@ jobs: fail-fast: false matrix: python-version: [2.7, 3.8] # , 3.9] - runs-on: ubuntu-20.04 + runs-on: ubuntu-xenial # Should match Dockerfile.olbase steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From 86ab2752dac0dbc04b946a6561395c40441786ee Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 19:43:36 +0100 Subject: [PATCH 12/17] ubuntu-18.04 and https://lxml.de/installation.html#requirements --- .github/workflows/python_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 15c75bcbbd5..ac8701c8de0 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -6,12 +6,13 @@ jobs: fail-fast: false matrix: python-version: [2.7, 3.8] # , 3.9] - runs-on: ubuntu-xenial # Should match Dockerfile.olbase + runs-on: ubuntu-18.04 # Should match Dockerfile.olbase steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + # https://lxml.de/installation.html#requirements - run: sudo apt-get install libxml2-dev libxslt-dev - run: pip install --upgrade pip setuptools wheel - run: pip install -r requirements_test.txt From 202b3a7925178b97385ee6a1aab26965aa0dce0c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 20:04:06 +0100 Subject: [PATCH 13/17] Update python_tests.yml --- .github/workflows/python_tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index ac8701c8de0..2403de54fcf 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -18,9 +18,12 @@ jobs: - run: pip install -r requirements_test.txt - run: pip list --outdated - run: make git - - if: matrix.python-version >= 3.0 - run: pushd vendor/infogami && git pull origin master && popd - - run: make lint - run: make i18n + # Run legacy Python tests on legacy Infogami + - if: matrix.python-version < 3.0 + run: make test-py && source scripts/run_doctests.sh + # Run both Py2 and Py3 tests on Infogami master + - run: pushd vendor/infogami && git pull origin master && popd + - run: make lint - run: make test-py - run: source scripts/run_doctests.sh From 117f00e11f1e555598ca1e89508e48fdeea1b8c2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 21:02:19 +0100 Subject: [PATCH 14/17] push, pull, click, click --- .github/workflows/python_tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 2403de54fcf..d1c84ab7c19 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -1,5 +1,9 @@ name: python_tests -on: [push] # pull_request, +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] jobs: python_tests: strategy: From 07f1b19bd7edf05007ab9d62276bdd2261f65eeb Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 21:02:57 +0100 Subject: [PATCH 15/17] Delete .travis.yml --- .travis.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index bb06241c6da..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Only run travis when pushing to master branch (and PRs) -branches: - only: - - master - -os: linux -dist: xenial -language: python -jobs: - include: - - name: Python 2.7 - python: "2.7" - - - name: Python 3.8 - python: "3.8" - before_script: - # Do more aggressive linting for newly added code - - make lint-diff - - - name: Node - language: node_js - # Should match what's in our Dockerfile - node_js: "12" - install: npm install - script: - - npm run lint - - make js - - make css - - make components - - npm run test - -install: - - pip install --upgrade pip setuptools wheel - - pip install -r requirements_test.txt - # Need to also test infogami-master - - git clone https://github.com/internetarchive/infogami.git vendor/infogami-master - # Remove symlink in favour of PYTHONPATH - - rm infogami - -script: - - make lint - - make i18n - # Python 2 should also be tested against production infogami - - if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then - PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami" make test-py; - PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami" source scripts/run_doctests.sh; - fi - # Both should be tested against infogami-master - - PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" make test-py - - PYTHONPATH="$PYTHONPATH:$PWD/vendor/infogami-master" source scripts/run_doctests.sh From 842e95908db7c44e4df4cd896176023fb809fab0 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 23:18:32 +0100 Subject: [PATCH 16/17] Update python_tests.yml --- .github/workflows/python_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index d1c84ab7c19..04ce122f617 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -21,6 +21,8 @@ jobs: - run: pip install --upgrade pip setuptools wheel - run: pip install -r requirements_test.txt - run: pip list --outdated + - if: matrix.python-version == 3.8 + run: make lint-diff - run: make git - run: make i18n # Run legacy Python tests on legacy Infogami From bcfa00e86583dc8da0d8e1ffa58602b7036d9239 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 11 Dec 2020 23:26:08 +0100 Subject: [PATCH 17/17] Update .github/workflows/python_tests.yml Co-authored-by: Drini Cami --- .github/workflows/python_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 04ce122f617..5dd2ca7c3ad 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: python-version: [2.7, 3.8] # , 3.9] - runs-on: ubuntu-18.04 # Should match Dockerfile.olbase + runs-on: ubuntu-16.04 # Should match Dockerfile.olbase steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2