BugFix #89
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # GitHub Action workflow to build and run filecrawler's tests | |
| # | |
| name: Test Build | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| env: | |
| TEST_TAG: filecrawler:test | |
| LATEST_TAG: filecrawler:latest | |
| jobs: | |
| lint: | |
| name: Check syntaxs errors and warnings | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python 3.8 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 | |
| - name: Check syntax errors | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Check PEP8 warnings | |
| run: | | |
| flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics | |
| test: | |
| name: Run unit tests and build wheel | |
| needs: lint | |
| runs-on: ${{ matrix.os }} | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10"] | |
| experimental: [false] | |
| os: [ubuntu-latest, macos-latest] | |
| #os: [ubuntu-latest, macos-latest, windows-latest] | |
| include: | |
| - python-version: "3.11" | |
| experimental: true | |
| os: ubuntu-latest | |
| - python-version: "3.11" | |
| experimental: true | |
| os: macos-latest | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Homebrew (MacOS) | |
| if: matrix.os == 'macos-latest' | |
| run: | | |
| NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" | |
| brew install libmagic | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip wheel | |
| python -m pip install -r tests/requirements-test.txt | |
| - name: Install filecrawler | |
| run: | | |
| pip install . | |
| - name: Run unit tests | |
| run: | | |
| pytest -s tests/tests.py | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| docker_test: | |
| name: Check docker build and run | |
| needs: test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and export to Docker | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| load: true | |
| push: false | |
| tags: ${{ env.TEST_TAG }} | |
| - name: Test | |
| run: | | |
| docker run --rm ${{ env.TEST_TAG }} | |
| #- name: Build and push | |
| # uses: docker/build-push-action@v4 | |
| # with: | |
| # context: . | |
| # platforms: linux/amd64,linux/arm64 | |
| # push: false | |
| # tags: ${{ env.LATEST_TAG }} | |
| publish: | |
| name: Publish on Test Environment | |
| needs: docker_test | |
| runs-on: ubuntu-latest | |
| if: | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
| github.repository | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9"] | |
| experimental: [false] | |
| continue-on-error: ${{ matrix.experimental }} | |
| steps: | |
| - name: Checkout filecrawler | |
| uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: wget curl jq | |
| version: 1.0 | |
| - name: Setup Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r tests/requirements-test.txt | |
| - name: Change Package version | |
| run: | | |
| oldv=$( grep '__version__' filecrawler/__meta__.py | grep -oE '[0-9\.]+') | |
| current=$(date '+%Y%m%d%H%M%S') | |
| meta=$(cat filecrawler/__meta__.py | sed "s/__version__.*/__version__ = '"${oldv}"-"${current}"'/") | |
| echo "$meta" > filecrawler/__meta__.py | |
| - name: Get tika server .jar | |
| run: | | |
| VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0') | |
| SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1") | |
| wget -nv -O "./filecrawler/libs/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar" | |
| echo "${SERVER_HASH} ./filecrawler/libs/bin/tika-server.jar" | sha1sum -c - || exit 1 | |
| - name: Install filecrawler | |
| run: | | |
| pip install . | |
| - name: Build artifact | |
| run: | | |
| python setup.py sdist | |
| - name: Publish package in Test Environment | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.test_pypi_password }} | |
| repository_url: https://test.pypi.org/legacy/ | |
| skip_existing: true |