Merge pull request #4 from bitextor/crawl2 #56
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and testing Monotextor only | |
| on: | |
| push: | |
| paths-ignore: | |
| - '**.md' | |
| - 'docs/**' | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| - 'docs/**' | |
| env: | |
| WORK: ${{ github.workspace }} | |
| jobs: | |
| build_and_testing: | |
| name: Build and testing | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Install required linux packages | |
| run: | | |
| sudo apt-get --assume-yes update | |
| sudo apt-get --assume-yes install git time python3 python3-venv python3-pip python3-setuptools \ | |
| golang-go build-essential cmake libboost-all-dev liblzma-dev time curl pigz parallel wget \ | |
| uchardet libuchardet-dev libzip-dev autopoint autoconf automake libtool | |
| - name: Set up Python3 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8.5' | |
| architecture: 'x64' | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Install additional python requirements | |
| run: | | |
| pip3 install --upgrade pip | |
| pip3 install . | |
| - name: Setup golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.17.3 | |
| - name: Install giashard | |
| run: | | |
| go install github.com/paracrawl/giashard/cmd/giashard@latest | |
| - name: Compiling monotextor and submodules | |
| run: | | |
| mkdir build_cmake && cd build_cmake | |
| cmake -DCMAKE_INSTALL_PREFIX=/usr .. | |
| make -j | |
| sudo make install | |
| - name: Run tests | |
| id: tests | |
| run: | | |
| chmod 775 ./tests/run-tests-monotextor-only.sh | |
| ./tests/run-tests-monotextor-only.sh -w "$WORK" -j 4 | |
| - name: Print log of tests which failed | |
| # https://github.com/actions/runner/issues/1173 | |
| #if: ${{ steps.tests.conclusion != 'success' }} | |
| if: ${{ always() }} | |
| run: | | |
| if [[ -f "${WORK}/data/fails.log" ]]; then | |
| while read line; do | |
| IFS=$'\t' read -r -a array <<< "$line" | |
| status=${array[0]} | |
| notest=${array[1]} | |
| exit_code=${array[2]} | |
| str="# Test $notest (exit code / desc.: $exit_code) #" | |
| eval $(echo printf '"#%0.s"' {1..${#str}}) && printf "\n" | |
| echo "$str" | |
| eval $(echo printf '"#%0.s"' {1..${#str}}) && printf "\n" | |
| report_file="${WORK}/reports/${notest}.report" | |
| if [[ -f "$report_file" ]]; then | |
| awk -v prefix="(log test $notest)" '{print prefix" "$0}' "$report_file" | |
| else | |
| echo "(warning) No report file found for test $notest (this might be normal depending on the test; check the description)" | |
| fi | |
| echo "" | |
| done < "${WORK}/data/fails.log" | |
| else | |
| >&2 echo "ERROR: could not find the file which contain the fails, and should exist" | |
| exit 1 | |
| fi | |
| - name: Upload sent.gz files (artifacts) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: all-sent.gz | |
| path: ${{ env.WORK }}/permanent/**/*.sent.gz | |
| - name: Upload raw.gz files (artifacts) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: all-raw.gz | |
| path: ${{ env.WORK }}/permanent/**/*.raw.gz | |
| - name: Upload raw.paragraphs.gz files (artifacts) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: all-paragraphs.gz | |
| path: ${{ env.WORK }}/permanent/**/*.raw.paragraphs.gz | |
| - name: Upload report files (artifacts) | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: reports | |
| path: ${{ env.WORK }}/reports/*.report |