Update data from upstream #23
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update data from upstream | |
| on: | |
| schedule: | |
| - cron: "0 2 * * 2" | |
| workflow_dispatch: | |
| jobs: | |
| update-data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v3 | |
| with: | |
| version: 9.1.0 | |
| package_json_file: webclient/package.json | |
| run_install: | | |
| - cwd: webclient | |
| args: [--frozen-lockfile, --strict-peer-dependencies] | |
| - args: [--global, openapi-format] | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install python dependencies | |
| run: pip install -r data/requirements.txt -r requirements-dev.txt | |
| - run: rm -rf data/external/results/* | |
| - name: Download public_transport data | |
| continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: | | |
| PYTHONPATH=$PYTHONPATH:.. python3 scrapers/public_transport.py | |
| working-directory: data/external | |
| - name: Download NAT data | |
| continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: | | |
| PYTHONPATH=$PYTHONPATH:.. python3 scrapers/nat.py | |
| working-directory: data/external | |
| - name: Download tumonline data | |
| continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: PYTHONPATH=$PYTHONPATH:.. python3 scrapers/tumonline.py | |
| env: | |
| CONNECTUM_OAUTH_CLIENT_ID: ${{ secrets.CONECTUM_OAUTH_CLIENT_ID }} | |
| CONNECTUM_OAUTH_CLIENT_SECRET: ${{ secrets.CONECTUM_OAUTH_CLIENT_SECRET }} | |
| working-directory: data/external | |
| - name: Download mytum data | |
| continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: PYTHONPATH=$PYTHONPATH:.. python3 scrapers/roomfinder.py | |
| env: | |
| TQDM_MININTERVAL: 100 | |
| working-directory: data/external | |
| - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: ls -lah data/external/results | |
| - continue-on-error: true # a PR deleting all data will be created if this fails => fail obvious | |
| run: du -sh data/external/results/* | |
| - name: Fix linting errors | |
| continue-on-error: true | |
| uses: pre-commit/action@v3.0.1 | |
| - name: Create Pull Request | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| add-paths: data/external/results | |
| commit-message: Synced the data with upstream | |
| title: "[data-update] Synced the data with upstream datasources" | |
| body: | | |
| PR comes from [this Workflow Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| - Updated the data with upstream sources | |
| - Auto-generated by [create-pull-request][1] | |
| [ 1 ]: https://github.com/peter-evans/create-pull-request | |
| assignees: CommanderStorm | |
| reviewers: CommanderStorm | |
| branch: chore/data-update | |
| delete-branch: true |