Apperception Version 2

.env

@@ -0,0 +1 @@
+mamba activate apperception

.env.leave

@@ -0,0 +1 @@
+mamba deactivate

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+ignore =
+  # let black handles line length
+  E501,
+  # let black handles spaces between colon (list slicing)
+  E203,
+  # let black handles line break before binary operator
+  W503,
+  # multiple '#' leading a line comment is ok
+  E266,
+  # module-level import not on top of file is ok
+  E402

.github/workflows/check.yml

@@ -0,0 +1,54 @@
+name: Type Check
+on:
+  push:
+
+jobs:
+  check:
+    name: Type Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GH_PAT || github.token }}
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10.6'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      # - name: Cache Poetry virtualenv
+      #   uses: actions/cache@v3
+      #   id: cached-poetry-dependencies 
+      #   with:
+      #     path: .venv
+      #     key: poetry-venv-${{ runner.os }}-py3.10-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install Dependencies
+        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction
+
+      - name: Check Python Version
+        run: poetry run python --version
+
+      - name: Check Installed Packages
+        run: poetry run pip list
+
+      - name: Type Check with mypy
+        if: always()
+        run: |
+          poetry run mypy apperception || true
+          (yes | poetry run mypy --install-types apperception) || true
+          poetry run mypy apperception
+
+      - name: Type Check with pyright
+        if: always()
+        run: poetry run pyright apperception
+        env:
+          PYRIGHT_PYTHON_FORCE_VERSION: latest

.github/workflows/clean-up-and-format.yml

@@ -0,0 +1,68 @@
+name: Clean Up and Format
+on:
+  push:
+    branches-ignore:
+      - 'main'
+      - 'dev'
+
+jobs:
+  clean-up:
+    name: Clean Up Python Notebooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GH_PAT || github.token }}
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Cleanup ipynb output
+        run: |
+          pip install --upgrade pip
+          pip install nbconvert
+          python -m jupyter nbconvert --clear-output --inplace benchmarks/**/*.ipynb benchmarks/*.ipynb
+          python -m jupyter nbconvert --clear-output --inplace scripts/**/*.ipynb scripts/*.ipynb
+          python -m jupyter nbconvert --clear-output --inplace playground/**/*.ipynb playground/*.ipynb
+
+      - name: Commit clean up changes
+        run: ./scripts/commit-and-push.sh "[CI] cleanup python notebooks"
+
+  format:
+    name: Format and Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GH_PAT || github.token }}
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install autoflake autopep8 black isort flake8
+
+      - name: Fix lint with autoflake
+        run: python -m autoflake apperception optimized_ingestion
+
+      - name: Fix lint with autopep8
+        run: python -m autopep8 apperception optimized_ingestion
+
+      - name: Format with black
+        run: python -m black --config pyproject.toml apperception
+
+      - name: Sort imports with isort
+        run: python -m isort apperception optimized_ingestion
+
+      - name: Commit formatted changes
+        run: ./scripts/commit-and-push.sh "[CI] format"
+
+      - name: Analyze the code with flake8
+        if: always()
+        run: python -m flake8 apperception optimized_ingestion --statistics

.github/workflows/examples.yml

@@ -0,0 +1,82 @@
+name: Generate Videos
+on:
+  push:
+
+jobs:
+  generate-videos:
+    name: Generate Videos
+    runs-on: ubuntu-latest
+    services:
+      mobilitydb:
+        image: mobilitydb/mobilitydb:14-3.2-1
+        ports:
+          - 25432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GH_PAT || github.token }}
+          submodules: recursive
+
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          init-shell: none
+          cache-downloads: true
+
+      - name: Check Python Versions
+        shell: micromamba-shell {0}
+        run: |
+          python --version
+          python3 --version
+          which python
+          which python3
+          which pip
+          which pip3
+          which poetry
+
+      - name: Install Dependencies
+        shell: micromamba-shell {0}
+        run: poetry install --no-interaction --without dev --with test
+
+      - name: Install lap (Hack)
+        shell: micromamba-shell {0}
+        run: |
+          pip install --upgrade pip
+          pip install lap
+
+      - name: Check Installed Packages
+        shell: micromamba-shell {0}
+        run: pip list
+
+      - name: Extend MobilityDB with User-Defined functions
+        shell: micromamba-shell {0}
+        run: |
+          pushd pg_extender
+          mv install.sql install.sql.bak
+          python ../scripts/generate_pg_extender.py
+          cat install.sql
+          psql -h localhost -p 25432 -d mobilitydb -U docker -c "SET client_min_messages TO WARNING;" -c "\i install.sql;"
+          mv install.sql.bak install.sql
+          popd
+        env:
+          PGPASSWORD: docker
+
+      - name: Ingest data
+        shell: micromamba-shell {0}
+        run: python ./scripts/ingest_road.py
+        env:
+          AP_PORT: 25432
+
+      - name: Generate Image data
+        shell: micromamba-shell {0}
+        run: python examples/inference_pipeline.py
+
+      - name: Commit generated examples
+        run: ./scripts/commit-and-push-if-latest.sh "[CI] generate video examples"

.github/workflows/test-pipeline.yml

@@ -0,0 +1,84 @@
+name: Test Pipeline
+on:
+  push:
+
+jobs:
+  test-pipeline:
+    name: Test Pipeline
+    runs-on: ubuntu-latest
+    services:
+      mobilitydb:
+        image: mobilitydb/mobilitydb:14-3.2-1
+        ports:
+          - 25440:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GH_PAT || github.token }}
+          submodules: recursive
+
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          init-shell: none
+          cache-downloads: true
+
+      - name: Check Python Versions
+        shell: micromamba-shell {0}
+        run: |
+          python --version
+          python3 --version
+          which python
+          which python3
+          which pip
+          which pip3
+          which poetry
+
+      - name: Install Dependencies
+        shell: micromamba-shell {0}
+        run: poetry install --no-interaction --without dev --with test
+
+      - name: Install lap (Hack)
+        shell: micromamba-shell {0}
+        run: |
+          pip install --upgrade pip
+          pip install lap
+
+      - name: Check Installed Packages
+        shell: micromamba-shell {0}
+        run: pip list
+
+      - name: Extend MobilityDB with User-Defined functions
+        shell: micromamba-shell {0}
+        run: |
+          pushd pg_extender
+          python ../scripts/generate_pg_extender.py
+          cat install.sql
+          psql -h localhost -p 25440 -d mobilitydb -U docker -c "SET client_min_messages TO WARNING;" -c "\i install.sql;"
+          popd
+        env:
+          PGPASSWORD: docker
+
+      - name: Ingest data
+        shell: micromamba-shell {0}
+        run: |
+          python ./scripts/ingest_road.py
+        env:
+          AP_PORT: 25440
+
+      - name: Generate Image data
+        shell: micromamba-shell {0}
+        run: python scripts/fake-images.py
+
+      - name: Unit Test
+        shell: micromamba-shell {0}
+        run: pytest tests-pipeline
+        env:
+          AP_PORT: 25440