Merge pull request #4 from allenai/refactor-upgrade

refactor

.dockerignore

@@ -1,2 +1,3 @@
 data/*
-.env/*
+.env/*
+**/.git

.github/workflows/build_and_push_docker_images.yml

@@ -4,79 +4,52 @@ on:
   push:
 
 env:
-  GHCR: ghcr.io
-  GHCR_IMAGE_NAME: ${{ github.repository }}
-  GCR: gcr.io
-  GCR_IMAGE_NAME: skylight-docker-images/gh-actions/${{ github.repository }}/${{ github.ref_name }}
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-and-push-image-ghcr-vessel-detection-download:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v3
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract GHCR metadata (tags, labels) for Docker vessel-detection-download
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection-download
-          tags: |
-            type=sha,format=short
-            type=raw,value=latest
-
-      - name: Build and push Docker image vessel-detection-download to GHCR
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: docker/downloader.dockerfile
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+  build-and-push-image:
+    runs-on: ubuntu-20.04
 
-  build-and-push-image-ghcr-vessel-detection-train:
-    runs-on: ubuntu-latest
     steps:
+      - name: Check disk space
+        run: df . -h
+      - name: Free disk space
+        run: |
+          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          sudo rm -rf \
+            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+            /usr/lib/jvm || true
+          echo "some directories deleted"
+          sudo apt install aptitude -y >/dev/null 2>&1
+          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
+            imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete libmysqlclient \
+            unixodbc-dev yarn chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
+            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
+            -y -f >/dev/null 2>&1
+          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1
+          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
+          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
+          sudo apt-get autoremove -y >/dev/null 2>&1
+          sudo apt-get autoclean -y >/dev/null 2>&1
+          echo "some packages purged"
+      - name: Check disk space
+        run: |
+          sudo dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -nr | head
+          df . -h
+          sudo du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
       - name: Checkout source code
         uses: actions/checkout@v3
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract GHCR metadata (tags, labels) for Docker vessel-detection-train
-        id: meta
-        uses: docker/metadata-action@v4
         with:
-          images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection-train
-          tags: |
-            type=sha,format=short
-            type=raw,value=latest
-
-      - name: Build and push Docker image vessel-detection-train to GHCR
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          file: docker/train.dockerfile
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-  build-and-push-image-ghcr-vessel-detection:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v3
+          lfs: true
 
       - name: Log in to the Container registry
         uses: docker/login-action@v2
@@ -85,20 +58,19 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract GHCR metadata (tags, labels) for Docker vessel-detection
+      - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=sha,format=short
-            type=raw,value=latest
+            type=raw,value=latest,enable={{is_default_branch}}
 
-      - name: Build and push Docker image vessel-detection to GHCR
+      - name: Build and push Docker image
         uses: docker/build-push-action@v3
         with:
           context: .
-          file: docker/inference.dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yaml

@@ -10,11 +10,46 @@ on:
 
 jobs:
   unit-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       COMPOSE_FILE: docker-compose.yml
 
     steps:
+      - name: Check disk space
+        run: df . -h
+      - name: Free disk space
+        run: |
+          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          sudo rm -rf \
+            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+            /usr/lib/jvm || true
+          echo "some directories deleted"
+          sudo apt install aptitude -y >/dev/null 2>&1
+          sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
+            esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
+            imagemagick \
+            libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
+            mercurial apt-transport-https mono-complete libmysqlclient \
+            unixodbc-dev yarn chrpath libssl-dev libxft-dev \
+            libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
+            snmp pollinate libpq-dev postgresql-client powershell ruby-full \
+            sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
+            -y -f >/dev/null 2>&1
+          sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
+          sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1
+          sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
+          sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
+          sudo apt-get autoremove -y >/dev/null 2>&1
+          sudo apt-get autoclean -y >/dev/null 2>&1
+          echo "some packages purged"
+      - name: Check disk space
+        run: |
+          sudo dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -nr | head
+          df . -h
+          sudo du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
+
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
@@ -38,4 +73,4 @@ jobs:
       - name: Run unit and integration tests
 
         run: |
-          docker compose run test
+          docker compose run test pytest --ignore=/src/tests/test_main.py -v

.gitignore

@@ -1,3 +1,8 @@
 *.pyc
 .vscode
-.env
+.env
+**/**/.DS_Store
+**/**/nohup.out
+**/.ipynb_checkpoints/
+**/.venv/
+**/*.ipynb

.pre-commit-config.yaml

@@ -0,0 +1,76 @@
+exclude: ais/data/machine_annotations/tsr/standard_rendezvous.csv
+exclude_types: ["*.geojson"]
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-json
+      - id: mixed-line-ending
+      - id: requirements-txt-fixer
+      - id: pretty-format-json
+        args: ["--autofix"]
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-byte-order-marker
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: debug-statements
+      - id: detect-aws-credentials
+        args: [--allow-missing-credentials]
+      - id: detect-private-key
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.1.1
+    hooks:
+      - id: mypy
+        args:
+          [
+            --install-types,
+            --ignore-missing-imports,
+            --disallow-untyped-defs,
+            --ignore-missing-imports,
+            --non-interactive,
+          ]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: detect-private-key
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files
+  - repo: https://github.com/PyCQA/bandit
+    rev: "1.7.5"
+    hooks:
+      - id: bandit
+        exclude: ^tests/
+        args:
+          - -s
+          - B101
+  - repo: local
+    hooks:
+      - id: interrogate
+        name: interrogate
+        language: system
+        entry: interrogate
+        types: [python]
+        args:
+          [
+            --ignore-init-method,
+            --ignore-init-module,
+            -p,
+            -vv,
+            ais,
+            --fail-under=80,
+          ]
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: "v0.0.257"
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --ignore, "E501"]

Dockerfile

@@ -0,0 +1,39 @@
+# Base Image
+FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04@sha256:9ccfe38d9cb31ae23f6f8f9595b450565da18399c2e71ebc9a5c079f786319e3
+
+# Environment Variables
+ENV WANDB_API_KEY=$WANDB_API_KEY \
+    RUN_TAG=$RUN_TAG \
+    WANDB_MODE=$WANDB_MODE \
+    WANDB_START_METHOD="thread" \
+    WANDB_PROJECT="vessel-detection" \
+    COPERNICUS_USERNAME=$COPERNICUS_USERNAME \
+    COPERNICUS_PASSWORD=$COPERNICUS_PASSWORD
+
+# System Dependencies and Cleanup
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y tzdata && \
+    apt-get install -y software-properties-common ffmpeg libsm6 libxext6 libhdf5-serial-dev netcdf-bin libnetcdf-dev && \
+    add-apt-repository ppa:ubuntugis/ubuntugis-unstable && \
+    apt-get update && \
+    apt-get install -y curl build-essential gdal-bin libgdal-dev libpq-dev python3-gdal python3-pip apt-transport-https ca-certificates gnupg && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy only the necessary files
+COPY requirements.txt /home/vessel_detection/requirements.txt
+
+# Install Python Packages
+RUN pip install --no-cache-dir -r /home/vessel_detection/requirements.txt
+
+# Set Working Directory and Prepare App
+WORKDIR /home/vessel_detection/src
+COPY src /home/vessel_detection/src
+COPY tests /home/vessel_detection/src/
+RUN mkdir -p /root/.cache/torch/hub/checkpoints/
+COPY torch_weights/swin_v2_s-637d8ceb.pth /root/.cache/torch/hub/checkpoints/swin_v2_s-637d8ceb.pth
+COPY torch_weights/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth
+COPY torch_weights/swin_v2_t-b137f0e2.pth /root/.cache/torch/hub/checkpoints/swin_v2_t-b137f0e2.pth
+
+# CMD
+CMD ["python3", "main.py"]