Sammyt/support python3.7

.github/workflows/driftdb-tests.yml

@@ -24,18 +24,13 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: 3.7
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip install -r requirements.txt
           pip install -r requirements-dev.txt
 
       - name: Run tests
         run: python -m unittest discover -s tests
-
-      - name: Mypy install types
-        run: mypy --install-types --non-interactive driftdb
-
-      - name: Run lint
-        run: mypy driftdb

tools/driftdb/.dockerignore

@@ -0,0 +1,37 @@
+# Include any files or directories that you don't want to be copied to your
+# container here (e.g., local build artifacts, temporary files, etc.).
+#
+# For more help, visit the .dockerignore file reference guide at
+# https://docs.docker.com/go/build-context-dockerignore/
+
+**/.DS_Store
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+**/dist
+**/build
+

tools/driftdb/Dockerfile

@@ -0,0 +1,58 @@
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/go/dockerfile-reference/
+
+ARG PYTHON_VERSION=3.7.17
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Update packages and install Git
+RUN apt-get update \
+    && apt-get install -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+# Create a non-privileged user that the app will run under.
+# See https://docs.docker.com/go/dockerfile-user-best-practices/
+ARG UID=10001
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    appuser
+
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.cache/pip to speed up subsequent builds.
+# Leverage a bind mount to requirements.txt to avoid having to copy them into
+# into this layer.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    python -m pip install -r requirements.txt
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
+    python -m pip install -r requirements-dev.txt
+
+# Switch to the non-privileged user to run the application.
+USER appuser
+
+COPY --chown=appuser:appuser . /app
+
+
+# Copy the source code into the container.
+COPY . .
+
+# Run the application.
+CMD bash -c "python -m driftdb.entrypoint --version && tail -f /dev/null"

tools/driftdb/README.Docker.md

@@ -0,0 +1,3 @@
+### Run the commands from the application
+
+`docker compose run --rm --build driftdb-cli python -m driftdb.entrypoint seed create`

tools/driftdb/compose.yaml

@@ -0,0 +1,4 @@
+services:
+  driftdb-cli:
+    build:
+      context: .

tools/driftdb/driftdb/alerting/handlers.py

@@ -1,7 +1,7 @@
 import traceback
-from typing import Callable, List
 
 import pandas as pd
+from typing_extensions import Callable, List
 
 from ..dataframe.detect_outliers import detect_outliers
 from ..logger import get_logger

tools/driftdb/driftdb/alerting/interface.py

@@ -1,6 +1,5 @@
-from typing import TypedDict
-
 import pandas as pd
+from typing_extensions import TypedDict
 
 
 class DriftSummary(TypedDict):

tools/driftdb/driftdb/cli/common.py

@@ -1,9 +1,7 @@
-from datetime import datetime
-from typing import List
-
 import inquirer
 import pandas as pd
 import typer
+from typing_extensions import List
 
 
 def prompt_from_list(prompt: str, choices: list):

tools/driftdb/driftdb/connectors/abstract_connector.py

@@ -1,8 +1,8 @@
 from abc import ABC, abstractmethod
 from datetime import datetime, timezone
-from typing import Dict, Optional
 
 import pandas as pd
+from typing_extensions import Dict, Optional
 
 from ..alerting import DriftHandler, NewDataHandler, auto_merge_drift, null_new_data_handler
 from ..dataframe.dataframe_update_breakdown import DataFrameUpdate, dataframe_update_breakdown

tools/driftdb/driftdb/connectors/github_connector.py

@@ -1,10 +1,10 @@
 import time
 from datetime import datetime
-from typing import Dict, List, Optional
 
 import pandas as pd
 from github import ContentFile, GithubException, Repository
 from github.MainClass import Github
+from typing_extensions import Dict, List, Optional
 
 from ..alerting import DriftEvaluatorContext, drift_summary_to_string
 from ..dataframe.dataframe_update_breakdown import DataFrameUpdate

tools/driftdb/driftdb/connectors/local_connector.py

@@ -1,14 +1,14 @@
 import os
 from datetime import datetime
-from typing import Dict, Iterator, Optional, Tuple
 
 import pandas as pd
 from git import Repo
 from git.objects.commit import Commit
+from typing_extensions import Dict, Iterator, Optional, Tuple
 
-from ..dataframe.dataframe_update_breakdown import DataFrameUpdate
 from ..alerting.handlers import drift_summary_to_string
 from ..alerting.interface import DriftEvaluation, DriftEvaluatorContext
+from ..dataframe.dataframe_update_breakdown import DataFrameUpdate
 from ..logger import get_logger
 from .abstract_connector import AbstractConnector
 
@@ -18,9 +18,9 @@
 class LocalConnector(AbstractConnector):
     home_dir = os.path.expanduser("~")
     datadrift_dir = os.path.join(home_dir, ".datadrift")
-    os.makedirs(datadrift_dir, exist_ok=True)
 
     def __init__(self, store_name="default"):
+        os.makedirs(self.datadrift_dir, exist_ok=True)
         self.store_name = store_name
         self.repo = self.get_or_init_repo(store_name=self.store_name)
         self.store_dir = self.repo.working_dir

tools/driftdb/driftdb/dataframe/dataframe_update_breakdown.py

@@ -1,7 +1,7 @@
 from enum import Enum
-from typing import Dict, Optional, Union
 
 import pandas as pd
+from typing_extensions import Dict, Optional, Union
 
 from ..alerting import (
     DriftEvaluation,

tools/driftdb/driftdb/dataframe/detect_outliers.py

@@ -1,6 +1,5 @@
-from typing import List, Optional
-
 import pandas as pd
+from typing_extensions import List, Optional
 
 
 def detect_outliers(

tools/driftdb/driftdb/dbt/snapshot.py

@@ -1,5 +1,6 @@
 import json
-from typing import List, TypedDict
+
+from typing_extensions import List, TypedDict
 
 from ..cli.common import dbt_adapter_query
 

tools/driftdb/driftdb/query_builder/query_builder.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Sequence
+from typing_extensions import List, Optional, Sequence
 
 
 def build_query(

tools/driftdb/requirements-dev.txt

@@ -1,12 +0,0 @@
-black
-click
-dbt-core
-Faker
-GitPython
-inquirer
-isort
-mypy
-pandas
-pandas-stubs
-PyGithub
-typer[all]

tools/driftdb/requirements.txt

@@ -0,0 +1,12 @@
+black
+click
+dbt-core
+Faker
+GitPython
+inquirer
+isort
+pandas
+pandas-stubs
+PyGithub
+typing_extensions
+typer[all]