Initial commit of adapter + dialect (#1)

Author: cancan101

.coveragerc

@@ -0,0 +1,15 @@
+[run]
+omit =
+    setup.py
+
+[report]
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise NotImplementedError
+
+    # These lines are not run by tests
+    if typing.TYPE_CHECKING:
+    if TYPE_CHECKING:

.flake8

@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203,R504
+exclude =
+    # No need to traverse our git directory
+    .git,
+    # There's no value in checking cache directories
+    __pycache__,
+per-file-ignores =
+    tests/**:S101

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: weekly
+  open-pull-requests-limit: 10
+  allow:
+  - dependency-type: direct
+  - dependency-type: indirect

.github/workflows/main.yml

@@ -0,0 +1,82 @@
+name: CI
+
+on:
+  push:
+    branches: ["main"]
+    tags:
+      - "*"
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "2"
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: "3.9"
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install -r requirements-dev.txt
+
+      - name: Test with black
+        run: |
+          black . --check --target-version py39
+
+      - name: Test with flake8
+        run: |
+          flake8 .
+
+      # Currently the version of apsw specified in requirements.txt file is quite old
+      # https://github.com/rogerbinns/apsw/issues/310
+      - name: Fix dependencies
+        run: |
+          pip uninstall -y apsw
+          pip install "apsw==3.36.0.post1"
+
+      - name: Install self
+        run: |
+          pip install -e .
+
+      - name: Test with pytest
+        run: |
+          pytest --cov=./ --cov-report=xml
+
+      - name: Set up mypy cache
+        uses: actions/cache@v2
+        with:
+          path: .mypy_cache
+          key: mypy1-${{ hashFiles('./airtabledb/**/*.py') }}-${{ hashFiles('./tests/**/*.py') }}
+          restore-keys: mypy1-
+
+      - name: Test with mypy
+        run: |
+          mypy .
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: true

.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+ensure_newline_before_comments = True
+line_length=88

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: 21.12b0
+    hooks:
+      - id: black
+        language_version: python3
+        args: [--target-version=py39]
+  - repo: https://github.com/PyCQA/isort
+    rev: "5.10.1"
+    hooks:
+      - id: isort
+        language_version: python3
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: "4.0.1"
+    hooks:
+      - id: flake8
+        language_version: python3
+        additional_dependencies:
+          [
+            flake8-bandit==3.0.0,
+            # No need for flake8-black
+            flake8-bugbear==22.1.11,
+            flake8-datetimez==20.10.0,
+            flake8-debugger==4.0.0,
+            # No need for flake8-isort
+            flake8-print==4.0.0,
+            flake8-return==1.1.3,
+          ]

README.md

@@ -1,2 +1,13 @@
 # airtable-db-api
 A Python DB API 2.0 for Airtable
+
+## Installation
+### Python
+```bash
+$ pip install -r requirements-dev.txt
+```
+
+### `pre-commit`
+```bash
+$ pre-commit install
+```

airtabledb/__init__.py

@@ -0,0 +1,82 @@
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+from pyairtable import Table
+from shillelagh.adapters.base import Adapter
+from shillelagh.fields import Field, Filter, String
+from shillelagh.typing import RequestedOrder
+
+from .types import BaseMetadata
+
+# -----------------------------------------------------------------------------
+
+
+class AirtableAdapter(Adapter):
+    safe = True
+
+    def __init__(
+        self,
+        table: str,
+        base_id: str,
+        api_key: str,
+        base_metadata: BaseMetadata,
+    ):
+        super().__init__()
+
+        self.table = table
+        self.base_metadata = base_metadata
+
+        self._table_api = Table(api_key, base_id, table)
+
+        fields: List[str]
+        if self.base_metadata is not None:
+            # TODO(cancan101): Better error handling here
+            # We search by name here.
+            # Alternatively we could have the user specify the name as an id
+            table_metadata = [
+                table_value
+                for table_value in self.base_metadata.values()
+                if table_value["name"] == table
+            ][0]
+            columns_metadata = table_metadata["columns"]
+            fields = [col["name"] for col in columns_metadata]
+            self.strict_col = True
+        else:
+            # This introspects the first row in the table.
+            # This is super not reliable
+            # as Airtable removes the key if the value is empty.
+            # We should probably look at more than one entry.
+            fields = self._table_api.first()["fields"]
+            self.strict_col = False
+
+        # TODO(cancan101): parse out types
+        self.columns: Dict[str, Field] = dict(
+            {k: String() for k in fields}, id=String()
+        )
+
+    @staticmethod
+    def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]:
+        # TODO the slow path here could connect to the GQL Server
+        return True
+
+    @staticmethod
+    def parse_uri(table: str) -> Tuple[str]:
+        return (table,)
+
+    def get_columns(self) -> Dict[str, Field]:
+        return self.columns
+
+    def get_data(
+        self,
+        bounds: Dict[str, Filter],
+        order: List[Tuple[str, RequestedOrder]],
+    ) -> Iterator[Dict[str, Any]]:
+        for page in self._table_api.iterate():
+            for result in page:
+                yield dict(
+                    {
+                        k: v
+                        for k, v in result["fields"].items()
+                        if self.strict_col or k in self.columns
+                    },
+                    id=result["id"],
+                )

airtabledb/adapter.py

@@ -0,0 +1,62 @@
+from typing import Any, Dict, List, Tuple
+
+from shillelagh.backends.apsw.dialects.base import APSWDialect
+from sqlalchemy.engine import Connection
+from sqlalchemy.engine.url import URL
+
+from .types import BaseMetadata
+
+# -----------------------------------------------------------------------------
+
+ADAPTER_NAME = "airtable"
+
+# -----------------------------------------------------------------------------
+
+
+class APSWAirtableDialect(APSWDialect):
+    supports_statement_cache = True
+
+    def __init__(
+        self,
+        airtable_api_key: str = None,
+        base_metadata: BaseMetadata = None,
+        **kwargs: Any,
+    ):
+        # We tell Shillelagh that this dialect supports just one adapter
+        super().__init__(safe=True, adapters=[ADAPTER_NAME], **kwargs)
+
+        self.airtable_api_key = airtable_api_key
+        self.base_metadata = base_metadata
+
+    def get_table_names(
+        self, connection: Connection, schema: str = None, **kwargs: Any
+    ) -> List[str]:
+        if self.base_metadata is not None:
+            return [table["name"] for table in self.base_metadata.values()]
+        return []
+
+    def create_connect_args(
+        self,
+        url: URL,
+    ) -> Tuple[Tuple[()], Dict[str, Any]]:
+        args, kwargs = super().create_connect_args(url)
+
+        if "adapter_kwargs" in kwargs and kwargs["adapter_kwargs"] != {}:
+            raise ValueError(
+                f"Unexpected adapter_kwargs found: {kwargs['adapter_kwargs']}"
+            )
+
+        if url.password and self.airtable_api_key:
+            raise ValueError("Both password and airtable_api_key were provided")
+
+        # At some point we might have args
+        adapter_kwargs = {
+            ADAPTER_NAME: {
+                "api_key": self.airtable_api_key or url.password,
+                "base_id": url.host,
+                "base_metadata": self.base_metadata,
+            }
+        }
+
+        # this seems gross, esp the path override. unclear why memory has to be set here
+        return args, {**kwargs, "path": ":memory:", "adapter_kwargs": adapter_kwargs}