Improve type handling for errors, attachments, and multiple related

airtabledb/adapter.py

@@ -13,7 +13,7 @@
 
 from pyairtable import Table
 from shillelagh.adapters.base import Adapter
-from shillelagh.fields import Field, ISODate, Order, String
+from shillelagh.fields import Field, ISODate, ISODateTime, Order, String
 from shillelagh.filters import Equal, Filter, IsNotNull, IsNull, NotEqual, Range
 from shillelagh.typing import RequestedOrder
 
@@ -87,7 +87,12 @@ def __init__(
             fields = [col["name"] for col in columns_metadata]
             self.strict_col = True
 
-            columns = {k: _create_field(MaybeListString, {}) for k in fields}
+            # For now just set allow_multiple = True here
+            # as we are not introspecting meta
+            columns = {
+                k: _create_field(MaybeListString, {"allow_multiple": True})
+                for k in fields
+            }
 
         # Attempts introspection by looking at data.
         # This is super not reliable
@@ -129,7 +134,14 @@ def maybe_guess_field(field_name: str, values: List[Any]) -> FieldInfo:
                 for k, v in field_values.items()
             }
 
-        self.columns = dict(columns, id=String())
+        # Right now I don't know how to sort on the backend:
+        # https://community.airtable.com/t/sort-on-rest-api-by-createdtime-without-adding-new-column/
+        # TODO(cancan101): implement filtering for id + createdTime
+        # using special formula.
+        # See:
+        # https://support.airtable.com/hc/en-us/articles/360051564873-Record-ID
+        # https://support.airtable.com/hc/en-us/articles/203255215-Formula-field-reference#record_functions
+        self.columns = dict(columns, id=String(), createdTime=ISODateTime())
 
     @staticmethod
     def supports(uri: str, fast: bool = True, **kwargs: Any) -> Optional[bool]:
@@ -165,4 +177,5 @@ def get_data(
                         if self.strict_col or k in self.columns
                     },
                     id=result["id"],
+                    createdTime=result["createdTime"],
                 )

airtabledb/fields.py

@@ -1,3 +1,4 @@
+import json
 import math
 from typing import List, Optional, Union
 
@@ -8,29 +9,40 @@
 # -----------------------------------------------------------------------------
 
 
-class AirtableFloatType(TypedDict):
+class AirtableFloatTypeSpecial(TypedDict):
     specialValue: str
 
 
+class AirtableFloatTypeError(TypedDict):
+    error: str
+
+
+AirtableFloatType = Union[AirtableFloatTypeError, AirtableFloatTypeSpecial]
+
+
 AirtableRawNumericInputTypes = Union[int, float, AirtableFloatType]
 AirtableRawInputTypes = Union[str, AirtableRawNumericInputTypes]
 AirtableInputTypes = Union[AirtableRawInputTypes, List[AirtableRawInputTypes]]
 AirtablePrimitiveTypes = Union[str, int, float]
 
 
 SPECIAL_VALUE_KEY = "specialValue"
+ERROR_VALUE_KEY = "error"
 
-NAN_REPRESENTATION = AirtableFloatType(specialValue="NaN")
-INF_REPRESENTATION = AirtableFloatType(specialValue="Infinity")
-INF_NEG_REPRESENTATION = AirtableFloatType(specialValue="-Infinity")
+NAN_REPRESENTATION = AirtableFloatTypeSpecial(specialValue="NaN")
+INF_REPRESENTATION = AirtableFloatTypeSpecial(specialValue="Infinity")
+INF_NEG_REPRESENTATION = AirtableFloatTypeSpecial(specialValue="-Infinity")
+ERROR_REPRESENTATION = AirtableFloatTypeError(error="#ERROR")
 
 
 class MaybeList(Field[AirtableInputTypes, AirtablePrimitiveTypes]):  # type: ignore
-    def __init__(self, field: Field, **kwargs) -> None:
+    def __init__(self, field: Field, *, allow_multiple=False, **kwargs) -> None:
         super().__init__(**kwargs)
 
         self._scalar_handler = field
-        self._list_handler = OverList(field=self._scalar_handler)
+        self._list_handler = OverList(
+            field=self._scalar_handler, allow_multiple=allow_multiple
+        )
 
         self.type = self._scalar_handler.type
         self.db_api_type = self._scalar_handler.db_api_type
@@ -54,12 +66,24 @@ def __init__(self, **kwargs) -> None:
 class OverList(
     Field[List[AirtableRawInputTypes], AirtablePrimitiveTypes]  # type: ignore
 ):
-    def __init__(self, field: Field, **kwargs):
+    def __init__(self, field: Field, *, allow_multiple=False, **kwargs):
         super().__init__(**kwargs)
+
         self.field = field
+        self.allow_multiple = allow_multiple
+
         self.type = field.type
         self.db_api_type = field.db_api_type
 
+    def _parse_item(
+        self, value: AirtableRawInputTypes
+    ) -> Optional[AirtablePrimitiveTypes]:
+        # TODO(cancan101): Do we have to handle nested arrays?
+        # We handle dict here to allow for "special values"
+        if value is not None and not isinstance(value, (str, int, float, dict)):
+            raise TypeError(f"Unknown type: {type(value)}")
+        return self.field.parse(value)
+
     def parse(
         self, value: Optional[List[AirtableRawInputTypes]]
     ) -> Optional[AirtablePrimitiveTypes]:
@@ -71,12 +95,11 @@ def parse(
             return None
         elif len(value) == 1:
             ret = value[0]
-            # TODO(cancan101): Do we have to handle nested arrays?
-            # We handle dict here to allow for "special values"
-            if ret is not None and not isinstance(ret, (str, int, float, dict)):
-                raise TypeError(f"Unknown type: {type(ret)}")
-            return self.field.parse(ret)
+            return self._parse_item(ret)
         else:
+            if self.allow_multiple:
+                # TODO(cancan101): improvements to list formatting, None handling, etc:
+                return ", ".join(str(self._parse_item(v)) for v in value)
             raise ValueError("Unable to handle list of length > 1")
 
 
@@ -98,6 +121,9 @@ def parse(
                 return math.inf
             elif value == INF_NEG_REPRESENTATION:
                 return -math.inf
+            elif value == ERROR_REPRESENTATION:
+                # We could have mapped this to None as well
+                return math.nan
             else:
                 raise ValueError(f"Unknown float representation: {value}")
         return value
@@ -125,7 +151,14 @@ def parse(
             return self._string_handler.parse(value)
         elif isinstance(value, (int, float)):
             return self._float_handler.parse(value)
-        elif isinstance(value, dict) and len(value) == 1 and SPECIAL_VALUE_KEY in value:
+        elif (
+            isinstance(value, dict)
+            and len(value) == 1
+            and (SPECIAL_VALUE_KEY in value or ERROR_VALUE_KEY in value)
+        ):
             return self._float_handler.parse(value)
+        elif isinstance(value, dict) and "id" in value:
+            # e.g. Attachments
+            return json.dumps(value)
         else:
-            raise TypeError(f"Unknown type: {type(value)}")
+            raise TypeError(f"Unknown type: {type(value)} (value: {value})")

airtabledb/lib.py

@@ -24,12 +24,18 @@ def guess_field(values: List[Any]) -> FieldInfo:
             item_field_cls, item_field_kwargs = guess_field(
                 [v for vi in values for v in vi]
             )
-            return OverList, {"field": item_field_cls(**item_field_kwargs)}
+            # TODO(cancan101): for now, we always set allow_multiple
+            return OverList, {
+                "field": item_field_cls(**item_field_kwargs),
+                "allow_multiple": True,
+            }
     elif types == {float, int}:
         return AirtableFloat, {}
     elif types == {float, dict} or types == {int, dict} or types == {int, float, dict}:
         # This seems safest as there are cases where we get floats and ints
         # TODO(cancan101) check the dict
         return AirtableFloat, {}
 
-    return MaybeListString, {}
+    # Not totally sure when we hit this block
+    # TODO(cancan101): for now, we always set allow_multiple
+    return MaybeListString, {"allow_multiple": True}

tests/test_fields.py

@@ -55,7 +55,15 @@ def test_airtable_float_special():
         field.parse({"specialValue": "XXX"})
 
 
-def test_maybe_list():
+def test_airtable_float_error():
+    field = AirtableFloat()
+    assert math.isnan(field.parse({"error": "#ERROR"}))
+
+    with pytest.raises(ValueError):
+        field.parse({"error": "XXX"})
+
+
+def test_over_list():
     field = OverList(AirtableScalar())
     assert field.parse(None) is None
     assert field.parse([]) is None
@@ -78,6 +86,11 @@ def test_maybe_list():
         assert field.parse([1, 2])
 
 
+def test_over_list_multiple():
+    field = OverList(AirtableScalar(), allow_multiple=True)
+    assert field.parse([1, 2]) == "1, 2"
+
+
 def test_airtable_scalar():
     field = AirtableScalar()
 
@@ -87,5 +100,10 @@ def test_airtable_scalar():
     assert field.parse("a") == "a"
     assert math.isnan(field.parse({"specialValue": "NaN"}))
 
+    assert (
+        field.parse({"id": "attXXXX", "url": "https://foo.local"})
+        == '{"id": "attXXXX", "url": "https://foo.local"}'
+    )
+
     with pytest.raises(TypeError):
         assert field.parse({})