fix: Fix JSON and UUID type checking, add JSON test, consolidate setup.py

cloudquery/sdk/scalar/scalar_factory.py

@@ -1,17 +1,18 @@
 import pyarrow as pa
-from .scalar import ScalarInvalidTypeError
+
+from cloudquery.sdk.types import UUIDType, JSONType
 from .binary import Binary
 from .bool import Bool
 from .date32 import Date32
 from .date64 import Date64
 from .float import Float
 from .int import Int
 from .list import List
+from .scalar import ScalarInvalidTypeError
 from .string import String
 from .timestamp import Timestamp
 from .uint import Uint
 from .uuid import UUID
-from cloudquery.sdk.types import UUIDType, JSONType
 
 
 class ScalarFactory:
@@ -85,9 +86,9 @@ def new_scalar(self, dt: pa.DataType):
         #     return ()
         elif dt_id == pa.types.lib.Type_TIMESTAMP:
             return Timestamp()
-        elif dt == UUIDType:
+        elif dt == UUIDType():
             return UUID()
-        elif dt == JSONType:
+        elif dt == JSONType():
             return String()
         else:
             raise ScalarInvalidTypeError("Invalid type {} for scalar".format(dt))

cloudquery/sdk/scheduler/scheduler.py

@@ -1,18 +1,16 @@
-from typing import List, Generator, Any
 import queue
-import time
+from concurrent import futures
+from typing import List, Generator, Any
+
 import structlog
-from enum import Enum
-from cloudquery.sdk.schema import Table, Resource
+
 from cloudquery.sdk.message import (
     SyncMessage,
     SyncInsertMessage,
     SyncMigrateTableMessage,
 )
-from concurrent import futures
-from typing import Generator
+from cloudquery.sdk.schema import Resource
 from .table_resolver import TableResolver
-import traceback
 
 QUEUE_PER_WORKER = 100
 
@@ -192,4 +190,4 @@ def sync(
                     break
                 continue
             yield message
-        thread.shutdown()
+        thread.shutdown(wait=True)

requirements.txt

@@ -1,21 +1 @@
-cloudquery-plugin-pb==0.0.14
-exceptiongroup==1.1.2
-black==23.7.0
-grpcio==1.56.2
-grpcio-tools==1.56.2
-iniconfig==2.0.0
-Jinja2==3.1.2
-MarkupSafe==2.1.3
-numpy==1.25.2
-packaging==23.1
-pandas==2.0.3
-pluggy==1.2.0
-protobuf==4.23.4
-pyarrow==12.0.1
-pytest==7.4.0
-python-dateutil==2.8.2
-pytz==2023.3
-six==1.16.0
-structlog==23.1.0
-tomli==2.0.1
-tzdata==2023.3
+.

setup.py

@@ -11,10 +11,26 @@
 
 dependencies = [
     "cloudquery-plugin-pb==0.0.14",
-    "pyarrow==12.0.1",
+    "exceptiongroup==1.1.2",
+    "black==23.7.0",
+    "grpcio==1.56.2",
+    "grpcio-tools==1.56.2",
+    "iniconfig==2.0.0",
     "Jinja2==3.1.2",
-    "structlog==23.1.0",
+    "MarkupSafe==2.1.3",
+    "numpy==1.25.2",
+    "packaging==23.1",
     "pandas==2.0.3",
+    "pluggy==1.2.0",
+    "protobuf==4.23.4",
+    "pyarrow==12.0.1",
+    "pytest==7.4.0",
+    "python-dateutil==2.8.2",
+    "pytz==2023.3",
+    "six==1.16.0",
+    "structlog==23.1.0",
+    "tomli==2.0.1",
+    "tzdata==2023.3",
 ]
 url = "https://github.com/cloudquery/plugin-sdk-python"
 

tests/types/json.py

@@ -0,0 +1,7 @@
+from cloudquery.sdk.types import JSONType
+
+
+def test_json_type():
+    j = JSONType()
+    # test equality
+    assert j == JSONType()