diff --git a/cloudquery/sdk/scalar/scalar_factory.py b/cloudquery/sdk/scalar/scalar_factory.py index 4b390c1..6eabaca 100644 --- a/cloudquery/sdk/scalar/scalar_factory.py +++ b/cloudquery/sdk/scalar/scalar_factory.py @@ -1,5 +1,6 @@ import pyarrow as pa -from .scalar import ScalarInvalidTypeError + +from cloudquery.sdk.types import UUIDType, JSONType from .binary import Binary from .bool import Bool from .date32 import Date32 @@ -7,11 +8,11 @@ from .float import Float from .int import Int from .list import List +from .scalar import ScalarInvalidTypeError from .string import String from .timestamp import Timestamp from .uint import Uint from .uuid import UUID -from cloudquery.sdk.types import UUIDType, JSONType class ScalarFactory: @@ -85,9 +86,9 @@ def new_scalar(self, dt: pa.DataType): # return () elif dt_id == pa.types.lib.Type_TIMESTAMP: return Timestamp() - elif dt == UUIDType: + elif dt == UUIDType(): return UUID() - elif dt == JSONType: + elif dt == JSONType(): return String() else: raise ScalarInvalidTypeError("Invalid type {} for scalar".format(dt)) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 8f713ba..ae9b7ba 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -1,18 +1,16 @@ -from typing import List, Generator, Any import queue -import time +from concurrent import futures +from typing import List, Generator, Any + import structlog -from enum import Enum -from cloudquery.sdk.schema import Table, Resource + from cloudquery.sdk.message import ( SyncMessage, SyncInsertMessage, SyncMigrateTableMessage, ) -from concurrent import futures -from typing import Generator +from cloudquery.sdk.schema import Resource from .table_resolver import TableResolver -import traceback QUEUE_PER_WORKER = 100 @@ -192,4 +190,4 @@ def sync( break continue yield message - thread.shutdown() + thread.shutdown(wait=True) diff --git a/requirements.txt b/requirements.txt index 9b267c9..945c9b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1 @@ -cloudquery-plugin-pb==0.0.14 -exceptiongroup==1.1.2 -black==23.7.0 -grpcio==1.56.2 -grpcio-tools==1.56.2 -iniconfig==2.0.0 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -numpy==1.25.2 -packaging==23.1 -pandas==2.0.3 -pluggy==1.2.0 -protobuf==4.23.4 -pyarrow==12.0.1 -pytest==7.4.0 -python-dateutil==2.8.2 -pytz==2023.3 -six==1.16.0 -structlog==23.1.0 -tomli==2.0.1 -tzdata==2023.3 +. \ No newline at end of file diff --git a/setup.py b/setup.py index 2be00b2..9d8703f 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,26 @@ dependencies = [ "cloudquery-plugin-pb==0.0.14", - "pyarrow==12.0.1", + "exceptiongroup==1.1.2", + "black==23.7.0", + "grpcio==1.56.2", + "grpcio-tools==1.56.2", + "iniconfig==2.0.0", "Jinja2==3.1.2", - "structlog==23.1.0", + "MarkupSafe==2.1.3", + "numpy==1.25.2", + "packaging==23.1", "pandas==2.0.3", + "pluggy==1.2.0", + "protobuf==4.23.4", + "pyarrow==12.0.1", + "pytest==7.4.0", + "python-dateutil==2.8.2", + "pytz==2023.3", + "six==1.16.0", + "structlog==23.1.0", + "tomli==2.0.1", + "tzdata==2023.3", ] url = "https://github.com/cloudquery/plugin-sdk-python" diff --git a/tests/types/__init__.py b/tests/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/types/json.py b/tests/types/json.py new file mode 100644 index 0000000..f0ba704 --- /dev/null +++ b/tests/types/json.py @@ -0,0 +1,7 @@ +from cloudquery.sdk.types import JSONType + + +def test_json_type(): + j = JSONType() + # test equality + assert j == JSONType()