From 3bd57676ecd08c888f4b1596fabbcb7ca7d98335 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 1 Aug 2023 15:45:51 +0100 Subject: [PATCH 1/2] Fix exception logging --- cloudquery/sdk/scheduler/scheduler.py | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 073e01d..46984e9 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -37,7 +37,7 @@ def __init__(self) -> None: class Scheduler: def __init__( - self, concurrency: int, queue_size: int = 0, max_depth: int = 3, logger=None + self, concurrency: int, queue_size: int = 0, max_depth: int = 3, logger=None ): self._queue = queue.Queue() self._max_depth = max_depth @@ -72,7 +72,7 @@ def shutdown(self): pool.shutdown() def resolve_resource( - self, resolver: TableResolver, client, parent: Resource, item: Any + self, resolver: TableResolver, client, parent: Resource, item: Any ) -> Resource: resource = Resource(resolver.table, parent, item) resolver.pre_resource_resolve(client, resource) @@ -82,12 +82,12 @@ def resolve_resource( return resource def resolve_table( - self, - resolver: TableResolver, - depth: int, - client, - parent_item: Resource, - res: queue.Queue, + self, + resolver: TableResolver, + depth: int, + client, + parent_item: Resource, + res: queue.Queue, ): table_resolvers_started = 0 try: @@ -110,7 +110,7 @@ def resolve_table( "failed to resolve resource", table=resolver.table.name, depth=depth, - exception=e, + exc_info=True, ) continue res.put(SyncInsertMessage(resource.to_arrow_record())) @@ -142,18 +142,18 @@ def resolve_table( "table resolver finished with error", table=resolver.table.name, depth=depth, - exec_info=e, + exc_info=True, ) finally: res.put(TableResolverStarted(count=table_resolvers_started)) res.put(TableResolverFinished()) def _sync( - self, - client, - resolvers: List[TableResolver], - res: queue.Queue, - deterministic_cq_id=False, + self, + client, + resolvers: List[TableResolver], + res: queue.Queue, + deterministic_cq_id=False, ): total_table_resolvers = 0 try: @@ -168,7 +168,7 @@ def _sync( res.put(TableResolverStarted(total_table_resolvers)) def sync( - self, client, resolvers: List[TableResolver], deterministic_cq_id=False + self, client, resolvers: List[TableResolver], deterministic_cq_id=False ) -> Generator[SyncMessage, None, None]: res = queue.Queue() for resolver in resolvers: From cd820fe206af0436b76017e1bd02f46dc939c8d1 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 1 Aug 2023 15:46:16 +0100 Subject: [PATCH 2/2] Format --- cloudquery/sdk/scheduler/scheduler.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 46984e9..d8e383c 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -37,7 +37,7 @@ def __init__(self) -> None: class Scheduler: def __init__( - self, concurrency: int, queue_size: int = 0, max_depth: int = 3, logger=None + self, concurrency: int, queue_size: int = 0, max_depth: int = 3, logger=None ): self._queue = queue.Queue() self._max_depth = max_depth @@ -72,7 +72,7 @@ def shutdown(self): pool.shutdown() def resolve_resource( - self, resolver: TableResolver, client, parent: Resource, item: Any + self, resolver: TableResolver, client, parent: Resource, item: Any ) -> Resource: resource = Resource(resolver.table, parent, item) resolver.pre_resource_resolve(client, resource) @@ -82,12 +82,12 @@ def resolve_resource( return resource def resolve_table( - self, - resolver: TableResolver, - depth: int, - client, - parent_item: Resource, - res: queue.Queue, + self, + resolver: TableResolver, + depth: int, + client, + parent_item: Resource, + res: queue.Queue, ): table_resolvers_started = 0 try: @@ -149,11 +149,11 @@ def resolve_table( res.put(TableResolverFinished()) def _sync( - self, - client, - resolvers: List[TableResolver], - res: queue.Queue, - deterministic_cq_id=False, + self, + client, + resolvers: List[TableResolver], + res: queue.Queue, + deterministic_cq_id=False, ): total_table_resolvers = 0 try: @@ -168,7 +168,7 @@ def _sync( res.put(TableResolverStarted(total_table_resolvers)) def sync( - self, client, resolvers: List[TableResolver], deterministic_cq_id=False + self, client, resolvers: List[TableResolver], deterministic_cq_id=False ) -> Generator[SyncMessage, None, None]: res = queue.Queue() for resolver in resolvers: