From 410f2f92490c1d540370a571bc6c0283f1f1347d Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:19:03 +0300 Subject: [PATCH 1/5] fix: Log error on table resolver exception/error --- cloudquery/sdk/scheduler/scheduler.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index ae9b7ba..2e2f12c 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -93,7 +93,7 @@ def resolve_table( try: if depth == 0: self._logger.info( - "table resolver started", table=resolver.table.name, depth=depth + "table resolver started what", table=resolver.table.name, depth=depth ) else: self._logger.debug( @@ -101,6 +101,7 @@ def resolve_table( ) total_resources = 0 for item in resolver.resolve(client, parent_item): + self._logger.info("resolving resource") try: resource = self.resolve_resource( resolver, client, parent_item, item @@ -113,6 +114,7 @@ def resolve_table( exception=e, ) continue + self._logger.info("resolved resource") res.put(SyncInsertMessage(resource.to_arrow_record())) for child_resolvers in resolver.child_resolvers: self._pools[depth + 1].submit( @@ -125,6 +127,7 @@ def resolve_table( ) table_resolvers_started += 1 total_resources += 1 + self._logger.info("what end") if depth == 0: self._logger.info( "table resolver finished successfully", @@ -132,18 +135,18 @@ def resolve_table( depth=depth, ) else: - self._logger.debug( + self._logger.info( "table resolver finished successfully", table=resolver.table.name, depth=depth, ) except Exception as e: - self._logger.error( - "table resolver finished with error", - table=resolver.table.name, - depth=depth, - exception=e, - ) + self._logger.error( + "table resolver finished with error", + table=resolver.table.name, + depth=depth, + exec_info=e, + ) finally: res.put(TableResolverStarted(count=table_resolvers_started)) res.put(TableResolverFinished()) @@ -182,11 +185,13 @@ def sync( if type(message) == TableResolverStarted: total_table_resolvers += message.count if total_table_resolvers == finished_table_resolvers: + self._logger.info("started: all table resolvers finished", total_table_resolvers=total_table_resolvers, finished_table_resolvers=finished_table_resolvers) break continue elif type(message) == TableResolverFinished: finished_table_resolvers += 1 if total_table_resolvers == finished_table_resolvers: + self._logger.info("finished: all table resolvers finished", total_table_resolvers=total_table_resolvers, finished_table_resolvers=finished_table_resolvers) break continue yield message From d3d156ad4665d35cdef514833a97b7fa5bbefc73 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:20:59 +0300 Subject: [PATCH 2/5] remove debug lines --- cloudquery/sdk/scheduler/scheduler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 2e2f12c..2048f17 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -93,7 +93,7 @@ def resolve_table( try: if depth == 0: self._logger.info( - "table resolver started what", table=resolver.table.name, depth=depth + "table resolver started", table=resolver.table.name, depth=depth ) else: self._logger.debug( @@ -185,13 +185,11 @@ def sync( if type(message) == TableResolverStarted: total_table_resolvers += message.count if total_table_resolvers == finished_table_resolvers: - self._logger.info("started: all table resolvers finished", total_table_resolvers=total_table_resolvers, finished_table_resolvers=finished_table_resolvers) break continue elif type(message) == TableResolverFinished: finished_table_resolvers += 1 if total_table_resolvers == finished_table_resolvers: - self._logger.info("finished: all table resolvers finished", total_table_resolvers=total_table_resolvers, finished_table_resolvers=finished_table_resolvers) break continue yield message From 9883065a517b6bfbb58c8c3b37db88d0d05f74f7 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:21:42 +0300 Subject: [PATCH 3/5] remove resourced resources --- cloudquery/sdk/scheduler/scheduler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 2048f17..a727f18 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -101,7 +101,6 @@ def resolve_table( ) total_resources = 0 for item in resolver.resolve(client, parent_item): - self._logger.info("resolving resource") try: resource = self.resolve_resource( resolver, client, parent_item, item @@ -114,7 +113,6 @@ def resolve_table( exception=e, ) continue - self._logger.info("resolved resource") res.put(SyncInsertMessage(resource.to_arrow_record())) for child_resolvers in resolver.child_resolvers: self._pools[depth + 1].submit( @@ -127,7 +125,6 @@ def resolve_table( ) table_resolvers_started += 1 total_resources += 1 - self._logger.info("what end") if depth == 0: self._logger.info( "table resolver finished successfully", From bf807b56a5544bad7a2ed0468c0cb2a7246931d7 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:22:59 +0300 Subject: [PATCH 4/5] fmt --- cloudquery/sdk/scheduler/scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index a727f18..996d1ff 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -138,12 +138,12 @@ def resolve_table( depth=depth, ) except Exception as e: - self._logger.error( - "table resolver finished with error", - table=resolver.table.name, - depth=depth, - exec_info=e, - ) + self._logger.error( + "table resolver finished with error", + table=resolver.table.name, + depth=depth, + exec_info=e, + ) finally: res.put(TableResolverStarted(count=table_resolvers_started)) res.put(TableResolverFinished()) From 84f7d696ff813b9ed2fd8fad491763f4be11ce2f Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:23:27 +0300 Subject: [PATCH 5/5] remove spam log line --- cloudquery/sdk/scheduler/scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index 996d1ff..073e01d 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -132,7 +132,7 @@ def resolve_table( depth=depth, ) else: - self._logger.info( + self._logger.debug( "table resolver finished successfully", table=resolver.table.name, depth=depth,