From f241e2ba27cdae3b99d7d15e303021ee3cce7cae Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 2 Aug 2023 11:13:02 +0100 Subject: [PATCH 1/4] Fix docs signature --- cloudquery/sdk/serve/plugin.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudquery/sdk/serve/plugin.py b/cloudquery/sdk/serve/plugin.py index 7842425..d412a74 100644 --- a/cloudquery/sdk/serve/plugin.py +++ b/cloudquery/sdk/serve/plugin.py @@ -1,16 +1,16 @@ import argparse -import structlog from concurrent import futures import grpc +import structlog import sys - from cloudquery.discovery_v1 import discovery_pb2_grpc from cloudquery.plugin_v3 import plugin_pb2_grpc + from cloudquery.sdk.docs.generator import Generator from cloudquery.sdk.internal.servers.discovery_v1.discovery import DiscoveryServicer from cloudquery.sdk.internal.servers.plugin_v3 import PluginServicer -from cloudquery.sdk.plugin.plugin import Plugin +from cloudquery.sdk.plugin.plugin import Plugin, TableOptions DOC_FORMATS = ["json", "markdown"] @@ -105,6 +105,13 @@ def stop(self): def _generate_docs(self, args): print("Generating docs in format: " + args.format) generator = Generator( - self._plugin.name(), self._plugin.get_tables(tables=["*"], skip_tables=[]) + self._plugin.name(), + self._plugin.get_tables( + options=TableOptions( + tables=["*"], + skip_tables=[], + skip_dependent_tables=False, + ) + ), ) generator.generate(args.directory, args.format) From e776faad7881fbf1d52bee7205d0b146aa5f2d22 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 2 Aug 2023 15:03:35 +0100 Subject: [PATCH 2/4] Command line args --- cloudquery/sdk/plugin/plugin.py | 3 + cloudquery/sdk/scheduler/scheduler.py | 2 + cloudquery/sdk/serve/plugin.py | 90 +++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/cloudquery/sdk/plugin/plugin.py b/cloudquery/sdk/plugin/plugin.py index 4765cf2..7fc76f6 100644 --- a/cloudquery/sdk/plugin/plugin.py +++ b/cloudquery/sdk/plugin/plugin.py @@ -38,6 +38,9 @@ def __init__(self, name: str, version: str) -> None: def init(self, spec: bytes) -> None: pass + def set_logger(self, logger) -> None: + pass + def name(self) -> str: return self._name diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index c4e594d..8f8d111 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -39,6 +39,8 @@ def __init__( self._max_depth = max_depth if logger is None: self._logger = structlog.get_logger() + else: + self._logger = logger if concurrency <= 0: raise ValueError("concurrency must be greater than 0") if max_depth <= 0: diff --git a/cloudquery/sdk/serve/plugin.py b/cloudquery/sdk/serve/plugin.py index d412a74..e7b81d3 100644 --- a/cloudquery/sdk/serve/plugin.py +++ b/cloudquery/sdk/serve/plugin.py @@ -1,4 +1,7 @@ import argparse +import datetime +import logging +import os from concurrent import futures import grpc @@ -6,6 +9,7 @@ import sys from cloudquery.discovery_v1 import discovery_pb2_grpc from cloudquery.plugin_v3 import plugin_pb2_grpc +from structlog import wrap_logger from cloudquery.sdk.docs.generator import Generator from cloudquery.sdk.internal.servers.discovery_v1.discovery import DiscoveryServicer @@ -14,9 +18,70 @@ DOC_FORMATS = ["json", "markdown"] +_IS_WINDOWS = sys.platform == "win32" + +try: + import colorama +except ImportError: + colorama = None + +if _IS_WINDOWS: # pragma: no cover + # On Windows, use colors by default only if Colorama is installed. + _has_colors = colorama is not None +else: + # On other OSes, use colors by default. + _has_colors = True + + +def add_timestamp(_, __, event_dict): + event_dict["timestamp"] = datetime.datetime.utcnow().isoformat() + return event_dict + def get_logger(args): - log = structlog.get_logger(processors=[structlog.processors.JSONRenderer()]) + log_level_map = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, + } + + logging.basicConfig( + format="%(message)s", + stream=sys.stdout, + level=log_level_map.get(args.log_level.lower(), logging.INFO), + ) + + processors = [ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.dev.set_exc_info, + structlog.stdlib.filter_by_level, + structlog.processors.TimeStamper(fmt="%Y-%m-%dT%H:%M:%SZ", utc=True), + ] + if args.log_format == "text": + processors.append( + structlog.dev.ConsoleRenderer( + colors=os.environ.get("NO_COLOR", "") == "" + and ( + os.environ.get("FORCE_COLOR", "") != "" + or ( + _has_colors + and sys.stdout is not None + and hasattr(sys.stdout, "isatty") + and sys.stdout.isatty() + ) + ) + ) + ) + else: + processors.append(structlog.processors.JSONRenderer()) + + log = wrap_logger( + logging.getLogger(), + processors=processors + ) return log @@ -29,6 +94,9 @@ def run(self, args): subparsers = parser.add_subparsers(dest="command", required=True) serve_parser = subparsers.add_parser("serve", help="Start plugin server") + serve_parser.add_argument( + "--log-format", type=str, default="text", choices=["text", "json"], help="logging format" + ) serve_parser.add_argument( "--log-level", type=str, @@ -37,8 +105,16 @@ def run(self, args): help="log level", ) serve_parser.add_argument( - "--log-format", type=str, default="text", choices=["text", "json"] + "--no-sentry", action="store_true", help="disable sentry" + ) + serve_parser.add_argument( + "--otel-endpoint", type=str, default="", help="Open Telemetry HTTP collector endpoint" + ) + serve_parser.add_argument( + "--otel-endpoint-insecure", type=str, default="", + help="Open Telemetry HTTP collector endpoint (for development only)" ) + serve_parser.add_argument( "--address", type=str, @@ -86,16 +162,17 @@ def run(self, args): sys.exit(1) def _serve(self, args): - logger = get_logger(args) + log = get_logger(args) + self._plugin.set_logger(log) self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) discovery_pb2_grpc.add_DiscoveryServicer_to_server( DiscoveryServicer([3]), self._server ) plugin_pb2_grpc.add_PluginServicer_to_server( - PluginServicer(self._plugin, logger), self._server + PluginServicer(self._plugin, log), self._server ) self._server.add_insecure_port(args.address) - print("Starting server. Listening on " + args.address) + log.info("Starting server", address=args.address) self._server.start() self._server.wait_for_termination() @@ -103,7 +180,8 @@ def stop(self): self._server.stop(5) def _generate_docs(self, args): - print("Generating docs in format: " + args.format) + logger = get_logger(args) + logger.info("Generating docs", format=args.format) generator = Generator( self._plugin.name(), self._plugin.get_tables( From 7a8d3bc7155249848a51582c4f2df130fd341c87 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 2 Aug 2023 15:04:10 +0100 Subject: [PATCH 3/4] Fmt --- cloudquery/sdk/serve/plugin.py | 40 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/cloudquery/sdk/serve/plugin.py b/cloudquery/sdk/serve/plugin.py index e7b81d3..840ad49 100644 --- a/cloudquery/sdk/serve/plugin.py +++ b/cloudquery/sdk/serve/plugin.py @@ -64,24 +64,21 @@ def get_logger(args): processors.append( structlog.dev.ConsoleRenderer( colors=os.environ.get("NO_COLOR", "") == "" - and ( - os.environ.get("FORCE_COLOR", "") != "" - or ( - _has_colors - and sys.stdout is not None - and hasattr(sys.stdout, "isatty") - and sys.stdout.isatty() - ) - ) + and ( + os.environ.get("FORCE_COLOR", "") != "" + or ( + _has_colors + and sys.stdout is not None + and hasattr(sys.stdout, "isatty") + and sys.stdout.isatty() + ) + ) ) ) else: processors.append(structlog.processors.JSONRenderer()) - log = wrap_logger( - logging.getLogger(), - processors=processors - ) + log = wrap_logger(logging.getLogger(), processors=processors) return log @@ -95,7 +92,11 @@ def run(self, args): serve_parser = subparsers.add_parser("serve", help="Start plugin server") serve_parser.add_argument( - "--log-format", type=str, default="text", choices=["text", "json"], help="logging format" + "--log-format", + type=str, + default="text", + choices=["text", "json"], + help="logging format", ) serve_parser.add_argument( "--log-level", @@ -108,11 +109,16 @@ def run(self, args): "--no-sentry", action="store_true", help="disable sentry" ) serve_parser.add_argument( - "--otel-endpoint", type=str, default="", help="Open Telemetry HTTP collector endpoint" + "--otel-endpoint", + type=str, + default="", + help="Open Telemetry HTTP collector endpoint", ) serve_parser.add_argument( - "--otel-endpoint-insecure", type=str, default="", - help="Open Telemetry HTTP collector endpoint (for development only)" + "--otel-endpoint-insecure", + type=str, + default="", + help="Open Telemetry HTTP collector endpoint (for development only)", ) serve_parser.add_argument( From c80e966b87b80f9c50faa942a75eb6ff3752affc Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 2 Aug 2023 15:08:40 +0100 Subject: [PATCH 4/4] update --- cloudquery/sdk/serve/plugin.py | 59 +++++++++++++--------------------- 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/cloudquery/sdk/serve/plugin.py b/cloudquery/sdk/serve/plugin.py index 179d76e..74082f7 100644 --- a/cloudquery/sdk/serve/plugin.py +++ b/cloudquery/sdk/serve/plugin.py @@ -1,5 +1,4 @@ import argparse -import datetime import logging import os from concurrent import futures @@ -33,26 +32,6 @@ _has_colors = True -def add_timestamp(_, __, event_dict): - event_dict["timestamp"] = datetime.datetime.utcnow().isoformat() - return event_dict - - -_IS_WINDOWS = sys.platform == "win32" - -try: - import colorama -except ImportError: - colorama = None - -if _IS_WINDOWS: # pragma: no cover - # On Windows, use colors by default only if Colorama is installed. - _has_colors = colorama is not None -else: - # On other OSes, use colors by default. - _has_colors = True - - def get_logger(args): log_level_map = { "debug": logging.DEBUG, @@ -80,15 +59,15 @@ def get_logger(args): processors.append( structlog.dev.ConsoleRenderer( colors=os.environ.get("NO_COLOR", "") == "" - and ( - os.environ.get("FORCE_COLOR", "") != "" - or ( - _has_colors - and sys.stdout is not None - and hasattr(sys.stdout, "isatty") - and sys.stdout.isatty() - ) - ) + and ( + os.environ.get("FORCE_COLOR", "") != "" + or ( + _has_colors + and sys.stdout is not None + and hasattr(sys.stdout, "isatty") + and sys.stdout.isatty() + ) + ) ) ) else: @@ -121,20 +100,26 @@ def run(self, args): choices=["trace", "debug", "info", "warn", "error"], help="log level", ) + + # ignored for now serve_parser.add_argument( - "--no-sentry", action="store_true", help="disable sentry" + "--no-sentry", + action="store_true", + help="disable sentry (placeholder for future use)", ) + # ignored for now serve_parser.add_argument( "--otel-endpoint", type=str, default="", - help="Open Telemetry HTTP collector endpoint", + help="Open Telemetry HTTP collector endpoint (placeholder for future use)", ) + # ignored for now serve_parser.add_argument( "--otel-endpoint-insecure", type=str, default="", - help="Open Telemetry HTTP collector endpoint (for development only)", + help="Open Telemetry HTTP collector endpoint (for development only) (placeholder for future use)", ) serve_parser.add_argument( @@ -184,17 +169,17 @@ def run(self, args): sys.exit(1) def _serve(self, args): - log = get_logger(args) - self._plugin.set_logger(log) + logger = get_logger(args) + self._plugin.set_logger(logger) self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) discovery_pb2_grpc.add_DiscoveryServicer_to_server( DiscoveryServicer([3]), self._server ) plugin_pb2_grpc.add_PluginServicer_to_server( - PluginServicer(self._plugin, log), self._server + PluginServicer(self._plugin, logger), self._server ) self._server.add_insecure_port(args.address) - log.info("Starting server", address=args.address) + logger.info("Starting server", address=args.address) self._server.start() self._server.wait_for_termination()