diff --git a/packages/cubejs-backend-native/python/cube/src/__init__.py b/packages/cubejs-backend-native/python/cube/src/__init__.py index 6e26569556da1..5acc1c62517fe 100644 --- a/packages/cubejs-backend-native/python/cube/src/__init__.py +++ b/packages/cubejs-backend-native/python/cube/src/__init__.py @@ -1,27 +1,182 @@ -class JinjaException(Exception): +import os +from typing import Union, Callable, Dict, Any + + +def file_repository(path): + files = [] + + for (dirpath, dirnames, filenames) in os.walk(path): + for fileName in filenames: + if fileName.endswith(".js") or fileName.endswith(".yml") or fileName.endswith(".yaml") or fileName.endswith(".jinja") or fileName.endswith(".py"): + path = os.path.join(dirpath, fileName) + + f = open(path, 'r') + content = f.read() + f.close() + + files.append({ + 'fileName': fileName, + 'content': content + }) + + return files + +class ConfigurationException(Exception): + pass + +class RequestContext: + url: str + method: str + headers: dict[str, str] + + +class Configuration: + web_sockets: bool + http: Dict + graceful_shutdown: int + process_subscriptions_interval: int + web_sockets_base_path: str + schema_path: str + base_path: str + dev_server: bool + api_secret: str + cache_and_queue_driver: str + allow_js_duplicate_props_in_schema: bool + jwt: Dict + scheduled_refresh_timer: Any + scheduled_refresh_timezones: list[str] + scheduled_refresh_concurrency: int + scheduled_refresh_batch_size: int + compiler_cache_size: int + update_compiler_cache_keep_alive: bool + max_compiler_cache_keep_alive: int + telemetry: bool + sql_cache: bool + live_preview: bool + # SQL API + pg_sql_port: int + sql_super_user: str + sql_user: str + sql_password: str + # Functions + logger: Callable + context_to_app_id: Union[str, Callable[[RequestContext], str]] + context_to_orchestrator_id: Union[str, Callable[[RequestContext], str]] + driver_factory: Callable[[RequestContext], Dict] + external_driver_factory: Callable[[RequestContext], Dict] + db_type: Union[str, Callable[[RequestContext], str]] + check_auth: Callable + check_sql_auth: Callable + can_switch_sql_user: Callable + extend_context: Callable + scheduled_refresh_contexts: Callable + context_to_api_scopes: Callable + repository_factory: Callable + schema_version: Union[str, Callable[[RequestContext], str]] + semantic_layer_sync: Union[Dict, Callable[[], Dict]] + pre_aggregations_schema: Union[Callable[[RequestContext], str]] + orchestrator_options: Union[Dict, Callable[[RequestContext], Dict]] + + def __init__(self): + self.web_sockets = None + self.http = None + self.graceful_shutdown = None + self.schema_path = None + self.base_path = None + self.dev_server = None + self.api_secret = None + self.web_sockets_base_path = None + self.pg_sql_port = None + self.cache_and_queue_driver = None + self.allow_js_duplicate_props_in_schema = None + self.process_subscriptions_interval = None + self.jwt = None + self.scheduled_refresh_timer = None + self.scheduled_refresh_timezones = None + self.scheduled_refresh_concurrency = None + self.scheduled_refresh_batch_size = None + self.compiler_cache_size = None + self.update_compiler_cache_keep_alive = None + self.max_compiler_cache_keep_alive = None + self.telemetry = None + self.sql_cache = None + self.live_preview = None + self.sql_super_user = None + self.sql_user = None + self.sql_password = None + # Functions + self.logger = None + self.context_to_app_id = None + self.context_to_orchestrator_id = None + self.driver_factory = None + self.external_driver_factory = None + self.db_type = None + self.check_auth = None + self.check_sql_auth = None + self.can_switch_sql_user = None + self.query_rewrite = None + self.extend_context = None + self.scheduled_refresh_contexts = None + self.context_to_api_scopes = None + self.repository_factory = None + self.schema_version = None + self.semantic_layer_sync = None + self.pre_aggregations_schema = None + self.orchestrator_options = None + + def __call__(self, func): + if isinstance(func, str): + return AttrRef(self, func) + + if not callable(func): + raise ConfigurationException("@config decorator must be used with functions, actual: '%s'" % type(func).__name__) + + if hasattr(self, func.__name__): + setattr(self, func.__name__, func) + else: + raise ConfigurationException("Unknown configuration property: '%s'" % func.__name__) + +class AttrRef: + config: Configuration + attribute: str + + def __init__(self, config: Configuration, attribue: str): + self.config = config + self.attribute = attribue + + def __call__(self, func): + if not callable(func): + raise ConfigurationException("@config decorator must be used with functions, actual: '%s'" % type(func).__name__) + + if hasattr(self.config, self.attribute): + setattr(self.config, self.attribute, func) + else: + raise ConfigurationException("Unknown configuration property: '%s'" % func.__name__) + + return func + +config = Configuration() +# backward compatibility +settings = config + +class TemplateException(Exception): pass -class JinjaContext: +class TemplateContext: def function(self, func): if not callable(func): - raise JinjaException("function registration must be used with functions, actual: '%s'" % type(func).__name__) + raise TemplateException("function registration must be used with functions, actual: '%s'" % type(func).__name__) return context_func(func) def filter(self, func): if not callable(func): - raise JinjaException("function registration must be used with functions, actual: '%s'" % type(func).__name__) + raise TemplateException("function registration must be used with functions, actual: '%s'" % type(func).__name__) - raise JinjaException("filter registration is not supported") + raise TemplateException("filter registration is not supported") def variable(self, func): - raise JinjaException("variable registration is not supported") - -class TemplateModule: - def JinjaContext(): - return JinjaContext() - -template = TemplateModule + raise TemplateException("variable registration is not supported") def context_func(func): func.cube_context_func = True @@ -29,5 +184,5 @@ def context_func(func): __all__ = [ 'context_func', - 'template' + 'TemplateContext' ] diff --git a/packages/cubejs-backend-native/python/cube/src/conf/__init__.py b/packages/cubejs-backend-native/python/cube/src/conf/__init__.py deleted file mode 100644 index e80be1c4de7cc..0000000000000 --- a/packages/cubejs-backend-native/python/cube/src/conf/__init__.py +++ /dev/null @@ -1,160 +0,0 @@ -import os -from typing import Union, Callable, Dict, Any - - -def file_repository(path): - files = [] - - for (dirpath, dirnames, filenames) in os.walk(path): - for fileName in filenames: - if fileName.endswith(".js") or fileName.endswith(".yml") or fileName.endswith(".yaml") or fileName.endswith(".jinja") or fileName.endswith(".py"): - path = os.path.join(dirpath, fileName) - - f = open(path, 'r') - content = f.read() - f.close() - - files.append({ - 'fileName': fileName, - 'content': content - }) - - return files - -class ConfigurationException(Exception): - pass - -class RequestContext: - url: str - method: str - headers: dict[str, str] - - -class Configuration: - web_sockets: bool - http: Dict - graceful_shutdown: int - process_subscriptions_interval: int - web_sockets_base_path: str - schema_path: str - base_path: str - dev_server: bool - api_secret: str - cache_and_queue_driver: str - allow_js_duplicate_props_in_schema: bool - jwt: Dict - scheduled_refresh_timer: Any - scheduled_refresh_timezones: list[str] - scheduled_refresh_concurrency: int - scheduled_refresh_batch_size: int - compiler_cache_size: int - update_compiler_cache_keep_alive: bool - max_compiler_cache_keep_alive: int - telemetry: bool - sql_cache: bool - live_preview: bool - # SQL API - pg_sql_port: int - sql_super_user: str - sql_user: str - sql_password: str - # Functions - logger: Callable - context_to_app_id: Union[str, Callable[[RequestContext], str]] - context_to_orchestrator_id: Union[str, Callable[[RequestContext], str]] - driver_factory: Callable[[RequestContext], Dict] - external_driver_factory: Callable[[RequestContext], Dict] - db_type: Union[str, Callable[[RequestContext], str]] - check_auth: Callable - check_sql_auth: Callable - can_switch_sql_user: Callable - extend_context: Callable - scheduled_refresh_contexts: Callable - context_to_api_scopes: Callable - repository_factory: Callable - schema_version: Union[str, Callable[[RequestContext], str]] - semantic_layer_sync: Union[Dict, Callable[[], Dict]] - pre_aggregations_schema: Union[Callable[[RequestContext], str]] - orchestrator_options: Union[Dict, Callable[[RequestContext], Dict]] - - def __init__(self): - self.web_sockets = None - self.http = None - self.graceful_shutdown = None - self.schema_path = None - self.base_path = None - self.dev_server = None - self.api_secret = None - self.web_sockets_base_path = None - self.pg_sql_port = None - self.cache_and_queue_driver = None - self.allow_js_duplicate_props_in_schema = None - self.process_subscriptions_interval = None - self.jwt = None - self.scheduled_refresh_timer = None - self.scheduled_refresh_timezones = None - self.scheduled_refresh_concurrency = None - self.scheduled_refresh_batch_size = None - self.compiler_cache_size = None - self.update_compiler_cache_keep_alive = None - self.max_compiler_cache_keep_alive = None - self.telemetry = None - self.sql_cache = None - self.live_preview = None - self.sql_super_user = None - self.sql_user = None - self.sql_password = None - # Functions - self.logger = None - self.context_to_app_id = None - self.context_to_orchestrator_id = None - self.driver_factory = None - self.external_driver_factory = None - self.db_type = None - self.check_auth = None - self.check_sql_auth = None - self.can_switch_sql_user = None - self.query_rewrite = None - self.extend_context = None - self.scheduled_refresh_contexts = None - self.context_to_api_scopes = None - self.repository_factory = None - self.schema_version = None - self.semantic_layer_sync = None - self.pre_aggregations_schema = None - self.orchestrator_options = None - - def __call__(self, func): - if isinstance(func, str): - return AttrRef(self, func) - - if not callable(func): - raise ConfigurationException("@config decorator must be used with functions, actual: '%s'" % type(func).__name__) - - if hasattr(self, func.__name__): - setattr(self, func.__name__, func) - else: - raise ConfigurationException("Unknown configuration property: '%s'" % func.__name__) - -class AttrRef: - config: Configuration - attribute: str - - def __init__(self, config: Configuration, attribue: str): - self.config = config - self.attribute = attribue - - def __call__(self, func): - if not callable(func): - raise ConfigurationException("@config decorator must be used with functions, actual: '%s'" % type(func).__name__) - - if hasattr(self.config, self.attribute): - setattr(self.config, self.attribute, func) - else: - raise ConfigurationException("Unknown configuration property: '%s'" % func.__name__) - - return func - -config = Configuration() -# backward compatibility -settings = config \ No newline at end of file diff --git a/packages/cubejs-backend-native/src/python/entry.rs b/packages/cubejs-backend-native/src/python/entry.rs index f7d7c9290a3a8..5dc78b1ebd004 100644 --- a/packages/cubejs-backend-native/src/python/entry.rs +++ b/packages/cubejs-backend-native/src/python/entry.rs @@ -20,11 +20,11 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult { py_runtime_init(&mut cx, channel.clone())?; let conf_res = Python::with_gil(|py| -> PyResult { - let cube_conf_code = include_str!(concat!( + let cube_code = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), - "/python/cube/src/conf/__init__.py" + "/python/cube/src/__init__.py" )); - PyModule::from_code(py, cube_conf_code, "__init__.py", "cube.conf")?; + PyModule::from_code(py, cube_code, "__init__.py", "cube")?; let config_module = PyModule::from_code(py, &file_content_arg, &options_file_name, "")?; let settings_py = if config_module.hasattr("__execution_context_locals")? { diff --git a/packages/cubejs-backend-native/test/config.py b/packages/cubejs-backend-native/test/config.py index c7cc7e1e27a42..f42eb1adb5dbc 100644 --- a/packages/cubejs-backend-native/test/config.py +++ b/packages/cubejs-backend-native/test/config.py @@ -1,4 +1,4 @@ -from cube.conf import ( +from cube import ( config, file_repository ) diff --git a/packages/cubejs-backend-native/test/old-config.py b/packages/cubejs-backend-native/test/old-config.py index faadd64ef5eef..a84df4bdfffb2 100644 --- a/packages/cubejs-backend-native/test/old-config.py +++ b/packages/cubejs-backend-native/test/old-config.py @@ -1,4 +1,4 @@ -from cube.conf import ( +from cube import ( settings, file_repository ) diff --git a/packages/cubejs-backend-native/test/scoped-config.py b/packages/cubejs-backend-native/test/scoped-config.py index 45bc1a8532db8..3663c938e612d 100644 --- a/packages/cubejs-backend-native/test/scoped-config.py +++ b/packages/cubejs-backend-native/test/scoped-config.py @@ -1,5 +1,5 @@ source_code = """ -from cube.conf import ( +from cube import ( settings, config ) diff --git a/packages/cubejs-backend-native/test/templates/jinja-instance.py b/packages/cubejs-backend-native/test/templates/jinja-instance.py index 6ea24f4b8113a..063da194b64e5 100644 --- a/packages/cubejs-backend-native/test/templates/jinja-instance.py +++ b/packages/cubejs-backend-native/test/templates/jinja-instance.py @@ -1,77 +1,77 @@ -from cube import template +from cube import TemplateContext -context = template.JinjaContext() +template = TemplateContext() -@context.function +@template.function def arg_sum_integers(a, b): return a + b -@context.function +@template.function def arg_bool(a): return a + 0 -@context.function +@template.function def arg_str(a): return a -@context.function +@template.function def arg_null(a): return a -@context.function +@template.function def arg_sum_tuple(tu): return tu[0] + tu[1] -@context.function +@template.function def arg_sum_map(obj): return obj['field_a'] + obj['field_b'] -@context.function +@template.function def arg_seq(a): return a -@context.function +@template.function def new_int_tuple(): return (1,2) -@context.function +@template.function def new_str_tuple(): return ("1", "2") -@context.function +@template.function def load_data_sync(): - client = MyApiClient("google.com") - return client.load_data() + client = MyApiClient("google.com") + return client.load_data() -@context.function +@template.function async def load_data(): - api_response = { - "cubes": [ - { - "name": "cube_from_api", - "measures": [ - { "name": "count", "type": "count" }, - { "name": "total", "type": "sum", "sql": "amount" } - ], - "dimensions": [] - }, - { - "name": "cube_from_api_with_dimensions", - "measures": [ - { "name": "active_users", "type": "count_distinct", "sql": "user_id" } - ], - "dimensions": [ - { "name": "city", "sql": "city_column", "type": "string" } - ] - } - ] - } - return api_response + api_response = { + "cubes": [ + { + "name": "cube_from_api", + "measures": [ + { "name": "count", "type": "count" }, + { "name": "total", "type": "sum", "sql": "amount" } + ], + "dimensions": [] + }, + { + "name": "cube_from_api_with_dimensions", + "measures": [ + { "name": "active_users", "type": "count_distinct", "sql": "user_id" } + ], + "dimensions": [ + { "name": "city", "sql": "city_column", "type": "string" } + ] + } + ] + } + return api_response class ExampleClassModelA: def get_name_method(self): return "example" -@context.function +@template.function def load_class_model(): return ExampleClassModelA() diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js index 57eced08cc727..097b55ead432f 100644 --- a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js +++ b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js @@ -58,7 +58,8 @@ export class DataSchemaCompiler { * @protected */ async loadPythonContext(files) { - const modules = await Promise.all(files.filter((f) => f.fileName.endsWith('.py')).map(async (file) => { + // TODO Appropriate template functions loading + const modules = await Promise.all(files.filter((f) => f.fileName === 'globals.py').map(async (file) => { const exports = await this.nativeInstance.loadPythonContext( file.fileName, file.content