diff --git a/docs/concepts/macros/macro_variables.md b/docs/concepts/macros/macro_variables.md index 21750af2de..31e82465bd 100644 --- a/docs/concepts/macros/macro_variables.md +++ b/docs/concepts/macros/macro_variables.md @@ -37,7 +37,7 @@ We describe SQLMesh's predefined variables below; user-defined macro variables a ## Predefined Variables SQLMesh comes with predefined variables that can be used in your queries. They are automatically set by the SQLMesh runtime. -These variables are related to time and comprise a combination of prefixes (start, end, execution) and postfixes (date, ds, ts, epoch, millis). +These variables are mostly related to time and comprise a combination of prefixes (start, end, execution) and postfixes (date, ds, ts, epoch, millis). SQLMesh uses the python [datetime module](https://docs.python.org/3/library/datetime.html) for handling dates and times. It uses the standard [Unix epoch](https://en.wikipedia.org/wiki/Unix_time) start of 1970-01-01. *All predefined variables with a time component use the [UTC time zone](https://en.wikipedia.org/wiki/Coordinated_Universal_Time).* @@ -86,3 +86,10 @@ All predefined macro variables: * @start_millis * @end_millis * @execution_millis + +Other macro variables: + +* @runtime_stage - A string value that denotes the current stage of the SQLMesh runtime. It can take one of the following values: + * 'loading' - The project is currently being loaded into SQLMesh's runtime context. + * 'creating' - The model tables are being created. + * 'evaluating' - The models' logic is being evaluated. diff --git a/docs/concepts/macros/sqlmesh_macros.md b/docs/concepts/macros/sqlmesh_macros.md index 223cb02785..f8c3ff06f5 100644 --- a/docs/concepts/macros/sqlmesh_macros.md +++ b/docs/concepts/macros/sqlmesh_macros.md @@ -260,10 +260,22 @@ It includes three elements: 2. A value to return if the condition is `TRUE` 3. A value to return if the condition is `FALSE` [optional] -These elements are specified as `@IF([logical condition], [value if TRUE], [value if FALSE])`. +These elements are specified as: + +```sql linenums="1" +@IF([logical condition], [value if TRUE], [value if FALSE]) +``` The value to return if the condition is `FALSE` is optional - if it is not provided and the condition is `FALSE`, then the macro has no effect on the resulting query. +It's also possible to use this macro in order to conditionally execute pre/post-statements: + +```sql linenums="1" +@IF([logical condition], [statement to execute if TRUE]); +``` + +The `@IF` pre/post-statement itself must end with a semi-colon, but the inner statement argument must not. + The logical condition should be written *in SQL* and is evaluated with [SQLGlot's](https://github.com/tobymao/sqlglot) SQL engine. It supports the following operators: - Equality: `=` for equals, `!=` or `<>` for not equals @@ -318,6 +330,33 @@ SELECT FROM table ``` +Another example is conditionally executing a pre/post-statement depending on the current [runtime stage](./macro_variables.md#predefined-variables). For instance, the following `@IF` post-statement will only be executed at model evaluation time: + +```sql linenums="1" +MODEL ( + name sqlmesh_example.full_model, + kind FULL, + cron '@daily', + grain item_id, + audits [assert_positive_order_ids], +); + +SELECT + item_id, + count(distinct id) AS num_orders, +FROM + sqlmesh_example.incremental_model +GROUP BY item_id +ORDER BY item_id; + +@IF( + @runtime_stage = 'evaluating', + ALTER TABLE sqlmesh_example.full_model ALTER item_id TYPE VARCHAR +); +``` + +NOTE: we can also, say, alter the type of a column if the `@runtime_stage` is `'creating'`, but that will only have meaningful effects if the corresponding model is of an incremental kind, since `FULL` models are rebuilt on each evaluation and hence any changes made at their creation stage will be overwritten. + #### @EVAL `@EVAL` evaluates its arguments with SQLGlot's SQL executor. diff --git a/sqlmesh/core/renderer.py b/sqlmesh/core/renderer.py index 5f30352a9e..b9e0f3722c 100644 --- a/sqlmesh/core/renderer.py +++ b/sqlmesh/core/renderer.py @@ -327,6 +327,7 @@ def render( snapshots=snapshots, table_mapping=table_mapping, is_dev=is_dev, + runtime_stage=runtime_stage, **kwargs, ) except ParsetimeAdapterCallError: diff --git a/sqlmesh/core/snapshot/evaluator.py b/sqlmesh/core/snapshot/evaluator.py index d9cf891025..58718767fd 100644 --- a/sqlmesh/core/snapshot/evaluator.py +++ b/sqlmesh/core/snapshot/evaluator.py @@ -152,6 +152,7 @@ def apply(query_or_df: QueryOrDF, index: int = 0) -> None: end=end, execution_time=execution_time, has_intervals=bool(snapshot.intervals), + runtime_stage=RuntimeStage.EVALUATING, **kwargs, ) @@ -159,7 +160,6 @@ def apply(query_or_df: QueryOrDF, index: int = 0) -> None: engine_adapter=self.adapter, snapshots=snapshots, is_dev=is_dev, - runtime_stage=RuntimeStage.EVALUATING, **common_render_kwargs, ) diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 037b0477f7..459434e80a 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -171,7 +171,7 @@ def increment_stage_counter(evaluator) -> None: @increment_stage_counter(); @if(@runtime_stage = 'evaluating', ALTER TABLE test_schema.foo MODIFY COLUMN c SET MASKING POLICY p); - SELECT 1 AS a; + SELECT 1 AS a, @runtime_stage AS b; """ ), macros=macro.get_registry(), @@ -196,6 +196,16 @@ def increment_stage_counter(evaluator) -> None: [parse_one("ALTER TABLE test_schema.foo MODIFY COLUMN c SET MASKING POLICY p")] ) + assert snapshot.model.render_query().sql() == '''SELECT 1 AS "a", 'loading' AS "b"''' + assert ( + snapshot.model.render_query(runtime_stage=RuntimeStage.CREATING).sql() + == '''SELECT 1 AS "a", 'creating' AS "b"''' + ) + assert ( + snapshot.model.render_query(runtime_stage=RuntimeStage.EVALUATING).sql() + == '''SELECT 1 AS "a", 'evaluating' AS "b"''' + ) + def test_evaluate_paused_forward_only_upstream(mocker: MockerFixture, make_snapshot): model = SqlModel(name="test_schema.test_model", query=parse_one("SELECT a, ds"))