Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix formatting and add convert_to_lower flag to elementary.generate_schema_baseline_test #444

Merged
merged 3 commits into from
May 28, 2024
Merged

Fix formatting and add convert_to_lower flag to elementary.generate_schema_baseline_test #444

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b9dcac
fix: indentation error of generate_schema_baseline_test macro
Arun-kc Jul 29, 2023
6f42a26
Merge branch 'elementary-data:master' into 810-indentation-correction…
Arun-kc Aug 20, 2023
32c8306
Merge branch 'master' into 810-indentation-correction-for-generate-sc…
ellakz Jan 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
70 changes: 47 additions & 23 deletions macros/commands/generate_schema_baseline_test.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
{% macro generate_schema_baseline_test(name=none, include_sources=True, include_models=False, fail_on_added=False, enforce_types=False) %}
{% macro generate_schema_baseline_test(name=none, include_sources=True, include_models=False, fail_on_added=False, enforce_types=False, convert_to_lower=False) %}
{% if name %}
{{ generate_schema_baseline_test_for_node(name, fail_on_added=fail_on_added, enforce_types=enforce_types) }}
{{ generate_schema_baseline_test_for_node(name, fail_on_added=fail_on_added, enforce_types=enforce_types, convert_to_lower=convert_to_lower) }}
{% else %}
{{ generate_schema_baseline_test_for_all_nodes(include_sources=include_sources, include_models=include_models,
fail_on_added=fail_on_added, enforce_types=enforce_types) }}
fail_on_added=fail_on_added, enforce_types=enforce_types, convert_to_lower=convert_to_lower) }}
{% endif %}
{% endmacro %}

{% macro generate_schema_baseline_test_for_all_nodes(include_sources=True, include_models=False, fail_on_added=False, enforce_types=False) %}
{% macro generate_schema_baseline_test_for_all_nodes(include_sources=True, include_models=False, fail_on_added=False, enforce_types=False, convert_to_lower=False) %}
{% set nodes = elementary.get_nodes_from_graph() %}
{% for node in nodes %}
{% if node.package_name != 'elementary' and
((include_sources and node.resource_type == 'source') or
(include_models and node.resource_type == 'model')) %}
{% do print("Generating schema changes from baseline test for {} '{}':".format(node.resource_type, node.name)) %}
{{ generate_schema_baseline_test_for_node(node, fail_on_added=fail_on_added, enforce_types=enforce_types) }}
{{ generate_schema_baseline_test_for_node(node, fail_on_added=fail_on_added, enforce_types=enforce_types, convert_to_lower=convert_to_lower) }}
{% do print('----------------------------------') %}
{% endif %}
{% endfor %}
{% endmacro %}

{% macro generate_schema_baseline_test_for_node(node, fail_on_added=False, enforce_types=False) %}
{% macro generate_schema_baseline_test_for_node(node, fail_on_added=False, enforce_types=False, convert_to_lower=False) %}
{% if node is string %}
{% set node_name = node %}
{% set node = elementary.get_node_by_name(node_name) %}
Expand Down Expand Up @@ -52,13 +52,52 @@
{% do test_params.update({"enforce_types": "true"}) %}
{% endif %}

{# Common yaml for sources and models #}
{% set common_yaml %}
{# Full yaml for sources and models #}
{% set full_yaml %}
{%- if node.resource_type == 'source' %}
{{generate_schema_baseline_test_for_source(node, columns, test_params, convert_to_lower)}}
{% else %}
{{generate_schema_baseline_test_for_model(node, columns, test_params, convert_to_lower)}}
{% endif -%}
{% endset %}

{% do print(full_yaml) %}
{% endmacro %}

{% macro generate_schema_baseline_test_for_source(node, columns, test_params, convert_to_lower) %}
sources:
- name: {{ node.source_name }}
tables:
- name: {{ node.name }}
columns:
{%- for column in columns %}
- name: {{ column.name }}
{%- if convert_to_lower %}
data_type: {{ column.dtype|lower }}
{% else %}
haritamar marked this conversation as resolved.
Show resolved Hide resolved
data_type: {{ column.dtype }}
{% endif -%}
{% endfor %}
tests:
- elementary.schema_changes_from_baseline
{%- if test_params %}:
{%- for param, param_val in test_params.items() %}
{{param}}: {{param_val}}
{%- endfor -%}
{% endif -%}
{% endmacro %}

{% macro generate_schema_baseline_test_for_model(node, columns, test_params, convert_to_lower) %}
models:
- name: {{ node.name }}
columns:
{%- for column in columns %}
- name: {{ column.name }}
{%- if convert_to_lower %}
data_type: {{ elementary.get_column_data_type(column) | lower }}
{% else %}
haritamar marked this conversation as resolved.
Show resolved Hide resolved
data_type: {{ elementary.get_column_data_type(column) }}
{% endif -%}
{% endfor %}
tests:
- elementary.schema_changes_from_baseline
Expand All @@ -67,19 +106,4 @@
{{param}}: {{param_val}}
{%- endfor -%}
{% endif -%}
{% endset %}

{% set full_yaml %}
{%- if node.resource_type == 'source' %}
sources:
- name: {{ node.source_name }}
tables:
{{- common_yaml }}
{% else %}
models:
{{- common_yaml }}
{% endif -%}
{% endset %}

{% do print(full_yaml) %}
{% endmacro %}