Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ def __init__(
):
super().__init__(**kwargs)
self.name = name
self.ruleset = ruleset.strip()
self.ruleset = ruleset
self.description = description
self.update_rule_set = update_rule_set
self.data_quality_ruleset_kwargs = data_quality_ruleset_kwargs or {}
Expand All @@ -449,6 +449,8 @@ def validate_inputs(self) -> None:
raise AttributeError("Target table must have DatabaseName and TableName")

def execute(self, context: Context):
# ruleset is a template field; strip the rendered value here, not in __init__.
Comment thread
shahar1 marked this conversation as resolved.
self.ruleset = self.ruleset.strip()
self.validate_inputs()

config = {
Expand Down
24 changes: 24 additions & 0 deletions providers/amazon/tests/unit/amazon/aws/operators/test_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from __future__ import annotations

from collections.abc import Generator
from datetime import datetime
from typing import TYPE_CHECKING
from unittest import mock

Expand All @@ -25,6 +26,7 @@
from boto3 import client
from moto import mock_aws

from airflow.models.dag import DAG
from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink
Expand Down Expand Up @@ -813,6 +815,28 @@ def test_execute_create_rule(self, glue_data_quality_mock_conn):
Ruleset=self.RULE_SET,
)

@mock.patch.object(GlueDataQualityHook, "conn")
def test_execute_strips_rendered_ruleset(self, glue_data_quality_mock_conn):
# ruleset is a template field; execute strips the rendered value (rendering can add whitespace).
with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1, 1)) as dag:
self.operator = GlueDataQualityOperator(
task_id="create_data_quality_ruleset",
name=self.RULE_SET_NAME,
ruleset="{{ params.rules }}",
dag=dag,
)
self.operator.defer = mock.MagicMock()
self.operator.render_template_fields({"params": {"rules": f" {self.RULE_SET} "}})
assert self.operator.ruleset == f" {self.RULE_SET} "

self.operator.execute({})

glue_data_quality_mock_conn.create_data_quality_ruleset.assert_called_once_with(
Description="AWS Glue Data Quality Rule Set With Airflow",
Name=self.RULE_SET_NAME,
Ruleset=self.RULE_SET,
)

@mock.patch.object(GlueDataQualityHook, "conn")
def test_execute_create_rule_should_fail_if_rule_already_exists(self, glue_data_quality_mock_conn):
self.operator = GlueDataQualityOperator(
Expand Down
1 change: 0 additions & 1 deletion scripts/ci/prek/validate_operators_init_exemptions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsModifyTas
providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsStartReplicationOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/ecs.py::EcsRunTaskOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py::EmrAddStepsOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py::GlueDataQualityOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::S3DeleteObjectsOperator
Expand Down
Loading