Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,6 @@ def __init__(
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
**kwargs,
) -> None:
if target_state not in self.VALID_STATES:
raise ValueError(
f"Invalid target_state: {target_state}. Must be one of {sorted(self.VALID_STATES)}"
)
super().__init__(**kwargs)
self.resource_group_name = resource_group_name
self.vm_name = vm_name
Expand All @@ -84,6 +80,10 @@ def execute(self, context: Context) -> None:
In deferrable mode, the polling is deferred to the triggerer. Otherwise
the sensor waits synchronously.
"""
if self.target_state not in self.VALID_STATES:
raise ValueError(
f"Invalid target_state: {self.target_state}. Must be one of {sorted(self.VALID_STATES)}"
)
if not self.deferrable:
super().execute(context=context)
else:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ def __init__(
self.flatten_structure = flatten_structure
self.gcp_user_project = gcp_user_project
self.create_container = create_container
self.match_glob = match_glob

if self.flatten_structure and self.keep_directory_structure:
self.log.warning("flatten_structure=True takes precedence over keep_directory_structure=True")
Expand All @@ -131,9 +132,6 @@ def __init__(
self._is_match_glob_supported = False
except ImportError:
self._is_match_glob_supported = False
if not self._is_match_glob_supported and match_glob:
raise ValueError("The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'.")
self.match_glob = match_glob

def _transform_file_path(self, file_path: str) -> str:
"""
Expand All @@ -160,6 +158,9 @@ def _should_skip_gcs_object(name: str) -> bool:
return name.endswith("/")

def execute(self, context: Context) -> list[str]:
if not self._is_match_glob_supported and self.match_glob:
raise ValueError("The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'.")

gcs_hook = GCSHook(
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.google_impersonation_chain,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,10 @@ def __init__(
**kwargs,
) -> None:
super().__init__(**kwargs)
if sql_params is None:
sql_params = {}
self.filename = filename
self.oracle_conn_id = oracle_conn_id
self.sql = sql
self.sql_params = sql_params
Comment on lines 70 to -76

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please restore the changes here (see #70505 for details)

self.sql_params = sql_params if sql_params is not None else {}
self.azure_data_lake_conn_id = azure_data_lake_conn_id
self.azure_data_lake_path = azure_data_lake_path
self.delimiter = delimiter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,17 @@ def test_init(self):
assert sensor.target_state == "running"
assert sensor.azure_conn_id == CONN_ID

def test_init_invalid_target_state(self):
def test_invalid_templated_target_state_is_validated_after_rendering(self):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only the rejection path is tested; the behaviour the PR restores (valid templated value surviving to execution) is untested.

sensor = AzureVirtualMachineStateSensor(
task_id="sense_vm",
resource_group_name=RESOURCE_GROUP,
vm_name=VM_NAME,
target_state="{{ state }}",
)
sensor.render_template_fields(context={"state": "invalid_state"})

with pytest.raises(ValueError, match="Invalid target_state"):
AzureVirtualMachineStateSensor(
task_id="sense_vm",
resource_group_name=RESOURCE_GROUP,
vm_name=VM_NAME,
target_state="invalid_state",
)
sensor.execute(context=None)

def test_template_fields(self):
sensor = AzureVirtualMachineStateSensor(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,17 @@ def test_init_defaults(self):
assert op.create_container is False

@mock.patch("airflow.providers.google.__version__", "10.2.0")
def test_match_glob_requires_recent_google_provider(self):
def test_templated_match_glob_requires_recent_google_provider_after_rendering(self):
op = GCSToAzureBlobStorageOperator(
task_id=TASK_ID,
gcs_bucket=GCS_BUCKET,
container_name=CONTAINER,
match_glob="{{ match_glob }}",
)
op.render_template_fields(context={"match_glob": "**/*.csv"})

with pytest.raises(ValueError, match="match_glob"):
GCSToAzureBlobStorageOperator(
task_id=TASK_ID,
gcs_bucket=GCS_BUCKET,
container_name=CONTAINER,
match_glob="**/*.csv",
)
op.execute(context=None)

@mock.patch("airflow.providers.microsoft.azure.transfers.gcs_to_wasb.WasbHook")
@mock.patch("airflow.providers.microsoft.azure.transfers.gcs_to_wasb.GCSHook")
Expand Down
3 changes: 0 additions & 3 deletions scripts/ci/prek/validate_operators_init_exemptions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,6 @@ providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_gcs.py::GCSToGCSOperator
providers/google/src/airflow/providers/google/cloud/transfers/gcs_to_local.py::GCSToLocalFilesystemOperator
providers/google/src/airflow/providers/google/marketing_platform/operators/campaign_manager.py::GoogleCampaignManagerDeleteReportOperator
providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/compute.py::AzureVirtualMachineStateSensor
providers/microsoft/azure/src/airflow/providers/microsoft/azure/transfers/gcs_to_wasb.py::GCSToAzureBlobStorageOperator
providers/microsoft/azure/src/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py::OracleToAzureDataLakeOperator
providers/microsoft/psrp/src/airflow/providers/microsoft/psrp/operators/psrp.py::PsrpOperator
providers/neo4j/src/airflow/providers/neo4j/operators/neo4j.py::Neo4jOperator
providers/oracle/src/airflow/providers/oracle/transfers/oracle_to_oracle.py::OracleToOracleOperator
Expand Down