Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -712,7 +712,6 @@ def __init__(
polling_interval_seconds: int = 10,
**kwargs,
) -> None:
# TODO: remove one day
if cluster_config is None and virtual_cluster_config is None:
warnings.warn(
f"Passing cluster parameters by keywords to `{type(self).__name__}` will be deprecated. "
Expand All @@ -732,6 +731,7 @@ def __init__(
"project_id argument is required when building cluster from keywords parameters"
)
kwargs["project_id"] = project_id

cluster_config = ClusterGenerator(**kwargs).make()

# Remove from kwargs cluster params passed for backward compatibility
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,25 @@ def test_deprecation_warning(self):

assert op.project_id == GCP_PROJECT
assert op.cluster_name == "cluster_name"
assert op.cluster_config is None
assert op._legacy_cluster_kwargs["num_workers"] == 2
assert op._legacy_cluster_kwargs["zone"] == "zone"

@mock.patch(DATAPROC_PATH.format("Cluster.to_dict"))
@mock.patch(DATAPROC_PATH.format("DataprocHook"))
def test_deprecated_kwargs_cluster_config_built_in_execute(self, mock_hook, to_dict_mock):
mock_hook.return_value.create_cluster.result.return_value = None
with pytest.warns(AirflowProviderDeprecationWarning):
op = DataprocCreateClusterOperator(
task_id=TASK_ID,
region=GCP_REGION,
project_id=GCP_PROJECT,
cluster_name="cluster_name",
num_workers=2,
zone="zone",
)
assert op.cluster_config is None
op.execute(context=self.mock_context)
assert op.cluster_config["worker_config"]["num_instances"] == 2
assert "zones/zone" in op.cluster_config["master_config"]["machine_type_uri"]

Expand Down
Loading