From 22241b8a8086ed8b0ae93fe78a2eee0ee639d1ef Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Mon, 6 May 2024 20:18:06 +0300 Subject: [PATCH 1/6] add yq sample --- .../{operators.rst => dataproc_operators.rst} | 0 .../apache-airflow-providers-yandex/index.rst | 3 +- .../yq_operators.rst | 27 ++++++++++ .../providers/yandex/example_yandexcloud.py | 4 +- .../yandex/example_yandexcloud_dataproc.py | 4 +- ...xample_yandexcloud_dataproc_lightweight.py | 4 +- .../yandex/example_yandexcloud_yq.py | 54 +++++++++++++++++++ 7 files changed, 89 insertions(+), 7 deletions(-) rename docs/apache-airflow-providers-yandex/{operators.rst => dataproc_operators.rst} (100%) create mode 100644 docs/apache-airflow-providers-yandex/yq_operators.rst create mode 100644 tests/system/providers/yandex/example_yandexcloud_yq.py diff --git a/docs/apache-airflow-providers-yandex/operators.rst b/docs/apache-airflow-providers-yandex/dataproc_operators.rst similarity index 100% rename from docs/apache-airflow-providers-yandex/operators.rst rename to docs/apache-airflow-providers-yandex/dataproc_operators.rst diff --git a/docs/apache-airflow-providers-yandex/index.rst b/docs/apache-airflow-providers-yandex/index.rst index d5eb31c75a34c..75ea08c7f09d1 100644 --- a/docs/apache-airflow-providers-yandex/index.rst +++ b/docs/apache-airflow-providers-yandex/index.rst @@ -37,7 +37,8 @@ Configuration Connection types Lockbox Secret Backend - Operators + Data Proc Operators + Yandex Query Operators .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-yandex/yq_operators.rst b/docs/apache-airflow-providers-yandex/yq_operators.rst new file mode 100644 index 0000000000000..c6ea9437ce2e6 --- /dev/null +++ b/docs/apache-airflow-providers-yandex/yq_operators.rst @@ -0,0 +1,27 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Yandex Query Operators +====================== +`Yandex Query ` __ is a service in the Yandex Cloud to process data from different sources such as +Object Storage, MDB Clickhouse, MDB PostgreSQL, Yandex DataStreams using SQL scripts. + +Using the operators +^^^^^^^^^^^^^^^^^^^ +To learn how to use Yandex Query operator, +see `example DAG `_. diff --git a/tests/system/providers/yandex/example_yandexcloud.py b/tests/system/providers/yandex/example_yandexcloud.py index c48b58c264005..2751458ae984a 100644 --- a/tests/system/providers/yandex/example_yandexcloud.py +++ b/tests/system/providers/yandex/example_yandexcloud.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import os from datetime import datetime import yandex.cloud.dataproc.v1.cluster_pb2 as cluster_pb @@ -32,8 +31,9 @@ from airflow import DAG from airflow.decorators import task from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook +from tests.system.utils import get_test_env_id -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_hook" # Fill it with your identifiers diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc.py b/tests/system/providers/yandex/example_yandexcloud_dataproc.py index a08e60daa8eba..cfae4e94e0aab 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc.py +++ b/tests/system/providers/yandex/example_yandexcloud_dataproc.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import os import uuid from datetime import datetime @@ -32,6 +31,7 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule +from tests.system.utils import get_test_env_id # should be filled with appropriate ids @@ -41,7 +41,7 @@ # Dataproc cluster jobs will produce logs in specified s3 bucket S3_BUCKET_NAME_FOR_JOB_LOGS = "" -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_dataproc_operator" with DAG( diff --git a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py b/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py index 930d6bfc9d8ac..fa5a3c758bc1f 100644 --- a/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py +++ b/tests/system/providers/yandex/example_yandexcloud_dataproc_lightweight.py @@ -16,7 +16,6 @@ # under the License. from __future__ import annotations -import os from datetime import datetime from airflow import DAG @@ -28,6 +27,7 @@ # Name of the datacenter where Dataproc cluster will be created from airflow.utils.trigger_rule import TriggerRule +from tests.system.utils import get_test_env_id # should be filled with appropriate ids @@ -37,7 +37,7 @@ # Dataproc cluster will use this bucket as distributed storage S3_BUCKET_NAME = "" -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +ENV_ID = get_test_env_id() DAG_ID = "example_yandexcloud_dataproc_lightweight" with DAG( diff --git a/tests/system/providers/yandex/example_yandexcloud_yq.py b/tests/system/providers/yandex/example_yandexcloud_yq.py new file mode 100644 index 0000000000000..69dafa5cf20a6 --- /dev/null +++ b/tests/system/providers/yandex/example_yandexcloud_yq.py @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from datetime import datetime + +from airflow.models.dag import DAG +from airflow.operators.empty import EmptyOperator +from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator +from airflow.operators.python import PythonOperator + +from tests.system.utils import get_test_env_id + +ENV_ID = get_test_env_id() +DAG_ID = "example_yandexcloud_yq" + +with DAG( + DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + tags=["example"], +) as dag: + run_this_last = EmptyOperator( + task_id="run_this_last", + ) + + yq_operator = YQExecuteQueryOperator(task_id="sample_query", sql="select 33 as d, 44 as t") + yq_operator >> run_this_last + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "teardown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From dbd9d6c2f8bdcd2d941648b0c4b2d45acc149d6c Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 7 May 2024 11:34:55 +0300 Subject: [PATCH 2/6] fix style and links to doc from provider.yaml --- airflow/providers/yandex/provider.yaml | 4 ++-- tests/system/providers/yandex/example_yandexcloud_yq.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index de488215a020d..0bd1df02f7d23 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -62,14 +62,14 @@ integrations: - integration-name: Yandex.Cloud Dataproc external-doc-url: https://cloud.yandex.com/dataproc how-to-guide: - - /docs/apache-airflow-providers-yandex/operators.rst + - /docs/apache-airflow-providers-yandex/dataproc_operators.rst logo: /integration-logos/yandex/Yandex-Cloud.png tags: [service] - integration-name: Yandex.Cloud YQ external-doc-url: https://cloud.yandex.com/en/services/query how-to-guide: - - /docs/apache-airflow-providers-yandex/operators.rst + - /docs/apache-airflow-providers-yandex/yq_operators.rst logo: /integration-logos/yandex/Yandex-Cloud.png tags: [service] diff --git a/tests/system/providers/yandex/example_yandexcloud_yq.py b/tests/system/providers/yandex/example_yandexcloud_yq.py index 69dafa5cf20a6..43c53dab2e4e8 100644 --- a/tests/system/providers/yandex/example_yandexcloud_yq.py +++ b/tests/system/providers/yandex/example_yandexcloud_yq.py @@ -16,13 +16,11 @@ # under the License. from __future__ import annotations -import os from datetime import datetime from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator -from airflow.operators.python import PythonOperator from tests.system.utils import get_test_env_id From 2b4fcb6a76784c8b55dd2b5a263db4a9f2238db2 Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 7 May 2024 12:01:09 +0300 Subject: [PATCH 3/6] fix style again --- docs/apache-airflow-providers-yandex/yq_operators.rst | 2 +- tests/system/providers/yandex/example_yandexcloud_yq.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/apache-airflow-providers-yandex/yq_operators.rst b/docs/apache-airflow-providers-yandex/yq_operators.rst index c6ea9437ce2e6..7a7068e46b72f 100644 --- a/docs/apache-airflow-providers-yandex/yq_operators.rst +++ b/docs/apache-airflow-providers-yandex/yq_operators.rst @@ -19,7 +19,7 @@ Yandex Query Operators ====================== `Yandex Query ` __ is a service in the Yandex Cloud to process data from different sources such as -Object Storage, MDB Clickhouse, MDB PostgreSQL, Yandex DataStreams using SQL scripts. +Object Storage, MDB ClickHouse, MDB PostgreSQL, Yandex DataStreams using SQL scripts. Using the operators ^^^^^^^^^^^^^^^^^^^ diff --git a/tests/system/providers/yandex/example_yandexcloud_yq.py b/tests/system/providers/yandex/example_yandexcloud_yq.py index 43c53dab2e4e8..0ebef685e24b7 100644 --- a/tests/system/providers/yandex/example_yandexcloud_yq.py +++ b/tests/system/providers/yandex/example_yandexcloud_yq.py @@ -21,7 +21,6 @@ from airflow.models.dag import DAG from airflow.operators.empty import EmptyOperator from airflow.providers.yandex.operators.yq import YQExecuteQueryOperator - from tests.system.utils import get_test_env_id ENV_ID = get_test_env_id() From 31079aafa08613fc17f75a17cf14b818cf46a0cb Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 7 May 2024 12:40:52 +0300 Subject: [PATCH 4/6] more links --- docs/apache-airflow-providers-yandex/yq_operators.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/apache-airflow-providers-yandex/yq_operators.rst b/docs/apache-airflow-providers-yandex/yq_operators.rst index 7a7068e46b72f..78bdb733ee1ff 100644 --- a/docs/apache-airflow-providers-yandex/yq_operators.rst +++ b/docs/apache-airflow-providers-yandex/yq_operators.rst @@ -18,10 +18,11 @@ Yandex Query Operators ====================== -`Yandex Query ` __ is a service in the Yandex Cloud to process data from different sources such as -Object Storage, MDB ClickHouse, MDB PostgreSQL, Yandex DataStreams using SQL scripts. +`Yandex Query `__ is a service in the Yandex Cloud to process data from different sources such as +`Object Storage `__, `MDB ClickHouse `__, +`MDB PostgreSQL `__, `Yandex DataStreams `__ using SQL scripts. Using the operators ^^^^^^^^^^^^^^^^^^^ To learn how to use Yandex Query operator, -see `example DAG `_. +see `example DAG `__. From 001c5ba32b34590690e25c8431273d55291c388b Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 7 May 2024 15:40:45 +0300 Subject: [PATCH 5/6] reorg operators doc --- .../apache-airflow-providers-yandex/index.rst | 3 +- .../dataproc.rst} | 0 .../operators/index.rst | 28 +++++++++++++++++++ .../{yq_operators.rst => operators/yq.rst} | 0 4 files changed, 29 insertions(+), 2 deletions(-) rename docs/apache-airflow-providers-yandex/{dataproc_operators.rst => operators/dataproc.rst} (100%) create mode 100644 docs/apache-airflow-providers-yandex/operators/index.rst rename docs/apache-airflow-providers-yandex/{yq_operators.rst => operators/yq.rst} (100%) diff --git a/docs/apache-airflow-providers-yandex/index.rst b/docs/apache-airflow-providers-yandex/index.rst index 75ea08c7f09d1..4fb3ed732ef71 100644 --- a/docs/apache-airflow-providers-yandex/index.rst +++ b/docs/apache-airflow-providers-yandex/index.rst @@ -37,8 +37,7 @@ Configuration Connection types Lockbox Secret Backend - Data Proc Operators - Yandex Query Operators + Operators .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-yandex/dataproc_operators.rst b/docs/apache-airflow-providers-yandex/operators/dataproc.rst similarity index 100% rename from docs/apache-airflow-providers-yandex/dataproc_operators.rst rename to docs/apache-airflow-providers-yandex/operators/dataproc.rst diff --git a/docs/apache-airflow-providers-yandex/operators/index.rst b/docs/apache-airflow-providers-yandex/operators/index.rst new file mode 100644 index 0000000000000..12b05418e100f --- /dev/null +++ b/docs/apache-airflow-providers-yandex/operators/index.rst @@ -0,0 +1,28 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Yandex.Cloud Operators +====================== + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-yandex/yq_operators.rst b/docs/apache-airflow-providers-yandex/operators/yq.rst similarity index 100% rename from docs/apache-airflow-providers-yandex/yq_operators.rst rename to docs/apache-airflow-providers-yandex/operators/yq.rst From a0ac5365dac3aaf88adcccaffb1707ff274593ed Mon Sep 17 00:00:00 2001 From: Sergey Uzhakov Date: Tue, 7 May 2024 16:06:09 +0300 Subject: [PATCH 6/6] fix links to how-to-guide --- airflow/providers/yandex/provider.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index 0bd1df02f7d23..4d8b6d6c9b602 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -62,14 +62,14 @@ integrations: - integration-name: Yandex.Cloud Dataproc external-doc-url: https://cloud.yandex.com/dataproc how-to-guide: - - /docs/apache-airflow-providers-yandex/dataproc_operators.rst + - /docs/apache-airflow-providers-yandex/operators/dataproc.rst logo: /integration-logos/yandex/Yandex-Cloud.png tags: [service] - integration-name: Yandex.Cloud YQ external-doc-url: https://cloud.yandex.com/en/services/query how-to-guide: - - /docs/apache-airflow-providers-yandex/yq_operators.rst + - /docs/apache-airflow-providers-yandex/operators/yq.rst logo: /integration-logos/yandex/Yandex-Cloud.png tags: [service]