From f54e25c55f60e8c8e4c0df4d7931a56c0433751a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Wed, 20 May 2020 15:22:26 +0200 Subject: [PATCH 1/4] Fix references in docs --- airflow/hooks/dbapi_hook.py | 8 ++--- airflow/macros/__init__.py | 4 +-- airflow/macros/hive.py | 2 +- airflow/models/dag.py | 2 +- airflow/operators/bash.py | 2 +- .../providers/amazon/aws/hooks/datasync.py | 2 +- .../providers/amazon/aws/hooks/sagemaker.py | 4 +-- airflow/providers/amazon/aws/hooks/sqs.py | 4 +-- .../kubernetes/operators/kubernetes_pod.py | 2 +- airflow/providers/exasol/hooks/exasol.py | 8 ++--- airflow/providers/exasol/operators/exasol.py | 2 +- .../google/cloud/hooks/natural_language.py | 10 +++--- airflow/providers/google/cloud/hooks/tasks.py | 8 ++--- .../providers/google/cloud/hooks/vision.py | 32 +++++++++---------- .../google/cloud/operators/automl.py | 2 +- .../google/cloud/operators/bigtable.py | 4 +-- .../google/cloud/operators/cloud_sql.py | 2 +- .../google/cloud/operators/dataproc.py | 2 +- .../cloud/operators/kubernetes_engine.py | 10 ------ .../providers/google/cloud/operators/tasks.py | 8 ++--- .../providers/google/cloud/sensors/pubsub.py | 4 +-- .../cloud/utils/credentials_provider.py | 6 ++-- airflow/providers/imap/hooks/imap.py | 2 +- airflow/providers/jdbc/operators/jdbc.py | 2 +- .../microsoft/mssql/operators/mssql.py | 2 +- airflow/providers/mysql/operators/mysql.py | 2 +- airflow/providers/oracle/operators/oracle.py | 2 +- .../providers/pagerduty/hooks/pagerduty.py | 2 +- .../providers/postgres/operators/postgres.py | 2 +- airflow/providers/qubole/sensors/qubole.py | 4 +-- .../providers/salesforce/hooks/salesforce.py | 10 +++--- .../snowflake/operators/snowflake.py | 2 +- airflow/providers/sqlite/operators/sqlite.py | 2 +- .../yandex/operators/yandexcloud_dataproc.py | 2 +- airflow/sensors/sql_sensor.py | 2 +- docs/best-practices.rst | 2 +- docs/build | 29 +++++++++++++---- docs/concepts.rst | 4 +-- docs/conf.py | 7 ++-- docs/executor/celery.rst | 2 +- docs/executor/debug.rst | 2 +- docs/exts/docroles.py | 2 +- docs/howto/connection/gcp_sql.rst | 10 +++--- docs/howto/connection/index.rst | 4 +-- docs/howto/connection/mysql.rst | 2 +- docs/howto/connection/oracle.rst | 2 +- docs/howto/connection/postgres.rst | 2 +- docs/howto/connection/ssh.rst | 6 ++-- docs/howto/custom-operator.rst | 4 +-- docs/howto/define_extra_link.rst | 2 +- docs/howto/operator/amazon/aws/datasync.rst | 2 +- docs/howto/operator/amazon/aws/emr.rst | 2 +- docs/howto/operator/dingding.rst | 2 +- docs/howto/operator/gcp/cloud_sql.rst | 4 +-- docs/howto/operator/gcp/gcs.rst | 2 +- docs/howto/operator/gcp/presto_to_gcs.rst | 4 +-- docs/howto/operator/gcp/sheets.rst | 2 +- docs/howto/operator/gcp/vision.rst | 20 ++++++------ docs/howto/operator/papermill.rst | 2 +- docs/howto/run-with-systemd.rst | 2 +- docs/howto/set-config.rst | 2 +- docs/howto/write-logs.rst | 4 +-- docs/operators-and-hooks-ref.rst | 12 +++---- scripts/perf/perf_kit/__init__.py | 2 +- tests/test_impersonation.py | 4 +-- tests/test_utils/gcp_system_helpers.py | 2 +- 66 files changed, 162 insertions(+), 150 deletions(-) diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py index c04f64fe80546..2ccbd686ad4ac 100644 --- a/airflow/hooks/dbapi_hook.py +++ b/airflow/hooks/dbapi_hook.py @@ -115,7 +115,7 @@ def get_pandas_df(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ import pandas.io.sql as psql @@ -130,7 +130,7 @@ def get_records(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ with closing(self.get_conn()) as conn: with closing(conn.cursor()) as cur: @@ -148,7 +148,7 @@ def get_first(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ with closing(self.get_conn()) as conn: with closing(conn.cursor()) as cur: @@ -171,7 +171,7 @@ def run(self, sql, autocommit=False, parameters=None): before executing the query. :type autocommit: bool :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ if isinstance(sql, str): sql = [sql] diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py index 9123bceb85f1a..c58455f9a2f7a 100644 --- a/airflow/macros/__init__.py +++ b/airflow/macros/__init__.py @@ -73,10 +73,10 @@ def datetime_diff_for_humans(dt, since=None): one and now. :param dt: The datetime to display the diff for - :type dt: datetime + :type dt: datetime.datetime :param since: When to display the date from. If ``None`` then the diff is between ``dt`` and now. - :type since: None or datetime + :type since: None or datetime.datetime :rtype: str """ import pendulum diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py index 285ce8e1a50f6..107f93fafd310 100644 --- a/airflow/macros/hive.py +++ b/airflow/macros/hive.py @@ -38,7 +38,7 @@ def max_partition( e.g. {'key1': 'value1', 'key2': 'value2'}. Only partitions matching all partition_key:partition_value pairs will be considered as candidates of max partition. - :type filter_map: map + :type filter_map: dict :param field: the field to get the max value from. If there's only one partition field, this will be inferred :type field: str diff --git a/airflow/models/dag.py b/airflow/models/dag.py index 5699ba9fa22dd..fdc5651096553 100644 --- a/airflow/models/dag.py +++ b/airflow/models/dag.py @@ -1385,7 +1385,7 @@ def run( :param local: True to run the tasks using the LocalExecutor :type local: bool :param executor: The executor instance to run the tasks - :type executor: airflow.executor.BaseExecutor + :type executor: airflow.executor.base_executor.BaseExecutor :param donot_pickle: True to avoid pickling DAG object and send to workers :type donot_pickle: bool :param ignore_task_deps: True to skip upstream tasks diff --git a/airflow/operators/bash.py b/airflow/operators/bash.py index f70ba2dbbed02..9eef881e0ec8f 100644 --- a/airflow/operators/bash.py +++ b/airflow/operators/bash.py @@ -43,7 +43,7 @@ class BashOperator(BaseOperator): :param bash_command: The command, set of commands or reference to a bash script (must be '.sh') to be executed. (templated) :type bash_command: str - :param env: If env is not None, it must be a mapping that defines the + :param env: If env is not None, it must be a dict that defines the environment variables for the new process; these are used instead of inheriting the current process environment, which is the default behavior. (templated) diff --git a/airflow/providers/amazon/aws/hooks/datasync.py b/airflow/providers/amazon/aws/hooks/datasync.py index 140cc5897369a..b9a786e353be3 100644 --- a/airflow/providers/amazon/aws/hooks/datasync.py +++ b/airflow/providers/amazon/aws/hooks/datasync.py @@ -191,7 +191,7 @@ def get_task_arns_for_location_arns( :param list source_location_arns: List of source LocationArns. :param list destination_location_arns: List of destination LocationArns. :return: list - :rtype: List of TaskArns. + :rtype: list(TaskArns) :raises AirflowBadRequest: if ``source_location_arns`` or ``destination_location_arns`` are empty. """ if not source_location_arns: diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py b/airflow/providers/amazon/aws/hooks/sagemaker.py index cba7255260ca7..82a2ad5329835 100644 --- a/airflow/providers/amazon/aws/hooks/sagemaker.py +++ b/airflow/providers/amazon/aws/hooks/sagemaker.py @@ -238,10 +238,10 @@ def check_tuning_config(self, tuning_config): def get_log_conn(self): """ This method is deprecated. - Please use :py:meth:`airflow.contrib.hooks.AwsLogsHook.get_conn` instead. + Please use :py:meth:`airflow.providers.amazon.aws.hooks.logs.AwsLogsHook.get_conn` instead. """ warnings.warn("Method `get_log_conn` has been deprecated. " - "Please use `airflow.contrib.hooks.AwsLogsHook.get_conn` instead.", + "Please use `airflow.providers.amazon.aws.hooks.logs.AwsLogsHook.get_conn` instead.", category=DeprecationWarning, stacklevel=2) diff --git a/airflow/providers/amazon/aws/hooks/sqs.py b/airflow/providers/amazon/aws/hooks/sqs.py index 15e76dbb07369..849979b2b7f46 100644 --- a/airflow/providers/amazon/aws/hooks/sqs.py +++ b/airflow/providers/amazon/aws/hooks/sqs.py @@ -43,11 +43,11 @@ def create_queue(self, queue_name, attributes=None): :param queue_name: name of the queue. :type queue_name: str :param attributes: additional attributes for the queue (default: None) - For details of the attributes parameter see :py:meth:`botocore.client.SQS.create_queue` + For details of the attributes parameter see :py:meth:`SQS.create_queue` :type attributes: dict :return: dict with the information about the queue - For details of the returned value see :py:meth:`botocore.client.SQS.create_queue` + For details of the returned value see :py:meth:`SQS.create_queue` :rtype: dict """ return self.get_conn().create_queue(QueueName=queue_name, Attributes=attributes or {}) diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py index bc5679db7b08a..3c4bf6fbd0e99 100644 --- a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +++ b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py @@ -41,7 +41,7 @@ class KubernetesPodOperator(BaseOperator): # pylint: disable=too-many-instance- .. note:: If you use `Google Kubernetes Engine `__, use - :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEPodOperator`, which + :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator`, which simplifies the authorization process. :param namespace: the namespace to run within kubernetes. diff --git a/airflow/providers/exasol/hooks/exasol.py b/airflow/providers/exasol/hooks/exasol.py index 529db581808e0..82b940d9da198 100644 --- a/airflow/providers/exasol/hooks/exasol.py +++ b/airflow/providers/exasol/hooks/exasol.py @@ -66,7 +66,7 @@ def get_pandas_df(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ with closing(self.get_conn()) as conn: conn.export_to_pandas(sql, query_params=parameters) @@ -79,7 +79,7 @@ def get_records(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ with closing(self.get_conn()) as conn: with closing(conn.execute(sql, parameters)) as cur: @@ -93,7 +93,7 @@ def get_first(self, sql, parameters=None): sql statements to execute :type sql: str or list :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ with closing(self.get_conn()) as conn: with closing(conn.execute(sql, parameters)) as cur: @@ -112,7 +112,7 @@ def run(self, sql, autocommit=False, parameters=None): before executing the query. :type autocommit: bool :param parameters: The parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ if isinstance(sql, basestring): sql = [sql] diff --git a/airflow/providers/exasol/operators/exasol.py b/airflow/providers/exasol/operators/exasol.py index 4a5e9626f6838..090f1b989e513 100644 --- a/airflow/providers/exasol/operators/exasol.py +++ b/airflow/providers/exasol/operators/exasol.py @@ -36,7 +36,7 @@ class ExasolOperator(BaseOperator): (default value: False) :type autocommit: bool :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping + :type parameters: dict :param schema: (optional) name of the schema which overwrite defined one in connection :type schema: string """ diff --git a/airflow/providers/google/cloud/hooks/natural_language.py b/airflow/providers/google/cloud/hooks/natural_language.py index fb37006fc8978..326c4bfa55415 100644 --- a/airflow/providers/google/cloud/hooks/natural_language.py +++ b/airflow/providers/google/cloud/hooks/natural_language.py @@ -76,7 +76,7 @@ def analyze_entities( :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document - :type document: dict or class google.cloud.language_v1.types.Document + :type document: dict or google.cloud.language_v1.types.Document :param encoding_type: The encoding type used by the API to calculate offsets. :type encoding_type: google.cloud.language_v1.enums.EncodingType :param retry: A retry object used to retry requests. If None is specified, requests will not be @@ -110,7 +110,7 @@ def analyze_entity_sentiment( :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document - :type document: dict or class google.cloud.language_v1.types.Document + :type document: dict or google.cloud.language_v1.types.Document :param encoding_type: The encoding type used by the API to calculate offsets. :type encoding_type: google.cloud.language_v1.enums.EncodingType :param retry: A retry object used to retry requests. If None is specified, requests will not be @@ -143,7 +143,7 @@ def analyze_sentiment( :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document - :type document: dict or class google.cloud.language_v1.types.Document + :type document: dict or google.cloud.language_v1.types.Document :param encoding_type: The encoding type used by the API to calculate offsets. :type encoding_type: google.cloud.language_v1.enums.EncodingType :param retry: A retry object used to retry requests. If None is specified, requests will not be @@ -177,7 +177,7 @@ def analyze_syntax( :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document - :type document: dict or class google.cloud.language_v1.types.Document# + :type document: dict or google.cloud.language_v1.types.Document :param encoding_type: The encoding type used by the API to calculate offsets. :type encoding_type: google.cloud.language_v1.enums.EncodingType :param retry: A retry object used to retry requests. If None is specified, requests will not be @@ -252,7 +252,7 @@ def classify_text( :param document: Input document. If a dict is provided, it must be of the same form as the protobuf message Document - :type document: dict or class google.cloud.language_v1.types.Document + :type document: dict or google.cloud.language_v1.types.Document :param retry: A retry object used to retry requests. If None is specified, requests will not be retried. :type retry: google.api_core.retry.Retry diff --git a/airflow/providers/google/cloud/hooks/tasks.py b/airflow/providers/google/cloud/hooks/tasks.py index a567cc5269d6c..f28b58afadf3e 100644 --- a/airflow/providers/google/cloud/hooks/tasks.py +++ b/airflow/providers/google/cloud/hooks/tasks.py @@ -84,7 +84,7 @@ def create_queue( :param task_queue: The task queue to create. Queue's name cannot be the same as an existing queue. If a dict is provided, it must be of the same form as the protobuf message Queue. - :type task_queue: dict or class google.cloud.tasks_v2.types.Queue + :type task_queue: dict or google.cloud.tasks_v2.types.Queue :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str @@ -140,7 +140,7 @@ def update_queue( :param task_queue: The task queue to update. This method creates the queue if it does not exist and updates the queue if it does exist. The queue's name must be specified. - :type task_queue: dict or class google.cloud.tasks_v2.types.Queue + :type task_queue: dict or google.cloud.tasks_v2.types.Queue :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str @@ -153,7 +153,7 @@ def update_queue( :param update_mask: A mast used to specify which fields of the queue are being updated. If empty, then all fields will be updated. If a dict is provided, it must be of the same form as the protobuf message. - :type update_mask: dict or class google.cloud.tasks_v2.types.FieldMask + :type update_mask: dict or google.cloud.tasks_v2.types.FieldMask :param retry: (Optional) A retry object used to retry requests. If None is specified, requests will not be retried. :type retry: google.api_core.retry.Retry @@ -449,7 +449,7 @@ def create_task( :type queue_name: str :param task: The task to add. If a dict is provided, it must be of the same form as the protobuf message Task. - :type task: dict or class google.cloud.tasks_v2.types.Task + :type task: dict or google.cloud.tasks_v2.types.Task :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str diff --git a/airflow/providers/google/cloud/hooks/vision.py b/airflow/providers/google/cloud/hooks/vision.py index 19a696ef4517f..acb9f7acec82a 100644 --- a/airflow/providers/google/cloud/hooks/vision.py +++ b/airflow/providers/google/cloud/hooks/vision.py @@ -175,7 +175,7 @@ def create_product_set( ) -> str: """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator` """ client = self.get_conn() parent = ProductSearchClient.location_path(project_id, location) @@ -209,7 +209,7 @@ def get_product_set( ) -> Dict: """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetGetOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionGetProductSetOperator` """ client = self.get_conn() name = ProductSearchClient.product_set_path(project_id, location, product_set_id) @@ -233,7 +233,7 @@ def update_product_set( ) -> Dict: """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetUpdateOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductSetOperator` """ client = self.get_conn() product_set = self.product_set_name_determiner.get_entity_with_name( @@ -259,7 +259,7 @@ def delete_product_set( ): """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetDeleteOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductSetOperator` """ client = self.get_conn() name = ProductSearchClient.product_set_path(project_id, location, product_set_id) @@ -280,7 +280,7 @@ def create_product( ): """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductCreateOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductOperator` """ client = self.get_conn() parent = ProductSearchClient.location_path(project_id, location) @@ -315,7 +315,7 @@ def get_product( ): """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductGetOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionGetProductOperator` """ client = self.get_conn() name = ProductSearchClient.product_path(project_id, location, product_id) @@ -339,7 +339,7 @@ def update_product( ): """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductUpdateOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductOperator` """ client = self.get_conn() product = self.product_name_determiner.get_entity_with_name(product, product_id, location, project_id) @@ -363,7 +363,7 @@ def delete_product( ): """ For the documentation see: - :class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionProductDeleteOperator` + :class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductOperator` """ client = self.get_conn() name = ProductSearchClient.product_path(project_id, location, product_id) @@ -453,7 +453,7 @@ def add_product_to_product_set( ) -> None: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionAddProductToProductSetOperator` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionAddProductToProductSetOperator` """ client = self.get_conn() @@ -481,7 +481,7 @@ def remove_product_from_product_set( ) -> None: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionRemoveProductFromProductSetOperator` # pylint: disable=line-too-long # noqa + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionRemoveProductFromProductSetOperator` # pylint: disable=line-too-long # noqa """ client = self.get_conn() @@ -504,7 +504,7 @@ def annotate_image( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_image_annotator_operator.CloudVisionAnnotateImage` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator` """ client = self.annotator_client @@ -526,7 +526,7 @@ def batch_annotate_images( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_image_annotator_operator.CloudVisionAnnotateImage` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator` """ client = self.annotator_client @@ -551,7 +551,7 @@ def text_detection( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectTextOperator` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectTextOperator` """ client = self.annotator_client @@ -581,7 +581,7 @@ def document_text_detection( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectDocumentTextOperator` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionTextDetectOperator` """ client = self.annotator_client @@ -611,7 +611,7 @@ def label_detection( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectImageLabelsOperator` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageLabelsOperator` """ client = self.annotator_client @@ -641,7 +641,7 @@ def safe_search_detection( ) -> Dict: """ For the documentation see: - :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectImageSafeSearchOperator` + :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageSafeSearchOperator` """ client = self.annotator_client diff --git a/airflow/providers/google/cloud/operators/automl.py b/airflow/providers/google/cloud/operators/automl.py index e0a1b91327ff0..d0589d18ca66b 100644 --- a/airflow/providers/google/cloud/operators/automl.py +++ b/airflow/providers/google/cloud/operators/automl.py @@ -778,7 +778,7 @@ def __init__( model_id: str, location: str, project_id: Optional[str] = None, - image_detection_metadata: Optional[Optional[Optional[dict]]] = None, + image_detection_metadata: Optional[dict] = None, metadata: Optional[Sequence[Tuple[str, str]]] = None, timeout: Optional[float] = None, retry: Optional[Retry] = None, diff --git a/airflow/providers/google/cloud/operators/bigtable.py b/airflow/providers/google/cloud/operators/bigtable.py index 826413281d1ef..5fe07f711f986 100644 --- a/airflow/providers/google/cloud/operators/bigtable.py +++ b/airflow/providers/google/cloud/operators/bigtable.py @@ -71,7 +71,7 @@ class BigtableCreateInstanceOperator(BaseOperator, BigtableValidationMixin): :param replica_cluster_id: (optional) The ID for replica cluster for the new instance. :type replica_cluster_zone: str :param replica_cluster_zone: (optional) The zone for replica cluster. - :type instance_type: enums.IntEnum + :type instance_type: enum.IntEnum :param instance_type: (optional) The type of the instance. :type instance_display_name: str :param instance_display_name: (optional) Human-readable name of the instance. Defaults @@ -81,7 +81,7 @@ class BigtableCreateInstanceOperator(BaseOperator, BigtableValidationMixin): with the instance. :type cluster_nodes: int :param cluster_nodes: (optional) Number of nodes for cluster. - :type cluster_storage_type: enums.IntEnum + :type cluster_storage_type: enum.IntEnum :param cluster_storage_type: (optional) The type of storage. :type timeout: int :param timeout: (optional) timeout (in seconds) for instance creation. diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py index 10ec0f2033e14..86ce6f7210c97 100644 --- a/airflow/providers/google/cloud/operators/cloud_sql.py +++ b/airflow/providers/google/cloud/operators/cloud_sql.py @@ -785,7 +785,7 @@ class CloudSQLExecuteQueryOperator(BaseOperator): :type gcp_conn_id: str :param gcp_cloudsql_conn_id: The connection ID used to connect to Google Cloud SQL its schema should be gcpcloudsql://. - See :class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` for + See :class:`~airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLDatabaseHook` for details on how to define gcpcloudsql:// connection. :type gcp_cloudsql_conn_id: str """ diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py index 1b01457e17a09..0f7233bb73f90 100644 --- a/airflow/providers/google/cloud/operators/dataproc.py +++ b/airflow/providers/google/cloud/operators/dataproc.py @@ -1294,7 +1294,7 @@ def __init__( main: str, arguments: Optional[List] = None, archives: Optional[List] = None, - pyfiles: Optional[Optional[Optional[List]]] = None, + pyfiles: Optional[List] = None, files: Optional[List] = None, *args, **kwargs diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 55bbdf89ac495..8c079b038c404 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -193,16 +193,6 @@ class GKEStartPodOperator(KubernetesPodOperator): ``task_id``, ``project_id``, ``location``, ``cluster_name``, ``name``, ``namespace``, and ``image`` - **Operator Creation**: :: - - operator = GKEPodOperator(task_id='pod_op', - project_id='my-project', - location='us-central1-a', - cluster_name='my-cluster-name', - name='task-name', - namespace='default', - image='perl') - .. seealso:: For more detail about application authentication have a look at the reference: https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application diff --git a/airflow/providers/google/cloud/operators/tasks.py b/airflow/providers/google/cloud/operators/tasks.py index 5dc7dcf16645f..e9a57e2a504d1 100644 --- a/airflow/providers/google/cloud/operators/tasks.py +++ b/airflow/providers/google/cloud/operators/tasks.py @@ -45,7 +45,7 @@ class CloudTasksQueueCreateOperator(BaseOperator): :param task_queue: The task queue to create. Queue's name cannot be the same as an existing queue. If a dict is provided, it must be of the same form as the protobuf message Queue. - :type task_queue: dict or class google.cloud.tasks_v2.types.Queue + :type task_queue: dict or google.cloud.tasks_v2.types.Queue :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str @@ -130,7 +130,7 @@ class CloudTasksQueueUpdateOperator(BaseOperator): :param task_queue: The task queue to update. This method creates the queue if it does not exist and updates the queue if it does exist. The queue's name must be specified. - :type task_queue: dict or class google.cloud.tasks_v2.types.Queue + :type task_queue: dict or google.cloud.tasks_v2.types.Queue :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str @@ -143,7 +143,7 @@ class CloudTasksQueueUpdateOperator(BaseOperator): :param update_mask: A mast used to specify which fields of the queue are being updated. If empty, then all fields will be updated. If a dict is provided, it must be of the same form as the protobuf message. - :type update_mask: dict or class google.cloud.tasks_v2.types.FieldMask + :type update_mask: dict or google.cloud.tasks_v2.types.FieldMask :param retry: (Optional) A retry object used to retry requests. If None is specified, requests will not be retried. :type retry: google.api_core.retry.Retry @@ -595,7 +595,7 @@ class CloudTasksTaskCreateOperator(BaseOperator): :type queue_name: str :param task: The task to add. If a dict is provided, it must be of the same form as the protobuf message Task. - :type task: dict or class google.cloud.tasks_v2.types.Task + :type task: dict or google.cloud.tasks_v2.types.Task :param project_id: (Optional) The ID of the GCP project that owns the Cloud Tasks. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str diff --git a/airflow/providers/google/cloud/sensors/pubsub.py b/airflow/providers/google/cloud/sensors/pubsub.py index 75bfab02d4ec8..d30320a3efa11 100644 --- a/airflow/providers/google/cloud/sensors/pubsub.py +++ b/airflow/providers/google/cloud/sensors/pubsub.py @@ -39,7 +39,7 @@ class PubSubPullSensor(BaseSensorOperator): .. seealso:: If you don't want to wait for at least one message to come, use Operator instead: - :class:`airflow.providers.google.cloud.operators.PubSubPullOperator` + :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubPullOperator` This sensor operator will pull up to ``max_messages`` messages from the specified PubSub subscription. When the subscription returns messages, @@ -68,7 +68,7 @@ class PubSubPullSensor(BaseSensorOperator): The Sensor task will (by definition) always wait for a message, regardless of this argument value. If you want a non-blocking task that does not to wait for messages, please use - :class:`airflow.providers.google.cloud.operators.PubSubPullOperator` + :class:`~airflow.providers.google.cloud.operators.pubsub.PubSubPullOperator` instead. :type return_immediately: bool :param ack_messages: If True, each message will be acknowledged diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/airflow/providers/google/cloud/utils/credentials_provider.py index d79af395ec445..c58424cb1081a 100644 --- a/airflow/providers/google/cloud/utils/credentials_provider.py +++ b/airflow/providers/google/cloud/utils/credentials_provider.py @@ -45,7 +45,7 @@ def build_gcp_conn( project_id: Optional[str] = None, ) -> str: """ - Builds a variable that can be used as ``AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT`` with provided service key, + Builds a uri that can be used as :envvar:`AIRFLOW_CONN_{CONN_ID}` with provided service key, scopes and project id. :param key_file_path: Path to service key. @@ -116,7 +116,7 @@ def provide_gcp_connection( project_id: Optional[str] = None, ): """ - Context manager that provides a temporary value of AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT + Context manager that provides a temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` connection. It build a new connection that includes path to provided service json, required scopes and project id. @@ -151,7 +151,7 @@ def provide_gcp_conn_and_credentials( - GCP credentials for application supporting `Application Default Credentials (ADC) strategy `__. - - temporary value of ``AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT`` connection + - temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` connection :param key_file_path: Path to file with GCP credentials .json file. :type key_file_path: str diff --git a/airflow/providers/imap/hooks/imap.py b/airflow/providers/imap/hooks/imap.py index 260dff63ea2fc..b7197bb7c1e63 100644 --- a/airflow/providers/imap/hooks/imap.py +++ b/airflow/providers/imap/hooks/imap.py @@ -285,7 +285,7 @@ def get_attachments_by_name(self, name, check_regex, find_first=False): :type find_first: bool :returns: a list of tuples each containing name and payload where the attachments name matches the given name. - :rtype: list of tuple + :rtype: list(tuple) """ attachments = [] diff --git a/airflow/providers/jdbc/operators/jdbc.py b/airflow/providers/jdbc/operators/jdbc.py index e626abd0ff6b2..b973f3fb3cc50 100644 --- a/airflow/providers/jdbc/operators/jdbc.py +++ b/airflow/providers/jdbc/operators/jdbc.py @@ -38,7 +38,7 @@ class JdbcOperator(BaseOperator): (default value: False) :type autocommit: bool :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ template_fields = ('sql',) diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/airflow/providers/microsoft/mssql/operators/mssql.py index 8fde7b0f15457..88f6146670d3e 100644 --- a/airflow/providers/microsoft/mssql/operators/mssql.py +++ b/airflow/providers/microsoft/mssql/operators/mssql.py @@ -39,7 +39,7 @@ class MsSqlOperator(BaseOperator): :param mssql_conn_id: reference to a specific mssql database :type mssql_conn_id: str :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable :param autocommit: if True, each command is automatically committed. (default value: False) :type autocommit: bool diff --git a/airflow/providers/mysql/operators/mysql.py b/airflow/providers/mysql/operators/mysql.py index 6ea7297ede06e..8dbf24fd38e4c 100644 --- a/airflow/providers/mysql/operators/mysql.py +++ b/airflow/providers/mysql/operators/mysql.py @@ -34,7 +34,7 @@ class MySqlOperator(BaseOperator): :param mysql_conn_id: reference to a specific mysql database :type mysql_conn_id: str :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable :param autocommit: if True, each command is automatically committed. (default value: False) :type autocommit: bool diff --git a/airflow/providers/oracle/operators/oracle.py b/airflow/providers/oracle/operators/oracle.py index 4a0d5b5d8c5af..19bc143ad2233 100644 --- a/airflow/providers/oracle/operators/oracle.py +++ b/airflow/providers/oracle/operators/oracle.py @@ -34,7 +34,7 @@ class OracleOperator(BaseOperator): :param oracle_conn_id: reference to a specific Oracle database :type oracle_conn_id: str :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable :param autocommit: if True, each command is automatically committed. (default value: False) :type autocommit: bool diff --git a/airflow/providers/pagerduty/hooks/pagerduty.py b/airflow/providers/pagerduty/hooks/pagerduty.py index de0a9d23a6bb9..d14971c53f88e 100644 --- a/airflow/providers/pagerduty/hooks/pagerduty.py +++ b/airflow/providers/pagerduty/hooks/pagerduty.py @@ -96,7 +96,7 @@ def create_event( :param class_type: The class/type of the event. :type class_type: str :param links: List of links to include. - :type class_type: list of str + :type class_type: list(str) """ if routing_key is None: routing_key = self.routing_key diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py index f11071435a18a..5304f74a36848 100644 --- a/airflow/providers/postgres/operators/postgres.py +++ b/airflow/providers/postgres/operators/postgres.py @@ -36,7 +36,7 @@ class PostgresOperator(BaseOperator): (default value: False) :type autocommit: bool :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable :param database: name of database which overwrite defined one in connection :type database: str """ diff --git a/airflow/providers/qubole/sensors/qubole.py b/airflow/providers/qubole/sensors/qubole.py index 1de779ba97dfe..fae8c281ca51c 100644 --- a/airflow/providers/qubole/sensors/qubole.py +++ b/airflow/providers/qubole/sensors/qubole.py @@ -76,7 +76,7 @@ class QuboleFileSensor(QuboleSensor): Check this `example `_ for sample payload structure. - :type data: a JSON object + :type data: dict .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can also use ``.txt`` files for template-driven use cases. @@ -99,7 +99,7 @@ class QubolePartitionSensor(QuboleSensor): Check this `example `_ for sample payload structure. - :type data: a JSON object + :type data: dict .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can also use ``.txt`` files for template-driven use cases. diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index 66aa0dd2e2c11..f7fce1ad405e2 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -121,7 +121,7 @@ def get_available_fields(self, obj): :param obj: The name of the Salesforce object that we are getting a description of. :type obj: str :return: the names of the fields. - :rtype: list of str + :rtype: list(str) """ self.get_conn() @@ -157,9 +157,9 @@ def _to_timestamp(cls, column): Convert a column of a dataframe to UNIX timestamps if applicable :param column: A Series object representing a column of a dataframe. - :type column: pd.Series + :type column: pandas.Series :return: a new series that maintains the same index as the original - :rtype: pd.Series + :rtype: pandas.Series """ # try and convert the column to datetimes # the column MUST have a four digit year somewhere in the string @@ -230,7 +230,7 @@ def write_object_to_file(self, to the resulting data that marks when the data was fetched from Salesforce. Default: False :type record_time_added: bool :return: the dataframe that gets written to the file. - :rtype: pd.Dataframe + :rtype: pandas.Dataframe """ fmt = fmt.lower() if fmt not in ['csv', 'json', 'ndjson']: @@ -285,7 +285,7 @@ def object_to_df(self, query_results, coerce_to_timestamp=False, to the resulting data that marks when the data was fetched from Salesforce. Default: False :type record_time_added: bool :return: the dataframe. - :rtype: pd.Dataframe + :rtype: pandas.Dataframe """ # this line right here will convert all integers to floats diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py index aa6db34690c59..f8410e65a5b49 100644 --- a/airflow/providers/snowflake/operators/snowflake.py +++ b/airflow/providers/snowflake/operators/snowflake.py @@ -34,7 +34,7 @@ class SnowflakeOperator(BaseOperator): (default value: True) :type autocommit: bool :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable :param warehouse: name of warehouse (will overwrite any warehouse defined in the connection's extra JSON) :type warehouse: str diff --git a/airflow/providers/sqlite/operators/sqlite.py b/airflow/providers/sqlite/operators/sqlite.py index 6357f67b1b376..1310b728939fd 100644 --- a/airflow/providers/sqlite/operators/sqlite.py +++ b/airflow/providers/sqlite/operators/sqlite.py @@ -32,7 +32,7 @@ class SqliteOperator(BaseOperator): :param sqlite_conn_id: reference to a specific sqlite database :type sqlite_conn_id: str :param parameters: (optional) the parameters to render the SQL query with. - :type parameters: mapping or iterable + :type parameters: dict or iterable """ template_fields = ('sql',) diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py index a9bd24d26f0d3..8ba4899a7d4ae 100644 --- a/airflow/providers/yandex/operators/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/operators/yandexcloud_dataproc.py @@ -445,7 +445,7 @@ class DataprocCreatePysparkJobOperator(BaseOperator): # pylint: disable=too-many-arguments @apply_defaults def __init__(self, - main_python_file_uri: Optional[Optional[str]] = None, + main_python_file_uri: Optional[str] = None, python_file_uris: Optional[Iterable[str]] = None, jar_file_uris: Optional[Iterable[str]] = None, archive_uris: Optional[Iterable[str]] = None, diff --git a/airflow/sensors/sql_sensor.py b/airflow/sensors/sql_sensor.py index 3700da4ba692f..e4f8e3aa6a7d5 100644 --- a/airflow/sensors/sql_sensor.py +++ b/airflow/sensors/sql_sensor.py @@ -40,7 +40,7 @@ class SqlSensor(BaseSensorOperator): that contains a non-zero / empty string value. :type sql: str :param parameters: The parameters to render the SQL query with (optional). - :type parameters: mapping or iterable + :type parameters: dict or iterable :param success: Success criteria for the sensor is a Callable that takes first_cell as the only argument, and returns a boolean (optional). :type: success: Optional diff --git a/docs/best-practices.rst b/docs/best-practices.rst index 7dfd62a945aba..26e3342efbdfe 100644 --- a/docs/best-practices.rst +++ b/docs/best-practices.rst @@ -303,7 +303,7 @@ Configuration Airflow comes bundled with a default ``airflow.cfg`` configuration file. You should use environment variables for configurations that change across deployments -e.g. metadata DB, password, etc. You can accomplish this using the format ``$AIRFLOW__{SECTION}__{KEY}`` +e.g. metadata DB, password, etc. You can accomplish this using the format :envvar:`AIRFLOW__{SECTION}__{KEY}` .. code:: diff --git a/docs/build b/docs/build index db62d925dcf85..9f4d162b33788 100755 --- a/docs/build +++ b/docs/build @@ -21,6 +21,7 @@ import re import shlex import shutil import sys +from functools import total_ordering from subprocess import run from contextlib import suppress from glob import glob @@ -34,11 +35,27 @@ if __name__ != "__main__": ) +@total_ordering class DocBuildError(NamedTuple): file_path: Optional[str] line_no: Optional[int] message: str + def __eq__(self, other): + left = (self.file_path, self.line_no, self.message) + right = (other.file_path, other.line_no, other.message) + return left == right + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + file_path_a = self.file_path or '' + file_path_b = other.file_path or '' + line_no_a = self.line_no or 0 + line_no_b = other.line_no or 0 + return (file_path_a, line_no_a, self.message) < (file_path_b, line_no_b, other.message) + build_errors: List[DocBuildError] = [] @@ -294,15 +311,15 @@ def build_sphinx_docs() -> None: with NamedTemporaryFile() as tmp_file: build_cmd = [ "sphinx-build", - "-b", + "-b", # builder to use "html", - "-d", + "-d", # path for the cached environment and doctree files "_build/doctrees", - "--color", - "-w", + "--color", # do emit colored output + "-w", # turn warnings into errors tmp_file.name, - ".", - "_build/html", + ".", # path to documentation source files + "_build/html", # path to output directory ] print("Executing cmd: ", " ".join([shlex.quote(c) for c in build_cmd])) diff --git a/docs/concepts.rst b/docs/concepts.rst index c8d5942a9e3fa..27c03ea6a5060 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -705,7 +705,7 @@ Storing Variables in Environment Variables .. versionadded:: 1.10.10 Airflow Variables can also be created and managed using Environment Variables. The environment variable -naming convention is ``AIRFLOW_VAR_``, all uppercase. +naming convention is :envvar:`AIRFLOW_VAR_{VARIABLE_NAME}`, all uppercase. So if your variable key is ``FOO`` then the variable name should be ``AIRFLOW_VAR_FOO``. For example, @@ -1066,7 +1066,7 @@ may look like inside your ``airflow_local_settings.py``: if task.timeout > timedelta(hours=48): task.timeout = timedelta(hours=48) -To define policy, add a ``airflow_local_settings`` module to your PYTHONPATH +To define policy, add a ``airflow_local_settings`` module to your :envvar:`PYTHONPATH` or to AIRFLOW_HOME/config folder that defines this ``policy`` function. It receives a ``TaskInstance`` object and can alter it where needed. diff --git a/docs/conf.py b/docs/conf.py index 95d2f5ec26d94..2aafb3074badd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -271,13 +271,16 @@ intersphinx_mapping = { 'boto3': ('https://boto3.amazonaws.com/v1/documentation/api/latest/', None), + 'celery': ('https://docs.celeryproject.org/en/stable/', None), + 'hdfs': ('https://hdfscli.readthedocs.io/en/latest/', None), + 'jinja2': ('https://jinja.palletsprojects.com/en/master/', None), 'mongodb': ('https://api.mongodb.com/python/current/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'python': ('https://docs.python.org/3/', None), 'requests': ('https://requests.readthedocs.io/en/master/', None), 'sqlalchemy': ('https://docs.sqlalchemy.org/en/latest/', None), - 'hdfs': ('https://hdfscli.readthedocs.io/en/latest/', None), - # google-cloud-python + # google-api + 'google-api-core': ('https://googleapis.dev/python/google-api-core/latest', None), 'google-cloud-automl': ('https://googleapis.dev/python/automl/latest', None), 'google-cloud-bigquery': ('https://googleapis.dev/python/bigquery/latest', None), 'google-cloud-bigquery-datatransfer': ('https://googleapis.dev/python/bigquerydatatransfer/latest', None), diff --git a/docs/executor/celery.rst b/docs/executor/celery.rst index 233761cea136c..a3fa7edea7ff8 100644 --- a/docs/executor/celery.rst +++ b/docs/executor/celery.rst @@ -37,7 +37,7 @@ Here are a few imperative requirements for your workers: met in that context. For example, if you use the ``HiveOperator``, the hive CLI needs to be installed on that box, or if you use the ``MySqlOperator``, the required Python library needs to be available in - the ``PYTHONPATH`` somehow + the :envvar:`PYTHONPATH` somehow - The worker needs to have access to its ``DAGS_FOLDER``, and you need to synchronize the filesystems by your own means. A common setup would be to store your ``DAGS_FOLDER`` in a Git repository and sync it across machines using diff --git a/docs/executor/debug.rst b/docs/executor/debug.rst index c4c0d7e4d02f4..93218749480ab 100644 --- a/docs/executor/debug.rst +++ b/docs/executor/debug.rst @@ -23,7 +23,7 @@ Debug Executor The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as a debug tool and can be used from IDE. It is a single process executor that -queues :class:`~models.taskinstance.TaskInstance` and executes them by running +queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by running ``_run_raw_task`` method. Due to its nature the executor can be used with SQLite database. When used diff --git a/docs/exts/docroles.py b/docs/exts/docroles.py index 14150f9a80c3c..4f74a66f6958c 100644 --- a/docs/exts/docroles.py +++ b/docs/exts/docroles.py @@ -33,7 +33,7 @@ def get_template_field(env, fullname): Gets template fields for specific operator class. :param fullname: Full path to operator class. - For example: ``airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator`` + For example: ``airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator`` :return: List of template field :rtype: list[str] """ diff --git a/docs/howto/connection/gcp_sql.rst b/docs/howto/connection/gcp_sql.rst index 3b42029f9d93a..79fa99c0456b1 100644 --- a/docs/howto/connection/gcp_sql.rst +++ b/docs/howto/connection/gcp_sql.rst @@ -21,14 +21,14 @@ Google Cloud SQL Connection =========================== The ``gcpcloudsql://`` connection is used by -:class:`airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator` to perform query +:class:`airflow.providers.google.cloud.operators.cloud_sql.CloudSQLExecuteQueryOperator` to perform query on a Google Cloud SQL database. Google Cloud SQL database can be either Postgres or MySQL, so this is a "meta" connection type. It introduces common schema for both MySQL and Postgres, including what kind of connectivity should be used. Google Cloud SQL supports connecting via public IP or via Cloud SQL Proxy. In the latter case the -:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` uses -:class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlProxyRunner` to automatically prepare +:class:`~airflow.providers.google.cloud.operators.hooks.cloud_sql.CloudSQLHook` uses +:class:`~airflow.providers.google.cloud.operators.hooks.CloudSqlProxyRunner` to automatically prepare and use temporary Postgres or MySQL connection that will use the proxy to connect (either via TCP or UNIX socket. @@ -52,7 +52,7 @@ Extra (optional) connection. Details of all the parameters supported in extra field can be found in - :class:`~airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook` + :class:`~airflow.providers.google.cloud.operators.hooks.cloud_sql.CloudSQLHook`. Example "extras" field: @@ -67,7 +67,7 @@ Extra (optional) "sql_proxy_use_tcp": false } - When specifying the connection as URI (in ``AIRFLOW_CONN_*`` variable), you should specify + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable), you should specify it following the standard syntax of DB connection, where extras are passed as parameters of the URI. Note that all components of the URI should be URL-encoded. diff --git a/docs/howto/connection/index.rst b/docs/howto/connection/index.rst index 074114a19e03f..e8344b9193bba 100644 --- a/docs/howto/connection/index.rst +++ b/docs/howto/connection/index.rst @@ -92,7 +92,7 @@ Alternatively you may specify each parameter individually: Storing a Connection in Environment Variables --------------------------------------------- -The environment variable naming convention is ``AIRFLOW_CONN_``, all uppercase. +The environment variable naming convention is :envvar:`AIRFLOW_CONN_{CONN_ID}`, all uppercase. So if your connection id is ``my_prod_db`` then the variable name should be ``AIRFLOW_CONN_MY_PROD_DB``. @@ -221,7 +221,7 @@ For example if your password has a ``/``, this fails: >>> c = Connection(uri='my-conn-type://my-login:my-pa/ssword@my-host:5432/my-schema?param1=val1¶m2=val2') ValueError: invalid literal for int() with base 10: 'my-pa' -To fix this, you can encode with :py:meth:`~urllib.parse.quote_plus`: +To fix this, you can encode with :func:`~urllib.parse.quote_plus`: .. code-block:: pycon diff --git a/docs/howto/connection/mysql.rst b/docs/howto/connection/mysql.rst index 88699c3eeab27..54020d6d205cf 100644 --- a/docs/howto/connection/mysql.rst +++ b/docs/howto/connection/mysql.rst @@ -83,7 +83,7 @@ Extra (optional) "ssl": "{\"cert\": \"/tmp/client-cert.pem\", \"ca\": \"/tmp/server-ca.pem\", \"key\": \"/tmp/client-key.pem\"}" } - When specifying the connection as URI (in ``AIRFLOW_CONN_*`` variable) you should specify it + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it following the standard syntax of DB connections - where extras are passed as parameters of the URI. Note that all components of the URI should be URL-encoded. diff --git a/docs/howto/connection/oracle.rst b/docs/howto/connection/oracle.rst index 735a81a847439..adc43f7b7cb17 100644 --- a/docs/howto/connection/oracle.rst +++ b/docs/howto/connection/oracle.rst @@ -76,7 +76,7 @@ Extra (optional) "purity": "new" } - When specifying the connection as URI (in ``AIRFLOW_CONN_*`` variable) you should specify it + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it following the standard syntax of DB connections, where extras are passed as parameters of the URI (note that all components of the URI should be URL-encoded). diff --git a/docs/howto/connection/postgres.rst b/docs/howto/connection/postgres.rst index 509e449f23e48..cd69ae505ed78 100644 --- a/docs/howto/connection/postgres.rst +++ b/docs/howto/connection/postgres.rst @@ -70,7 +70,7 @@ Extra (optional) "sslkey": "/tmp/client-key.pem" } - When specifying the connection as URI (in ``AIRFLOW_CONN_*`` variable) you should specify it + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it following the standard syntax of DB connections, where extras are passed as parameters of the URI (note that all components of the URI should be URL-encoded). diff --git a/docs/howto/connection/ssh.rst b/docs/howto/connection/ssh.rst index 19425b9180511..6f19edcc30d57 100644 --- a/docs/howto/connection/ssh.rst +++ b/docs/howto/connection/ssh.rst @@ -19,7 +19,9 @@ SSH Connection ============== -The SSH connection type provides connection to use :class:`~airflow.providers.ssh.hooks.ssh.SSHHook` to run commands on a remote server using :class:`~airflow.providers.ssh.operators.ssh.SSHOperator` or transfer file from/to the remote server using :class:`~airflow.providers.ssh.operators.ssh.SFTPOperator`. +The SSH connection type provides connection to use :class:`~airflow.providers.ssh.hooks.ssh.SSHHook` to run +commands on a remote server using :class:`~airflow.providers.ssh.operators.ssh.SSHOperator` or transfer +file from/to the remote server using :class:`~airflow.providers.sftp.operators.sftp.SFTPOperator`. Configuring the Connection -------------------------- @@ -59,7 +61,7 @@ Extra (optional) "allow_host_key_change": "false" } - When specifying the connection as URI (in ``AIRFLOW_CONN_*`` variable) you should specify it + When specifying the connection as URI (in :envvar:`AIRFLOW_CONN_{CONN_ID}` variable) you should specify it following the standard syntax of connections, where extras are passed as parameters of the URI (note that all components of the URI should be URL-encoded). diff --git a/docs/howto/custom-operator.rst b/docs/howto/custom-operator.rst index 51f94dc168b17..b728e38cc8457 100644 --- a/docs/howto/custom-operator.rst +++ b/docs/howto/custom-operator.rst @@ -59,8 +59,8 @@ Let's implement an example ``HelloOperator`` in a new file ``hello_operator.py`` .. note:: For imports to work, you should place the file in a directory that - is present in the ``PYTHONPATH`` env. Airflow adds ``dags/``, ``plugins/``, and ``config/`` directories - in the Airflow home to ``PYTHONPATH`` by default. e.g., In our example, + is present in the :envvar:`PYTHONPATH` env. Airflow adds ``dags/``, ``plugins/``, and ``config/`` directories + in the Airflow home to :envvar:`PYTHONPATH` by default. e.g., In our example, the file is placed in the ``custom_operator`` directory. You can now use the derived custom operator as follows: diff --git a/docs/howto/define_extra_link.rst b/docs/howto/define_extra_link.rst index 557904539673e..ee6506a78cb55 100644 --- a/docs/howto/define_extra_link.rst +++ b/docs/howto/define_extra_link.rst @@ -101,7 +101,7 @@ tasks using :class:`~airflow.providers.amazon.aws.operators.gcs_to_s3.GCSToS3Ope **Overriding Operator Links of Existing Operators**: It is also possible to replace a built in link on an operator via a Plugin. For example -:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryOperator` includes a link to the GCP +:class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator` includes a link to the GCP console, but if we wanted to change that link we could: .. code-block:: python diff --git a/docs/howto/operator/amazon/aws/datasync.rst b/docs/howto/operator/amazon/aws/datasync.rst index ca52a30b60c37..0ae3e88e15a93 100644 --- a/docs/howto/operator/amazon/aws/datasync.rst +++ b/docs/howto/operator/amazon/aws/datasync.rst @@ -36,7 +36,7 @@ in action. - example_datasync_2.py Both examples use the :class:`~airflow.providers.amazon.aws.hooks.datasync.AWSDataSyncHook` -to create a boto3 DataSync client. This hook in turn uses the :class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsHook` +to create a boto3 DataSync client. This hook in turn uses the :class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook` Note this guide differentiates between an *Airflow task* (identified by a task_id on Airflow), and an *AWS DataSync Task* (identified by a TaskArn on AWS). diff --git a/docs/howto/operator/amazon/aws/emr.rst b/docs/howto/operator/amazon/aws/emr.rst index 2866e4c6af79b..dc70f862aaefb 100644 --- a/docs/howto/operator/amazon/aws/emr.rst +++ b/docs/howto/operator/amazon/aws/emr.rst @@ -35,7 +35,7 @@ Overview Airflow to AWS EMR integration provides several operators to create and interact with EMR service. - - :class:`~airflow.providers.amazon.aws.sensors.emr_step.EmrJobFlowSensor` + - :class:`~airflow.providers.amazon.aws.sensors.emr_job_flow.EmrJobFlowSensor` - :class:`~airflow.providers.amazon.aws.sensors.emr_step.EmrStepSensor` - :class:`~airflow.providers.amazon.aws.operators.emr_create_job_flow.EmrCreateJobFlowOperator` - :class:`~airflow.providers.amazon.aws.operators.emr_add_steps.EmrAddStepsOperator` diff --git a/docs/howto/operator/dingding.rst b/docs/howto/operator/dingding.rst index ed4877ee3df1e..aafee9379a649 100644 --- a/docs/howto/operator/dingding.rst +++ b/docs/howto/operator/dingding.rst @@ -34,7 +34,7 @@ To use this operators, you must do a few things: Basic Usage ^^^^^^^^^^^ -Use the :class:`~airflow.contrib.operators.dingding_operator.DingdingOperator` +Use the :class:`~airflow.providers.dingding.operators.dingding.DingdingOperator` to send Dingding message: .. exampleinclude:: ../../../airflow/providers/dingding/example_dags/example_dingding.py diff --git a/docs/howto/operator/gcp/cloud_sql.rst b/docs/howto/operator/gcp/cloud_sql.rst index 90c9e5720cd1a..276a6aeac6696 100644 --- a/docs/howto/operator/gcp/cloud_sql.rst +++ b/docs/howto/operator/gcp/cloud_sql.rst @@ -518,7 +518,7 @@ used to create tables in an idempotent way. Arguments """"""""" -If you define connection via ``AIRFLOW_CONN_*`` URL defined in an environment +If you define connection via :envvar:`AIRFLOW_CONN_{CONN_ID}` URL defined in an environment variable, make sure the URL components in the URL are URL-encoded. See examples below for details. @@ -539,7 +539,7 @@ Using the operator """""""""""""""""" Example operators below are using all connectivity options. Note connection id -from the operator matches the ``AIRFLOW_CONN_*`` postfix uppercase. This is +from the operator matches the :envvar:`AIRFLOW_CONN_{CONN_ID}` postfix uppercase. This is standard AIRFLOW notation for defining connection via environment variables): .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py diff --git a/docs/howto/operator/gcp/gcs.rst b/docs/howto/operator/gcp/gcs.rst index 9b184f2ce1527..b35659fef78bd 100644 --- a/docs/howto/operator/gcp/gcs.rst +++ b/docs/howto/operator/gcp/gcs.rst @@ -85,7 +85,7 @@ GCSObjectCreateAclEntryOperator Creates a new ACL entry on the specified object. For parameter definition, take a look at -:class:`~airflow.providers.google.cloud.operators.gcs_acl_operator.GCSObjectCreateAclEntryOperator` +:class:`~airflow.providers.google.cloud.operators.gcs.GCSObjectCreateAclEntryOperator` Using the operator """""""""""""""""" diff --git a/docs/howto/operator/gcp/presto_to_gcs.rst b/docs/howto/operator/gcp/presto_to_gcs.rst index 082c61eb0402b..d961e3cf48dd2 100644 --- a/docs/howto/operator/gcp/presto_to_gcs.rst +++ b/docs/howto/operator/gcp/presto_to_gcs.rst @@ -35,7 +35,7 @@ Data transfer ------------- Transfer files between Presto and Google Storage is performed with the -:class:`~airflow.providers.google.cloud.operators.presto_to_gcss.PrestoToGCSOperator` operator. +:class:`~airflow.providers.google.cloud.operators.presto_to_gcs.PrestoToGCSOperator` operator. This operator has 3 required parameters: @@ -45,7 +45,7 @@ This operator has 3 required parameters: A ``{}`` should be specified in the filename to allow the operator to inject file numbers in cases where the file is split due to size. -All parameters are described in the reference documentation - :class:`~airflow.providers.google.cloud.operators.presto_to_gcss.PrestoToGCSOperator`. +All parameters are described in the reference documentation - :class:`~airflow.providers.google.cloud.operators.presto_to_gcs.PrestoToGCSOperator`. An example operator call might look like this: diff --git a/docs/howto/operator/gcp/sheets.rst b/docs/howto/operator/gcp/sheets.rst index b1f5f6292226c..6ebc56f9b40e9 100644 --- a/docs/howto/operator/gcp/sheets.rst +++ b/docs/howto/operator/gcp/sheets.rst @@ -47,7 +47,7 @@ Create spreadsheet ^^^^^^^^^^^^^^^^^^ To create new spreadsheet you can use the -:class:`~airflow.providers.google.suite.operators.sheet.GoogleSheetsCreateSpreadsheet`. +:class:`~airflow.providers.google.suite.operators.sheets.GoogleSheetsCreateSpreadsheet`. .. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_sheets.py :language: python diff --git a/docs/howto/operator/gcp/vision.rst b/docs/howto/operator/gcp/vision.rst index e7a3fe1b080ac..372f23001c972 100644 --- a/docs/howto/operator/gcp/vision.rst +++ b/docs/howto/operator/gcp/vision.rst @@ -43,7 +43,7 @@ Using the operator """""""""""""""""" We are using the :class:`~google.cloud.vision_v1.types.Product`, -:class:`~google.cloud.vision_v1.types.ProductSet` and ``Retry`` objects from +:class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -108,7 +108,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`~google.cloud.enums` and ``Retry`` objects from +We are using the :class:`~google.cloud.vision.enums` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -171,7 +171,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the ``Product`` and ``Retry`` objects from Google libraries: +We are using the ``Product`` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py :language: python @@ -333,7 +333,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the ``ProductSet`` and ``Retry`` objects from Google libraries: +We are using the ``ProductSet`` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py :language: python @@ -634,7 +634,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and ``Retry`` objects from Google libraries: +We are using the :class:`~google.cloud.vision_v1.types.ReferenceImage` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py :language: python @@ -697,7 +697,7 @@ Using the operator """""""""""""""""" We are using the :class:`~google.cloud.vision_v1.types.Product`, -:class:`~google.cloud.vision_v1.types.ProductSet` and ``Retry`` objects from +:class:`~google.cloud.vision_v1.types.ProductSet` and :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -763,7 +763,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`Retry` objects from +We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -815,7 +815,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`Retry` objects from +We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -868,7 +868,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`Retry` objects from +We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py @@ -920,7 +920,7 @@ For parameter definition, take a look at Using the operator """""""""""""""""" -We are using the :class:`Retry` objects from +We are using the :class:`~google.api_core.retry.Retry` objects from Google libraries: .. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_vision.py diff --git a/docs/howto/operator/papermill.rst b/docs/howto/operator/papermill.rst index 05d568d8fd2cf..ac5e8b1fe0873 100644 --- a/docs/howto/operator/papermill.rst +++ b/docs/howto/operator/papermill.rst @@ -50,7 +50,7 @@ supports S3, GCS, Azure and Local. HDFS is *not* supported. Example DAG ''''''''''' -Use the :class:`~airflow.contrib.operators.papermill_operator.PapermillOperator` +Use the :class:`~airflow.providers.papermill.operators.papermill.PapermillOperator` to execute a jupyter notebook: .. exampleinclude:: ../../../airflow/providers/papermill/example_dags/example_papermill.py diff --git a/docs/howto/run-with-systemd.rst b/docs/howto/run-with-systemd.rst index ad6211a156ca1..92dafd5760cd4 100644 --- a/docs/howto/run-with-systemd.rst +++ b/docs/howto/run-with-systemd.rst @@ -37,4 +37,4 @@ If this is not the case, appropriate changes will need to be made. Please **note** that environment configuration is picked up from ``/etc/sysconfig/airflow``. An example file is supplied within ``scripts/systemd``. -You can also define configuration at ``AIRFLOW_HOME`` or ``AIRFLOW_CONFIG``. +You can also define configuration at :envvar:`AIRFLOW_HOME` or :envvar:`AIRFLOW_CONFIG`. diff --git a/docs/howto/set-config.rst b/docs/howto/set-config.rst index d1d30ec7a401a..241720fe325f1 100644 --- a/docs/howto/set-config.rst +++ b/docs/howto/set-config.rst @@ -23,7 +23,7 @@ Setting Configuration Options The first time you run Airflow, it will create a file called ``airflow.cfg`` in your ``$AIRFLOW_HOME`` directory (``~/airflow`` by default). This file contains Airflow's configuration and you can edit it to change any of the settings. You can also set options with environment variables by using this format: -``$AIRFLOW__{SECTION}__{KEY}`` (note the double underscores). +:envvar:`AIRFLOW__{SECTION}__{KEY}` (note the double underscores). For example, the metadata database connection string can either be set in ``airflow.cfg`` like this: diff --git a/docs/howto/write-logs.rst b/docs/howto/write-logs.rst index 0812483efdd43..1db6318e8d40e 100644 --- a/docs/howto/write-logs.rst +++ b/docs/howto/write-logs.rst @@ -47,7 +47,7 @@ Advanced configuration Not all configuration options are available from the ``airflow.cfg`` file. Some configuration options require that the logging config class be overwritten. This can be done by ``logging_config_class`` option in ``airflow.cfg`` file. This option should specify the import path indicating to a configuration compatible with -:class:`logging.config.dictConfig`. If your file is a standard import location, then you should set a :any:`PYTHONPATH` environment. +:func:`logging.config.dictConfig`. If your file is a standard import location, then you should set a :envvar:`PYTHONPATH` environment. Follow the steps below to enable custom logging config class: @@ -147,7 +147,7 @@ Airflow can be configured to read and write task logs in Azure Blob Storage. Follow the steps below to enable Azure Blob Storage logging: -#. Airflow's logging system requires a custom ``.py`` file to be located in the ``PYTHONPATH``, so that it's importable from Airflow. Start by creating a directory to store the config file, ``$AIRFLOW_HOME/config`` is recommended. +#. Airflow's logging system requires a custom ``.py`` file to be located in the :envvar:`PYTHONPATH`, so that it's importable from Airflow. Start by creating a directory to store the config file, ``$AIRFLOW_HOME/config`` is recommended. #. Create empty files called ``$AIRFLOW_HOME/config/log_config.py`` and ``$AIRFLOW_HOME/config/__init__.py``. #. Copy the contents of ``airflow/config_templates/airflow_local_settings.py`` into the ``log_config.py`` file created in ``Step 2``. #. Customize the following portions of the template: diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 31fc2c2bc68c7..4432fdd1b1409 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -585,7 +585,7 @@ These integrations allow you to copy data from/to Amazon Web Services. - :mod:`airflow.providers.amazon.aws.operators.sftp_to_s3` :ref:`[1] ` Those discovery-based operators use -:class:`airflow.providers.google.cloud.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google +:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google Services via the `Google API Python Client `__. Please note that this library is in maintenance mode hence it won't fully support GCP in the future. Therefore it is recommended that you use the custom GCP Service Operators for working with the Google @@ -598,7 +598,7 @@ Google Airflow has support for the `Google service `__. -All hooks are based on :class:`airflow.providers.google.common.hooks.base.GoogleBaseHook`. Some integration +All hooks are based on :class:`airflow.providers.google.common.hooks.base_google.GoogleBaseHook`. Some integration also use :mod:`airflow.providers.google.common.hooks.discovery_api`. See the :doc:`GCP connection type ` documentation to @@ -964,7 +964,7 @@ These integrations allow you to copy data from/to Google Cloud Platform. .. _integration:GCP-Discovery: :ref:`[1] ` Those discovery-based operators use -:class:`airflow.providers.google.cloud.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google +:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google Services via the `Google API Python Client `__. Please note that this library is in maintenance mode hence it won't fully support Google in the future. Therefore it is recommended that you use the custom Google Service Operators for working with the Google @@ -1067,7 +1067,7 @@ Airflow has a limited support for the `Yandex.Cloud ` See the :doc:`Yandex.Cloud connection type ` documentation to configure connections to Yandex.Cloud. -All hooks are based on :class:`airflow.contrib.hooks.yandexcloud_base_hook.YandexGoogleBaseHook`. +All hooks are based on :class:`airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook`. .. note:: You can learn how to use Yandex.Cloud integrations by analyzing the @@ -1093,7 +1093,7 @@ These integrations allow you to perform various operations within the Yandex.Clo * - `Base Classes `__ - :doc:`How to use ` - :mod:`airflow.providers.yandex.hooks.yandex` - - :mod:`airflow.providers.yandex.operators.yandexcloud_base_operator` + - - * - `Data Proc `__ @@ -1369,7 +1369,7 @@ These integrations allow you to perform various operations using various softwar * - `Presto `__ - - :mod:`airflow.providers.presto.hooks.presto` - - :mod:`airflow.providers.presto.operators.presto_check` + - - * - `Python `__ diff --git a/scripts/perf/perf_kit/__init__.py b/scripts/perf/perf_kit/__init__.py index 7eb70b9c6c449..48f2de8b88f72 100644 --- a/scripts/perf/perf_kit/__init__.py +++ b/scripts/perf/perf_kit/__init__.py @@ -21,7 +21,7 @@ Useful decorators and context managers used when testing the performance of various Airflow components. -To use this package, you must add the parent directory to the ``PYTHONPATH`` environment variable. If the +To use this package, you must add the parent directory to the :envvar:`PYTHONPATH` environment variable. If the repository root directory is ``/opt/airflow/``, then you need to run the following command: .. code-block:: bash diff --git a/tests/test_impersonation.py b/tests/test_impersonation.py index 53d6de0f17744..c09a8a4fa6b16 100644 --- a/tests/test_impersonation.py +++ b/tests/test_impersonation.py @@ -50,8 +50,8 @@ def mock_custom_module_path(path: str): """ - This decorator adds a path to sys.path to simulate running the current script with the ``PYTHONPATH`` - environment variable set and sets the environment variable ``PYTHONPATH`` to change the + This decorator adds a path to sys.path to simulate running the current script with the :envvar:`PYTHONPATH` + environment variable set and sets the environment variable :envvar:`PYTHONPATH` to change the module load directory for child scripts. """ def wrapper(func): diff --git a/tests/test_utils/gcp_system_helpers.py b/tests/test_utils/gcp_system_helpers.py index 5edf0b1a7ad26..6914f31cb4f83 100644 --- a/tests/test_utils/gcp_system_helpers.py +++ b/tests/test_utils/gcp_system_helpers.py @@ -72,7 +72,7 @@ def provide_gcp_context( - GCP credentials for application supporting `Application Default Credentials (ADC) strategy `__. - - temporary value of ``AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT`` connection + - temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` variable - the ``gcloud`` config directory isolated from user configuration Moreover it resolves full path to service keys so user can pass ``myservice.json`` From 1cbc27ea91fa68d08caebb1d697378f758ae9fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 23 May 2020 11:08:19 +0200 Subject: [PATCH 2/4] Update tests/test_impersonation.py --- tests/test_impersonation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_impersonation.py b/tests/test_impersonation.py index c09a8a4fa6b16..676a461dc2143 100644 --- a/tests/test_impersonation.py +++ b/tests/test_impersonation.py @@ -50,8 +50,9 @@ def mock_custom_module_path(path: str): """ - This decorator adds a path to sys.path to simulate running the current script with the :envvar:`PYTHONPATH` - environment variable set and sets the environment variable :envvar:`PYTHONPATH` to change the + This decorator adds a path to sys.path to simulate running the current script with the + :envvar:`PYTHONPATH` environment variable set and sets the environment variable + :envvar:`PYTHONPATH` to change the module load directory for child scripts. """ def wrapper(func): From 818e223f5c24972f1094909ee5f89db9ddce7447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 23 May 2020 13:16:46 +0200 Subject: [PATCH 3/4] Update test_impersonation.py --- tests/test_impersonation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_impersonation.py b/tests/test_impersonation.py index 676a461dc2143..56c5cdf43b0c3 100644 --- a/tests/test_impersonation.py +++ b/tests/test_impersonation.py @@ -50,10 +50,9 @@ def mock_custom_module_path(path: str): """ - This decorator adds a path to sys.path to simulate running the current script with the - :envvar:`PYTHONPATH` environment variable set and sets the environment variable - :envvar:`PYTHONPATH` to change the - module load directory for child scripts. + This decorator adds a path to sys.path to simulate running the current script with + the :envvar:`PYTHONPATH` environment variable set and sets the environment variable + :envvar:`PYTHONPATH` to change the module load directory for child scripts. """ def wrapper(func): @functools.wraps(func) From 88c99a9b44ae613cf63dd5041e2c7f5099ff5ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= Date: Sat, 23 May 2020 13:56:50 +0200 Subject: [PATCH 4/4] Update __init__.py --- scripts/perf/perf_kit/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/perf/perf_kit/__init__.py b/scripts/perf/perf_kit/__init__.py index 48f2de8b88f72..e60ff187c15d6 100644 --- a/scripts/perf/perf_kit/__init__.py +++ b/scripts/perf/perf_kit/__init__.py @@ -21,8 +21,8 @@ Useful decorators and context managers used when testing the performance of various Airflow components. -To use this package, you must add the parent directory to the :envvar:`PYTHONPATH` environment variable. If the -repository root directory is ``/opt/airflow/``, then you need to run the following command: +To use this package, you must add the parent directory to the :envvar:`PYTHONPATH` environment variable. +If the repository root directory is ``/opt/airflow/``, then you need to run the following command: .. code-block:: bash