From ba9c044d20ff784630a09eecc0a30029b0f5e199 Mon Sep 17 00:00:00 2001 From: mucio Date: Thu, 29 Oct 2020 15:39:51 +0100 Subject: [PATCH] Add How-to guide for JDBC Operator (#11472) --- .../providers/jdbc/example_dags/__init__.py | 16 +++ .../example_dags/example_jdbc_operator.py | 66 +++++++++++++ docs/howto/connection/jdbc.rst | 2 + docs/howto/operator/index.rst | 1 + docs/howto/operator/jdbc.rst | 97 +++++++++++++++++++ docs/operators-and-hooks-ref.rst | 2 +- 6 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 airflow/providers/jdbc/example_dags/__init__.py create mode 100644 airflow/providers/jdbc/example_dags/example_jdbc_operator.py create mode 100644 docs/howto/operator/jdbc.rst diff --git a/airflow/providers/jdbc/example_dags/__init__.py b/airflow/providers/jdbc/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/jdbc/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/jdbc/example_dags/example_jdbc_operator.py b/airflow/providers/jdbc/example_dags/example_jdbc_operator.py new file mode 100644 index 0000000000000..ca958aa63d40c --- /dev/null +++ b/airflow/providers/jdbc/example_dags/example_jdbc_operator.py @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Example DAG demonstrating the usage of the BashOperator.""" + +from datetime import timedelta + +from airflow import DAG +from airflow.operators.dummy_operator import DummyOperator +from airflow.providers.jdbc.operators.jdbc import JdbcOperator +from airflow.utils.dates import days_ago + +args = { + 'owner': 'airflow', +} + +with DAG( + dag_id='example_jdbc_operator', + default_args=args, + schedule_interval='0 0 * * *', + start_date=days_ago(2), + dagrun_timeout=timedelta(minutes=60), + tags=['example'], +) as dag: + + run_this_last = DummyOperator( + task_id='run_this_last', + dag=dag, + ) + + # [START howto_operator_jdbc_template] + delete_data = JdbcOperator( + task_id='delete_data', + sql='delete from my_schema.my_table where dt = {{ ds }}', + jdbc_conn_id='my_jdbc_connection', + autocommit=True, + dag=dag, + ) + # [END howto_operator_jdbc_template] + + # [START howto_operator_jdbc] + insert_data = JdbcOperator( + task_id='insert_data', + sql='insert into my_schema.my_table select dt, value from my_schema.source_data', + jdbc_conn_id='my_jdbc_connection', + autocommit=True, + dag=dag, + ) + # [END howto_operator_jdbc] + + delete_data >> insert_data >> run_this_last diff --git a/docs/howto/connection/jdbc.rst b/docs/howto/connection/jdbc.rst index dc5dbadc461bf..712f39dafd580 100644 --- a/docs/howto/connection/jdbc.rst +++ b/docs/howto/connection/jdbc.rst @@ -15,6 +15,8 @@ specific language governing permissions and limitations under the License. +.. _howto/connection:jdbc: + JDBC connection =============== diff --git a/docs/howto/operator/index.rst b/docs/howto/operator/index.rst index bc1d0d5df27f7..823bbd6c52d75 100644 --- a/docs/howto/operator/index.rst +++ b/docs/howto/operator/index.rst @@ -37,6 +37,7 @@ determine what actually executes when your DAG runs. dingding google/index http + jdbc kubernetes microsoft/index papermill diff --git a/docs/howto/operator/jdbc.rst b/docs/howto/operator/jdbc.rst new file mode 100644 index 0000000000000..3e1a82bbb40bf --- /dev/null +++ b/docs/howto/operator/jdbc.rst @@ -0,0 +1,97 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:JdbcOperator: + +JdbcOperator +============ + +Java Database Connectivity (JDBC) is an application programming interface +(API) for the programming language Java, which defines how a client may +access a database. + +.. contents:: + :depth: 1 + :local: + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +To use this operator you need: + + * Install the python module jaydebeapi: + .. code-block:: bash + + pip install jaydebeapi + + * Install a `JVM `_ and + add a ``JAVA_HOME`` env variable. + * Have the JDBC driver for your database installed. + +Once these prerequisites are satisfied you should be able to run +this Python snippet (replacing the variables values with the ones +related to your driver). + +Other error messages will inform you in case the ``jaydebeapi`` module +is missing or the driver is not available. A ``Connection Refused`` +error means that the connection string is pointing to host where no +database is listening for new connections. + + .. code-block:: python + + import apache-airflow[jdbc] + + driver_class = "com.exasol.jdbc.EXADriver" + driver_path = "/opt/airflow/drivers/exasol/EXASolution_JDBC-7.0.2/exajdbc.jar" + connection_url = "jdbc:exa:localhost" + credentials = ["", ""] + + conn = jaydebeapi.connect(driver_class, + connection_url, + credentials, + driver_path,) + +Usage +^^^^^ +Use the :class:`~airflow.providers.jdbc.operators.jdbc` to execute +commands against a database (or data storage) accessible via a JDBC driver. + +The :doc:`JDBC Connection ` must be passed as +``jdbc_conn_id``. + +.. exampleinclude:: /../airflow/providers/jdbc/example_dags/example_jdbc_operator.py + :language: python + :start-after: [START howto_operator_jdbc] + :end-before: [END howto_operator_jdbc] + +The parameter ``sql`` can receive a string or a list of strings. +Each string can be an SQL statement or a reference to a template file. +Template reference are recognized by ending in '.sql'. + +The parameter ``autocommit`` if set to ``True`` will execute a commit after +each command (default is ``False``) + +Templating +---------- + +You can use :ref:`Jinja templates ` to parameterize +``sql``. + +.. exampleinclude:: /../airflow/providers/jdbc/example_dags/example_jdbc_operator.py + :language: python + :start-after: [START howto_operator_jdbc_template] + :end-before: [END howto_operator_jdbc_template] diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index cd37930f5f6d0..4533a270d1803 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -1662,7 +1662,7 @@ communication protocols or interface. - :mod:`airflow.providers.imap.sensors.imap_attachment` * - `Java Database Connectivity (JDBC) `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.jdbc.hooks.jdbc` - :mod:`airflow.providers.jdbc.operators.jdbc` -