From 3b3287d7acc76430f12b758d52cec61c7f74e726 Mon Sep 17 00:00:00 2001 From: Ephraim Anierobi Date: Wed, 5 Aug 2020 19:47:16 +0100 Subject: [PATCH] Enforce keyword only arguments on apache operators (#10170) --- airflow/providers/apache/cassandra/sensors/record.py | 2 +- airflow/providers/apache/cassandra/sensors/table.py | 2 +- airflow/providers/apache/druid/operators/druid.py | 2 +- airflow/providers/apache/druid/operators/druid_check.py | 1 + airflow/providers/apache/druid/transfers/hive_to_druid.py | 2 +- airflow/providers/apache/hdfs/sensors/hdfs.py | 1 + airflow/providers/apache/hdfs/sensors/web_hdfs.py | 1 + airflow/providers/apache/hive/operators/hive.py | 2 +- airflow/providers/apache/hive/operators/hive_stats.py | 2 +- airflow/providers/apache/hive/sensors/hive_partition.py | 2 +- airflow/providers/apache/hive/sensors/metastore_partition.py | 2 +- airflow/providers/apache/hive/sensors/named_hive_partition.py | 2 +- airflow/providers/apache/hive/transfers/hive_to_mysql.py | 2 +- airflow/providers/apache/hive/transfers/hive_to_samba.py | 2 +- airflow/providers/apache/hive/transfers/mssql_to_hive.py | 2 +- airflow/providers/apache/hive/transfers/mysql_to_hive.py | 1 + airflow/providers/apache/hive/transfers/s3_to_hive.py | 1 + airflow/providers/apache/hive/transfers/vertica_to_hive.py | 1 + airflow/providers/apache/kylin/operators/kylin_cube.py | 2 +- airflow/providers/apache/livy/operators/livy.py | 2 +- airflow/providers/apache/livy/sensors/livy.py | 2 +- airflow/providers/apache/pig/operators/pig.py | 2 +- airflow/providers/apache/spark/operators/spark_jdbc.py | 2 +- airflow/providers/apache/spark/operators/spark_sql.py | 2 +- airflow/providers/apache/spark/operators/spark_submit.py | 2 +- airflow/providers/apache/sqoop/operators/sqoop.py | 2 +- 26 files changed, 26 insertions(+), 20 deletions(-) diff --git a/airflow/providers/apache/cassandra/sensors/record.py b/airflow/providers/apache/cassandra/sensors/record.py index 89fafc6443338..ea67ac5841c57 100644 --- a/airflow/providers/apache/cassandra/sensors/record.py +++ b/airflow/providers/apache/cassandra/sensors/record.py @@ -56,7 +56,7 @@ class CassandraRecordSensor(BaseSensorOperator): template_fields = ('table', 'keys') @apply_defaults - def __init__(self, table: str, keys: Dict[str, str], cassandra_conn_id: str, **kwargs: Any) -> None: + def __init__(self, *, table: str, keys: Dict[str, str], cassandra_conn_id: str, **kwargs: Any) -> None: super().__init__(**kwargs) self.cassandra_conn_id = cassandra_conn_id self.table = table diff --git a/airflow/providers/apache/cassandra/sensors/table.py b/airflow/providers/apache/cassandra/sensors/table.py index 89385773289ca..82cd411c0ada4 100644 --- a/airflow/providers/apache/cassandra/sensors/table.py +++ b/airflow/providers/apache/cassandra/sensors/table.py @@ -54,7 +54,7 @@ class CassandraTableSensor(BaseSensorOperator): template_fields = ('table',) @apply_defaults - def __init__(self, table: str, cassandra_conn_id: str, **kwargs: Any) -> None: + def __init__(self, *, table: str, cassandra_conn_id: str, **kwargs: Any) -> None: super().__init__(**kwargs) self.cassandra_conn_id = cassandra_conn_id self.table = table diff --git a/airflow/providers/apache/druid/operators/druid.py b/airflow/providers/apache/druid/operators/druid.py index bf8ee6ae6b091..f046ff1552a93 100644 --- a/airflow/providers/apache/druid/operators/druid.py +++ b/airflow/providers/apache/druid/operators/druid.py @@ -38,7 +38,7 @@ class DruidOperator(BaseOperator): template_ext = ('.json',) @apply_defaults - def __init__(self, json_index_file: str, + def __init__(self, *, json_index_file: str, druid_ingest_conn_id: str = 'druid_ingest_default', max_ingestion_time: Optional[int] = None, **kwargs: Any) -> None: diff --git a/airflow/providers/apache/druid/operators/druid_check.py b/airflow/providers/apache/druid/operators/druid_check.py index 380910cb35930..2f6114dfec0f7 100644 --- a/airflow/providers/apache/druid/operators/druid_check.py +++ b/airflow/providers/apache/druid/operators/druid_check.py @@ -59,6 +59,7 @@ class DruidCheckOperator(CheckOperator): @apply_defaults def __init__( self, + *, sql: str, druid_broker_conn_id: str = 'druid_broker_default', **kwargs: Any diff --git a/airflow/providers/apache/druid/transfers/hive_to_druid.py b/airflow/providers/apache/druid/transfers/hive_to_druid.py index 4f712bbadcb2b..595db0f5106bd 100644 --- a/airflow/providers/apache/druid/transfers/hive_to_druid.py +++ b/airflow/providers/apache/druid/transfers/hive_to_druid.py @@ -84,7 +84,7 @@ class HiveToDruidOperator(BaseOperator): @apply_defaults def __init__( # pylint: disable=too-many-arguments - self, + self, *, sql: str, druid_datasource: str, ts_dim: str, diff --git a/airflow/providers/apache/hdfs/sensors/hdfs.py b/airflow/providers/apache/hdfs/sensors/hdfs.py index 2307c111ea695..757f350787104 100644 --- a/airflow/providers/apache/hdfs/sensors/hdfs.py +++ b/airflow/providers/apache/hdfs/sensors/hdfs.py @@ -37,6 +37,7 @@ class HdfsSensor(BaseSensorOperator): @apply_defaults def __init__(self, + *, filepath: str, hdfs_conn_id: str = 'hdfs_default', ignored_ext: Optional[List[str]] = None, diff --git a/airflow/providers/apache/hdfs/sensors/web_hdfs.py b/airflow/providers/apache/hdfs/sensors/web_hdfs.py index f380d8deb043b..8d21b3e84dfd1 100644 --- a/airflow/providers/apache/hdfs/sensors/web_hdfs.py +++ b/airflow/providers/apache/hdfs/sensors/web_hdfs.py @@ -29,6 +29,7 @@ class WebHdfsSensor(BaseSensorOperator): @apply_defaults def __init__(self, + *, filepath: str, webhdfs_conn_id: str = 'webhdfs_default', **kwargs: Any) -> None: diff --git a/airflow/providers/apache/hive/operators/hive.py b/airflow/providers/apache/hive/operators/hive.py index 7ab55fe16a799..48d75742de5ed 100644 --- a/airflow/providers/apache/hive/operators/hive.py +++ b/airflow/providers/apache/hive/operators/hive.py @@ -70,7 +70,7 @@ class HiveOperator(BaseOperator): # pylint: disable=too-many-arguments @apply_defaults def __init__( - self, + self, *, hql: str, hive_cli_conn_id: str = 'hive_cli_default', schema: str = 'default', diff --git a/airflow/providers/apache/hive/operators/hive_stats.py b/airflow/providers/apache/hive/operators/hive_stats.py index b9b47ac37774d..6fc689e82fc52 100644 --- a/airflow/providers/apache/hive/operators/hive_stats.py +++ b/airflow/providers/apache/hive/operators/hive_stats.py @@ -63,7 +63,7 @@ class HiveStatsCollectionOperator(BaseOperator): ui_color = '#aff7a6' @apply_defaults - def __init__(self, + def __init__(self, *, table: str, partition: Any, extra_exprs: Optional[Dict[str, Any]] = None, diff --git a/airflow/providers/apache/hive/sensors/hive_partition.py b/airflow/providers/apache/hive/sensors/hive_partition.py index 19f7918f5ef53..8e1b8279da048 100644 --- a/airflow/providers/apache/hive/sensors/hive_partition.py +++ b/airflow/providers/apache/hive/sensors/hive_partition.py @@ -46,7 +46,7 @@ class HivePartitionSensor(BaseSensorOperator): ui_color = '#C5CAE9' @apply_defaults - def __init__(self, + def __init__(self, *, table: str, partition: Optional[str] = "ds='{{ ds }}'", metastore_conn_id: str = 'metastore_default', diff --git a/airflow/providers/apache/hive/sensors/metastore_partition.py b/airflow/providers/apache/hive/sensors/metastore_partition.py index 5dbdcd398ae17..1e54440f3ada7 100644 --- a/airflow/providers/apache/hive/sensors/metastore_partition.py +++ b/airflow/providers/apache/hive/sensors/metastore_partition.py @@ -45,7 +45,7 @@ class MetastorePartitionSensor(SqlSensor): ui_color = '#8da7be' @apply_defaults - def __init__(self, + def __init__(self, *, table: str, partition_name: str, schema: str = "default", diff --git a/airflow/providers/apache/hive/sensors/named_hive_partition.py b/airflow/providers/apache/hive/sensors/named_hive_partition.py index 43cc1130fcb3d..f69e2b2d4ea51 100644 --- a/airflow/providers/apache/hive/sensors/named_hive_partition.py +++ b/airflow/providers/apache/hive/sensors/named_hive_partition.py @@ -42,7 +42,7 @@ class NamedHivePartitionSensor(BaseSensorOperator): ui_color = '#8d99ae' @apply_defaults - def __init__(self, + def __init__(self, *, partition_names: List[str], metastore_conn_id: str = 'metastore_default', poke_interval: int = 60 * 3, diff --git a/airflow/providers/apache/hive/transfers/hive_to_mysql.py b/airflow/providers/apache/hive/transfers/hive_to_mysql.py index c937ceda01356..724c7919f8e80 100644 --- a/airflow/providers/apache/hive/transfers/hive_to_mysql.py +++ b/airflow/providers/apache/hive/transfers/hive_to_mysql.py @@ -67,7 +67,7 @@ class HiveToMySqlOperator(BaseOperator): ui_color = '#a0e08c' @apply_defaults - def __init__(self, + def __init__(self, *, sql: str, mysql_table: str, hiveserver2_conn_id: str = 'hiveserver2_default', diff --git a/airflow/providers/apache/hive/transfers/hive_to_samba.py b/airflow/providers/apache/hive/transfers/hive_to_samba.py index 742df32981823..5f08b83d09487 100644 --- a/airflow/providers/apache/hive/transfers/hive_to_samba.py +++ b/airflow/providers/apache/hive/transfers/hive_to_samba.py @@ -48,7 +48,7 @@ class HiveToSambaOperator(BaseOperator): template_ext = ('.hql', '.sql',) @apply_defaults - def __init__(self, + def __init__(self, *, hql: str, destination_filepath: str, samba_conn_id: str = 'samba_default', diff --git a/airflow/providers/apache/hive/transfers/mssql_to_hive.py b/airflow/providers/apache/hive/transfers/mssql_to_hive.py index 77e63ebd419d3..01a932724f9cb 100644 --- a/airflow/providers/apache/hive/transfers/mssql_to_hive.py +++ b/airflow/providers/apache/hive/transfers/mssql_to_hive.py @@ -76,7 +76,7 @@ class MsSqlToHiveOperator(BaseOperator): ui_color = '#a0e08c' @apply_defaults - def __init__(self, + def __init__(self, *, sql: str, hive_table: str, create: bool = True, diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py index b3d070f27e474..99650ec59f024 100644 --- a/airflow/providers/apache/hive/transfers/mysql_to_hive.py +++ b/airflow/providers/apache/hive/transfers/mysql_to_hive.py @@ -86,6 +86,7 @@ class MySqlToHiveOperator(BaseOperator): @apply_defaults def __init__( # pylint: disable=too-many-arguments self, + *, sql: str, hive_table: str, create: bool = True, diff --git a/airflow/providers/apache/hive/transfers/s3_to_hive.py b/airflow/providers/apache/hive/transfers/s3_to_hive.py index 734d3335f03bf..6c730a0a782bd 100644 --- a/airflow/providers/apache/hive/transfers/s3_to_hive.py +++ b/airflow/providers/apache/hive/transfers/s3_to_hive.py @@ -108,6 +108,7 @@ class S3ToHiveOperator(BaseOperator): # pylint: disable=too-many-instance-attri @apply_defaults def __init__( # pylint: disable=too-many-arguments self, + *, s3_key: str, field_dict: Dict, hive_table: str, diff --git a/airflow/providers/apache/hive/transfers/vertica_to_hive.py b/airflow/providers/apache/hive/transfers/vertica_to_hive.py index 1b9a4d63328e9..02a4f80ab2c66 100644 --- a/airflow/providers/apache/hive/transfers/vertica_to_hive.py +++ b/airflow/providers/apache/hive/transfers/vertica_to_hive.py @@ -74,6 +74,7 @@ class VerticaToHiveOperator(BaseOperator): @apply_defaults def __init__( self, + *, sql, hive_table, create=True, diff --git a/airflow/providers/apache/kylin/operators/kylin_cube.py b/airflow/providers/apache/kylin/operators/kylin_cube.py index 7b8e907c78937..cf53b63ce4211 100644 --- a/airflow/providers/apache/kylin/operators/kylin_cube.py +++ b/airflow/providers/apache/kylin/operators/kylin_cube.py @@ -96,7 +96,7 @@ class KylinCubeOperator(BaseOperator): # pylint: disable=too-many-arguments,inconsistent-return-statements @apply_defaults - def __init__(self, + def __init__(self, *, kylin_conn_id: Optional[str] = 'kylin_default', project: Optional[str] = None, cube: Optional[str] = None, diff --git a/airflow/providers/apache/livy/operators/livy.py b/airflow/providers/apache/livy/operators/livy.py index 44cc29d8ee4a8..16be339f3f007 100644 --- a/airflow/providers/apache/livy/operators/livy.py +++ b/airflow/providers/apache/livy/operators/livy.py @@ -74,7 +74,7 @@ class LivyOperator(BaseOperator): @apply_defaults def __init__( - self, + self, *, file: str, class_name: Optional[str] = None, args: Optional[Sequence[Union[str, int, float]]] = None, diff --git a/airflow/providers/apache/livy/sensors/livy.py b/airflow/providers/apache/livy/sensors/livy.py index bb54a9aa6e43e..b9d0bc429de0b 100644 --- a/airflow/providers/apache/livy/sensors/livy.py +++ b/airflow/providers/apache/livy/sensors/livy.py @@ -39,7 +39,7 @@ class LivySensor(BaseSensorOperator): @apply_defaults def __init__( - self, + self, *, batch_id: Union[int, str], livy_conn_id: str = 'livy_default', **kwargs: Any diff --git a/airflow/providers/apache/pig/operators/pig.py b/airflow/providers/apache/pig/operators/pig.py index f80754e4ef305..3f3c57897a18b 100644 --- a/airflow/providers/apache/pig/operators/pig.py +++ b/airflow/providers/apache/pig/operators/pig.py @@ -47,7 +47,7 @@ class PigOperator(BaseOperator): @apply_defaults def __init__( - self, + self, *, pig: str, pig_cli_conn_id: str = 'pig_cli_default', pigparams_jinja_translate: bool = False, diff --git a/airflow/providers/apache/spark/operators/spark_jdbc.py b/airflow/providers/apache/spark/operators/spark_jdbc.py index c7f8fde11aa64..8c1c7be494559 100644 --- a/airflow/providers/apache/spark/operators/spark_jdbc.py +++ b/airflow/providers/apache/spark/operators/spark_jdbc.py @@ -120,7 +120,7 @@ class SparkJDBCOperator(SparkSubmitOperator): # pylint: disable=too-many-arguments,too-many-locals @apply_defaults - def __init__(self, + def __init__(self, *, spark_app_name: str = 'airflow-spark-jdbc', spark_conn_id: str = 'spark-default', spark_conf: Optional[Dict[str, Any]] = None, diff --git a/airflow/providers/apache/spark/operators/spark_sql.py b/airflow/providers/apache/spark/operators/spark_sql.py index 796228a632c6e..59a1001a582f1 100644 --- a/airflow/providers/apache/spark/operators/spark_sql.py +++ b/airflow/providers/apache/spark/operators/spark_sql.py @@ -64,7 +64,7 @@ class SparkSqlOperator(BaseOperator): # pylint: disable=too-many-arguments @apply_defaults - def __init__(self, + def __init__(self, *, sql: str, conf: Optional[str] = None, conn_id: str = 'spark_sql_default', diff --git a/airflow/providers/apache/spark/operators/spark_submit.py b/airflow/providers/apache/spark/operators/spark_submit.py index 87208d9701db6..24d684ae140ca 100644 --- a/airflow/providers/apache/spark/operators/spark_submit.py +++ b/airflow/providers/apache/spark/operators/spark_submit.py @@ -102,7 +102,7 @@ class SparkSubmitOperator(BaseOperator): # pylint: disable=too-many-arguments,too-many-locals @apply_defaults - def __init__(self, + def __init__(self, *, application: str = '', conf: Optional[Dict[str, Any]] = None, conn_id: str = 'spark_default', diff --git a/airflow/providers/apache/sqoop/operators/sqoop.py b/airflow/providers/apache/sqoop/operators/sqoop.py index 1ae9841428701..3400360fc5d40 100644 --- a/airflow/providers/apache/sqoop/operators/sqoop.py +++ b/airflow/providers/apache/sqoop/operators/sqoop.py @@ -95,7 +95,7 @@ class SqoopOperator(BaseOperator): # pylint: disable=too-many-arguments,too-many-locals @apply_defaults - def __init__(self, + def __init__(self, *, conn_id: str = 'sqoop_default', cmd_type: str = 'import', table: Optional[str] = None,