Update Airflow to 2.2.4 (#372)

* Update Airflow to 2.2.4

* Fix deprecation warnings

docker/airflow/Dockerfile

@@ -35,13 +35,14 @@ RUN useradd -m -d ${AIRFLOW_HOME} airflow && \
     chown airflow:airflow ${OUTPUT_DIR}
 USER airflow
 
+ARG AIRFLOW_VERSION=2.2.4
 WORKDIR  ${AIRFLOW_HOME}
 # Always add the prod req because the dev reqs depend on it for deduplication
 COPY ${REQUIREMENTS_FILE} requirements_prod.txt ${AIRFLOW_HOME}/
 COPY docker/airflow/wait_for_db.py /opt/airflow/
 
 # https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files
-ARG CONSTRAINTS_FILE="https://raw.githubusercontent.com/apache/airflow/constraints-2.2.3/constraints-3.9.txt"
+ARG CONSTRAINTS_FILE="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt"
 
 RUN pip install --user -r ${REQUIREMENTS_FILE} -c ${CONSTRAINTS_FILE}
 

openverse_catalog/dags/commoncrawl/commoncrawl_etl.py

@@ -3,15 +3,12 @@
 
 from airflow import DAG
 from airflow.operators.python import PythonOperator
-from airflow.providers.amazon.aws.operators.emr_create_job_flow import (
+from airflow.providers.amazon.aws.operators.emr import (
     EmrCreateJobFlowOperator,
-)
-from airflow.providers.amazon.aws.operators.emr_terminate_job_flow import (
     EmrTerminateJobFlowOperator,
 )
-from airflow.providers.amazon.aws.sensors.emr_job_flow import EmrJobFlowSensor
-from airflow.providers.amazon.aws.sensors.s3_key import S3KeySensor
-from airflow.providers.amazon.aws.sensors.s3_prefix import S3PrefixSensor
+from airflow.providers.amazon.aws.sensors.emr import EmrJobFlowSensor
+from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor, S3PrefixSensor
 from airflow.utils.trigger_rule import TriggerRule
 from commoncrawl.commoncrawl_utils import get_load_s3_task_id, load_file_to_s3
 

requirements_prod.txt

@@ -1,4 +1,4 @@
-apache-airflow[amazon,postgres,http]==2.2.3
+apache-airflow[amazon,postgres,http]==2.2.4
 lxml
 psycopg2-binary
 requests-file==1.5.1