From 928ba836514030068dc28f1034c67f1e4ece50a2 Mon Sep 17 00:00:00 2001
From: Justin Bandoro <79104794+jbandoro@users.noreply.github.com>
Date: Fri, 26 Jan 2024 15:02:33 -0800
Subject: [PATCH] Use Airflow constraint file for test environment setup (#812)

This PR installs apache-airflow in the hatch `pre-install-commands`
section so Airflow related dependency conflicts do not need to be
managed in the override section that has been removed.

Installing from the Airflow constraint file for versions <=2.6 fails
because PyYAML is pinned in the constraint file for those versions to
6.0.0 and [a workaround ](https://github.com/yaml/pyyaml/issues/736) is
required becaused older versions of PyYAML can no longer be installed
from unmodified source or sdist with the release of Cython3.

Closes: #811
---
 .github/workflows/test.yml |  4 ++--
 pyproject.toml             | 27 ++++++++-------------------
 2 files changed, 10 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index af71efc1c..a5a66eedf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -41,7 +41,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7"]
+        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -80,7 +80,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
-        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7"]
+        airflow-version: ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
     services:
       postgres:
         image: postgres
diff --git a/pyproject.toml b/pyproject.toml
index 67865ac84..cba71f41b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -155,30 +155,19 @@ dependencies = [
     "apache-airflow-providers-cncf-kubernetes>=5.1.1",
     "apache-airflow-providers-docker>=3.5.0",
 ]
-post-install-commands = [
-  "pip uninstall -y apache-airflow-providers-common-io"
+# Airflow install with constraint file, Airflow versions < 2.7 require a workaround for PyYAML
+pre-install-commands = ["""
+    if [[ "2.3 2.4 2.5 2.6" =~ "{matrix:airflow}" ]]; then
+        echo "Cython < 3" >> /tmp/constraint.txt
+        pip wheel "PyYAML==6.0.0" -c /tmp/constraint.txt
+    fi
+    pip install 'apache-airflow=={matrix:airflow}' --constraint 'https://raw.githubusercontent.com/apache/airflow/constraints-{matrix:airflow}.0/constraints-{matrix:python}.txt'
+    """
 ]
-
 [[tool.hatch.envs.tests.matrix]]
 python = ["3.8", "3.9", "3.10"]
 airflow = ["2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
 
-[tool.hatch.envs.tests.overrides]
-matrix.airflow.dependencies = [
-    { value = "apache-airflow==2.3", if = ["2.3"] },
-    { value = "pendulum<3.0.0", if = ["2.3"] },
-    { value = "apache-airflow==2.4", if = ["2.4"] },
-    { value = "pendulum<3.0.0", if = ["2.4"] },
-    { value = "apache-airflow==2.5", if = ["2.5"] },
-    { value = "pendulum<3.0.0", if = ["2.5"] },
-    { value = "apache-airflow==2.6", if = ["2.6"] },
-    { value = "pendulum<3.0.0", if = ["2.6"] },
-    { value = "pydantic>=1.10.0,<2.0.0", if = ["2.6"]},
-    { value = "apache-airflow==2.7", if = ["2.7"] },
-    { value = "pendulum<3.0.0", if = ["2.7"] },
-    { value = "apache-airflow==2.8", if = ["2.8"] },
-    { value = "apache-airflow-providers-common-io", if = ["2.8"] },
-]
 
 [tool.hatch.envs.tests.scripts]
 freeze = "pip freeze"