Add Astronomer Starship Airflow plugin to enable migration of DAGs to…

… Astro (#654)
The-Academic-Observatory · Mar 18, 2024 · 9c10f85 · 9c10f85
1 parent c7e9331
commit 9c10f85
Show file tree

Hide file tree

Showing 9 changed files with 14 additions and 10 deletions.
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -26,8 +26,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e observatory-api[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
-          pip install -e observatory-platform[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -e observatory-api[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -e observatory-platform[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
 
       - name: Check licenses
         run: |

diff --git a/bin/generate-api-client.sh b/bin/generate-api-client.sh
@@ -11,7 +11,7 @@ if ! command -v openapi-generator-cli &>/dev/null; then
 fi
 
 if ! command -v observatory-api &>/dev/null; then
-  pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-3.10.txt
+  pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-3.10.txt
 fi
 
 # Directories

diff --git a/docs/api/api_client.md b/docs/api/api_client.md
@@ -19,7 +19,7 @@ To install the package from source:
 ```bash
 git clone https://github.com/The-Academic-Observatory/observatory-platform.git
 cd observatory-platform
-pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-3.10.txt
+pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-3.10.txt
 ```
 
 ## Getting Started

diff --git a/install.sh b/install.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 venv_observatory_platform="observatory_venv"
-airflow_version="2.6.3"
+airflow_version="2.7.3"
 python_version="3.10"
 
 function set_os_arch() {

diff --git a/observatory-api/requirements.txt b/observatory-api/requirements.txt
@@ -1,10 +1,10 @@
-gunicorn>=20.1.0,<21
+gunicorn>=21.2.0,<22
 Flask>=2,<3
 connexion[swagger-ui]>=2.7.0,<3
 pyyaml>=6,<7
 SQLAlchemy>=1.4,<2
 psycopg2-binary>=2.9.1,<3
-urllib3>=1.25.11,<2
+urllib3>=2.0.7,<3
 python-dateutil>=2.8.2,<3
 certifi
 pendulum>=2.1.2,<3

diff --git a/observatory-platform/observatory/platform/docker/docker-compose.observatory.yml.jinja2 b/observatory-platform/observatory/platform/docker/docker-compose.observatory.yml.jinja2
@@ -151,7 +151,7 @@ services:
   webserver:
     container_name: webserver
     hostname: webserver 
-    image: apache/airflow:{{ airflow_version }}-python{{ python_version }}
+    build: *build
     volumes: *volumes
     environment: *environment
     restart: always

diff --git a/observatory-platform/observatory/platform/docker/platform_runner.py b/observatory-platform/observatory/platform/docker/platform_runner.py
@@ -28,7 +28,7 @@
 HOST_UID = os.getuid()
 DEBUG = False
 PYTHON_VERSION = "3.10"
-AIRFLOW_VERSION = "2.6.3"
+AIRFLOW_VERSION = "2.7.3"
 
 
 class PlatformRunner(ComposeRunner):

diff --git a/observatory-platform/requirements.txt b/observatory-platform/requirements.txt
@@ -2,7 +2,8 @@
 observatory-api
 
 # Airflow
-apache-airflow[redis,celery,slack,http]==2.6.3
+apache-airflow[redis,celery,slack,http]==2.7.3
+astronomer-starship
 
 # Google Cloud
 google-crc32c>=1.1.0,<2

diff --git a/strategy.ini b/strategy.ini
@@ -68,3 +68,6 @@ precipy: >=0.2.2
 # Apache 2.0 license, but this is not classified properly, see https://github.com/p1c2u/openapi-spec-validator/issues/138
 openapi-spec-validator: >=0.3.1,<1
 
+# Propriety license, however this is required to migrate to Astronomer.io
+astronomer-starship: >=2,<3
+