Merge pull request #3 from anilkulkarni87/feature/Airflow-2.7.3

Airflow version updgrade and fix superset

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
   # This workflow contains a single job called "build"
   build:
     # The type of runner that the job will run on
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

Dockerfile

@@ -1,2 +1,2 @@
-FROM apache/airflow:2.3.0-python3.10
+FROM apache/airflow:2.7.3-python3.10
 RUN pip install --no-cache-dir td-client

README.md

@@ -18,6 +18,12 @@
 </p>
 
 #### Recent Updates:
+03-Dec-2023
+- Upgrade to airflow 2.7.3
+- Upgraded superset to add secret key
+- Added superset database connection image
+- Works on M1 Mac
+
 03-May-2022
 - Added Dockerfile to extend airflow image
 - Adding additional Pypi package (td-client)

SUPERSET.md

@@ -12,7 +12,7 @@ This chart provides the total tests done in Albany county per day and the new po
 Once the instructions to start Airflow is complete, follow the below steps:
 - Connect to the superset container. 
 ```
-docker exec -it airflowdocker_superset_1 /bin/bash
+docker exec -it airflow-docker-superset-1 /bin/bash
 ```
 - Create user in superset
 ```
@@ -32,10 +32,13 @@ superset init
 
 - Add Database connection. Connection string would be as below:
 ```
-postgresql://airflow:XXXXXXXXXX@postgres:5432/userdata
+postgresql://airflow:XXXXXXXXXX@host.docker.internal:5433/userdata
 ```
+When you create a connection the host should be : host.docker.internal
 
+![connect_details](https://github.com/anilkulkarni87/airflow-docker/assets/10644132/c9af3af1-2442-4fab-9946-7bdd15ea5ab5.png)
 ![image](https://user-images.githubusercontent.com/10644132/108793463-6d75ff00-7538-11eb-8b23-c0c9ffa86358.png)
 
+
 ## TODO
 - Create a script for the above steps and automate user creation in superset

docker-compose-githubworkflow.yaml

@@ -39,7 +39,7 @@
 version: '3'
 x-airflow-common:
   &airflow-common
-  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.3.0}
+  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.7.3}
   environment:
     &airflow-common-env
     AIRFLOW__CORE__EXECUTOR: CeleryExecutor

docker-compose.yaml

@@ -53,7 +53,7 @@ x-airflow-common:
     #Example Dags will not be loaded
     AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
     AIRFLOW__WEBSERVER__RBAC: 'true'
-    _AIRFLOW_DB_UPGRADE: 'true'
+    _AIRFLOW_DB_MIGRATE: 'true'
     _AIRFLOW_WWW_USER_CREATE: 'true'
     _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
     _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
@@ -107,15 +107,19 @@ services:
 
   superset:
     image: apache/superset:latest
-    # command:
-    #   - superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin
-    #   - superset db upgrade
-    #   - superset init
+    platform: linux/amd64
+    environment:
+      SUPERSET_SECRET_KEY: 123abc
+    # command: >
+    #   bash -c " superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin
+    #   && superset db upgrade
+    #   && superset init"
     ports:
-      - 8090:8080
+      - 8091:8088
 
   minio:
     image: minio/minio:latest
+    platform: linux/amd64
     container_name: minio
     command: minio server /data --console-address ":9001"
     environment:
@@ -262,7 +266,7 @@ services:
     # yamllint enable rule:line-length
     environment:
       <<: *airflow-common-env
-      _AIRFLOW_DB_UPGRADE: 'true'
+      _AIRFLOW_DB_MIGRATE: 'true'
       _AIRFLOW_WWW_USER_CREATE: 'true'
       _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
       _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}