diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c264952..fa48fad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: # This workflow contains a single job called "build" build: # The type of runner that the job will run on - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 # Steps represent a sequence of tasks that will be executed as part of the job steps: diff --git a/Dockerfile b/Dockerfile index 3d2baef..76d586c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,2 @@ -FROM apache/airflow:2.3.0-python3.10 +FROM apache/airflow:2.7.3-python3.10 RUN pip install --no-cache-dir td-client \ No newline at end of file diff --git a/README.md b/README.md index 6906a4c..9af24f8 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@

#### Recent Updates: +03-Dec-2023 +- Upgrade to airflow 2.7.3 +- Upgraded superset to add secret key +- Added superset database connection image +- Works on M1 Mac + 03-May-2022 - Added Dockerfile to extend airflow image - Adding additional Pypi package (td-client) diff --git a/SUPERSET.md b/SUPERSET.md index 4757a17..20a03c9 100644 --- a/SUPERSET.md +++ b/SUPERSET.md @@ -12,7 +12,7 @@ This chart provides the total tests done in Albany county per day and the new po Once the instructions to start Airflow is complete, follow the below steps: - Connect to the superset container. ``` -docker exec -it airflowdocker_superset_1 /bin/bash +docker exec -it airflow-docker-superset-1 /bin/bash ``` - Create user in superset ``` @@ -32,10 +32,13 @@ superset init - Add Database connection. Connection string would be as below: ``` -postgresql://airflow:XXXXXXXXXX@postgres:5432/userdata +postgresql://airflow:XXXXXXXXXX@host.docker.internal:5433/userdata ``` +When you create a connection the host should be : host.docker.internal +![connect_details](https://github.com/anilkulkarni87/airflow-docker/assets/10644132/c9af3af1-2442-4fab-9946-7bdd15ea5ab5.png) ![image](https://user-images.githubusercontent.com/10644132/108793463-6d75ff00-7538-11eb-8b23-c0c9ffa86358.png) + ## TODO - Create a script for the above steps and automate user creation in superset diff --git a/docker-compose-githubworkflow.yaml b/docker-compose-githubworkflow.yaml index e0f7eb7..2590b66 100755 --- a/docker-compose-githubworkflow.yaml +++ b/docker-compose-githubworkflow.yaml @@ -39,7 +39,7 @@ version: '3' x-airflow-common: &airflow-common - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.3.0} + image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.7.3} environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor diff --git a/docker-compose.yaml b/docker-compose.yaml index b51cb7e..66b6e8e 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -53,7 +53,7 @@ x-airflow-common: #Example Dags will not be loaded AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__WEBSERVER__RBAC: 'true' - _AIRFLOW_DB_UPGRADE: 'true' + _AIRFLOW_DB_MIGRATE: 'true' _AIRFLOW_WWW_USER_CREATE: 'true' _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} @@ -107,15 +107,19 @@ services: superset: image: apache/superset:latest - # command: - # - superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin - # - superset db upgrade - # - superset init + platform: linux/amd64 + environment: + SUPERSET_SECRET_KEY: 123abc + # command: > + # bash -c " superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin + # && superset db upgrade + # && superset init" ports: - - 8090:8080 + - 8091:8088 minio: image: minio/minio:latest + platform: linux/amd64 container_name: minio command: minio server /data --console-address ":9001" environment: @@ -262,7 +266,7 @@ services: # yamllint enable rule:line-length environment: <<: *airflow-common-env - _AIRFLOW_DB_UPGRADE: 'true' + _AIRFLOW_DB_MIGRATE: 'true' _AIRFLOW_WWW_USER_CREATE: 'true' _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}