diff --git a/airflow/airflow.go b/airflow/airflow.go index a557e449c..3a7499195 100644 --- a/airflow/airflow.go +++ b/airflow/airflow.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/astronomerio/astro-cli/airflow/include" "github.com/astronomerio/astro-cli/utils" ) @@ -63,10 +64,11 @@ func Init(path string) error { // Map of files to create files := map[string]string{ - ".dockerignore": dockerignore, - "Dockerfile": dockerfile, - "packages.txt": "", - "requirements.txt": "", + ".dockerignore": include.Dockerignore, + "Dockerfile": include.Dockerfile, + "packages.txt": "", + "requirements.txt": "", + "dags/example-dag.py": include.Exampledag, } // Initailize directories diff --git a/airflow/docker.go b/airflow/docker.go index d8a4997cb..158e86e0a 100644 --- a/airflow/docker.go +++ b/airflow/docker.go @@ -11,6 +11,7 @@ import ( "strings" "text/tabwriter" + "github.com/astronomerio/astro-cli/airflow/include" "github.com/astronomerio/astro-cli/config" docker "github.com/astronomerio/astro-cli/docker" dockercompose "github.com/docker/libcompose/docker" @@ -60,7 +61,7 @@ func imageBuild(path, imageName string) { // generateConfig generates the docker-compose config func generateConfig(projectName, airflowHome string) string { - tmpl, err := template.New("yml").Parse(composeyml) + tmpl, err := template.New("yml").Parse(include.Composeyml) if err != nil { fmt.Println(err) os.Exit(1) diff --git a/airflow/files.go b/airflow/files.go deleted file mode 100644 index c33aa4f37..000000000 --- a/airflow/files.go +++ /dev/null @@ -1,91 +0,0 @@ -package airflow - -import "strings" - -// dockerfile is the Dockerfile template -var dockerfile = strings.TrimSpace(` -FROM astronomerinc/ap-airflow:latest-onbuild -`) - -// dockerignore is the .dockerignore template -var dockerignore = strings.TrimSpace(` -.astro -.git -`) - -// composeyml is the docker-compose template -var composeyml = strings.TrimSpace(` -version: '2' - -networks: - airflow: - driver: bridge - -volumes: - postgres_data: {} - airflow_logs: {} - -services: - postgres: - image: postgres:10.1-alpine - restart: unless-stopped - networks: - - airflow - labels: - io.astronomer.docker: "true" - io.astronomer.docker.cli: "true" - ports: - - {{ .PostgresPort }}:{{ .PostgresPort }} - volumes: - - postgres_data:/var/lib/postgresql/data - environment: - POSTGRES_USER: {{ .PostgresUser }} - POSTGRES_PASSWORD: {{ .PostgresPassword }} - - scheduler: - image: {{ .AirflowImage }} - command: ["airflow", "scheduler"] - restart: unless-stopped - networks: - - airflow - user: {{ .AirflowUser }} - labels: - io.astronomer.docker: "true" - io.astronomer.docker.cli: "true" - io.astronomer.docker.component: "airflow-scheduler" - depends_on: - - postgres - environment: - AIRFLOW__CORE__EXECUTOR: LocalExecutor - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:{{ .PostgresPort }} - volumes: - - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro - - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro - - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro - - airflow_logs:/usr/local/airflow/logs - - webserver: - image: {{ .AirflowImage }} - command: ["airflow", "webserver"] - restart: unless-stopped - networks: - - airflow - user: {{ .AirflowUser }} - labels: - io.astronomer.docker: "true" - io.astronomer.docker.cli: "true" - io.astronomer.docker.component: "airflow-webserver" - depends_on: - - scheduler - - postgres - environment: - AIRFLOW__CORE__EXECUTOR: LocalExecutor - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:{{ .PostgresPort }} - ports: - - {{ .AirflowWebserverPort }}:{{ .AirflowWebserverPort }} - volumes: - - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro - - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro - - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro - - airflow_logs:/usr/local/airflow/logs -`) diff --git a/airflow/include/composeyml.go b/airflow/include/composeyml.go new file mode 100644 index 000000000..93a4ddf11 --- /dev/null +++ b/airflow/include/composeyml.go @@ -0,0 +1,80 @@ +package include + +import "strings" + +// Composeyml is the docker-compose template +var Composeyml = strings.TrimSpace(` +version: '2' + +networks: + airflow: + driver: bridge + +volumes: + postgres_data: {} + airflow_logs: {} + +services: + postgres: + image: postgres:10.1-alpine + restart: unless-stopped + networks: + - airflow + labels: + io.astronomer.docker: "true" + io.astronomer.docker.cli: "true" + ports: + - {{ .PostgresPort }}:{{ .PostgresPort }} + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: {{ .PostgresUser }} + POSTGRES_PASSWORD: {{ .PostgresPassword }} + + scheduler: + image: {{ .AirflowImage }} + command: ["airflow", "scheduler"] + restart: unless-stopped + networks: + - airflow + user: {{ .AirflowUser }} + labels: + io.astronomer.docker: "true" + io.astronomer.docker.cli: "true" + io.astronomer.docker.component: "airflow-scheduler" + depends_on: + - postgres + environment: + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:{{ .PostgresPort }} + volumes: + - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro + - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro + - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro + - airflow_logs:/usr/local/airflow/logs + + webserver: + image: {{ .AirflowImage }} + command: ["airflow", "webserver"] + restart: unless-stopped + networks: + - airflow + user: {{ .AirflowUser }} + labels: + io.astronomer.docker: "true" + io.astronomer.docker.cli: "true" + io.astronomer.docker.component: "airflow-webserver" + depends_on: + - scheduler + - postgres + environment: + AIRFLOW__CORE__EXECUTOR: LocalExecutor + AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:{{ .PostgresPort }} + ports: + - {{ .AirflowWebserverPort }}:{{ .AirflowWebserverPort }} + volumes: + - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro + - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro + - {{ .AirflowHome }}/include:/usr/local/airflow/include:ro + - airflow_logs:/usr/local/airflow/logs +`) diff --git a/airflow/include/dockerfile.go b/airflow/include/dockerfile.go new file mode 100644 index 000000000..03b10e2d1 --- /dev/null +++ b/airflow/include/dockerfile.go @@ -0,0 +1,8 @@ +package include + +import "strings" + +// Dockerfile is the Dockerfile template +var Dockerfile = strings.TrimSpace(` +FROM astronomerinc/ap-airflow:latest-onbuild +`) diff --git a/airflow/include/dockerignore.go b/airflow/include/dockerignore.go new file mode 100644 index 000000000..0a3d1317a --- /dev/null +++ b/airflow/include/dockerignore.go @@ -0,0 +1,9 @@ +package include + +import "strings" + +// Dockerignore is the .dockerignore template +var Dockerignore = strings.TrimSpace(` +.astro +.git +`) diff --git a/airflow/include/exampledag.go b/airflow/include/exampledag.go new file mode 100644 index 000000000..16f4df35e --- /dev/null +++ b/airflow/include/exampledag.go @@ -0,0 +1,97 @@ +package include + +import "strings" + +// Exampledag created with astro airflow init +var Exampledag = strings.TrimSpace(` +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from datetime import datetime, timedelta + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'start_date': datetime(2018, 1, 1), + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +} + +dag = DAG('example_dag', + schedule_interval=timedelta(minutes=5), + default_args=default_args) + +t1 = BashOperator( + task_id='print_date1', + bash_command='sleep 2m', + dag=dag) + +t2 = BashOperator( + task_id='print_date2', + bash_command='sleep 2m', + dag=dag) + +t3 = BashOperator( + task_id='print_date3', + bash_command='sleep 2m', + dag=dag) + +t4 = BashOperator( + task_id='print_date4', + bash_command='sleep 2m', + dag=dag) + +t5 = BashOperator( + task_id='print_date5', + bash_command='sleep 2m', + dag=dag) + +t6 = BashOperator( + task_id='print_date6', + bash_command='sleep 2m', + dag=dag) + +t7 = BashOperator( + task_id='print_date7', + bash_command='sleep 2m', + dag=dag) + +t8 = BashOperator( + task_id='print_date8', + bash_command='sleep 2m', + dag=dag) + +t2.set_upstream(t1) +t3.set_upstream(t2) +t4.set_upstream(t3) +t5.set_upstream(t3) +t6.set_upstream(t3) +t7.set_upstream(t3) +t8.set_upstream(t3) + +t9 = BashOperator( + task_id='print_date9', + bash_command='sleep 2m', + dag=dag) + +t10 = BashOperator( + task_id='print_date10', + bash_command='sleep 2m', + dag=dag) + +t11 = BashOperator( + task_id='print_date11', + bash_command='sleep 2m', + dag=dag) + +t12 = BashOperator( + task_id='print_date12', + bash_command='sleep 2m', + dag=dag) + +t9.set_upstream(t8) +t10.set_upstream(t8) +t11.set_upstream(t8) +t12.set_upstream(t8) +`)