-
Notifications
You must be signed in to change notification settings - Fork 1
/
getting_started.sh.j2
executable file
·158 lines (138 loc) · 4.86 KB
/
getting_started.sh.j2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/usr/bin/env bash
set -euo pipefail
# This script contains all the code snippets from the guide, as well as some assert tests
# to test if the instructions in the guide work. The user *could* use it, but it is intended
# for testing only.
# The script will install the operators, create an airflow instance and briefly open a port
# forward and connect to the airflow instance to make sure it is up and running.
# No running processes are left behind (i.e. the port-forwarding is closed at the end)
if [ $# -eq 0 ]
then
echo "Installation method argument ('helm' or 'stackablectl') required."
exit 1
fi
echo "Adding bitnami Helm Chart repository and dependencies (Postgresql and Redis)"
# tag::helm-add-bitnami-repo[]
helm repo add bitnami https://charts.bitnami.com/bitnami
# end::helm-add-bitnami-repo[]
# tag::helm-add-bitnami-pgs[]
helm install --wait airflow-postgresql bitnami/postgresql --version {{ versions.postgresql }} \
--set auth.username=airflow \
--set auth.password=airflow \
--set auth.database=airflow
# end::helm-add-bitnami-pgs[]
# tag::helm-add-bitnami-redis[]
helm install --wait airflow-redis bitnami/redis \
--set auth.password=redis \
--version {{ versions.redis }} \
--set replica.replicaCount=1
# end::helm-add-bitnami-redis[]
case "$1" in
"helm")
echo "Adding '{{ helm.repo_name }}' Helm Chart repository"
# tag::helm-add-repo[]
helm repo add {{ helm.repo_name }} {{ helm.repo_url }}
# end::helm-add-repo[]
echo "Updating Helm repo"
helm repo update
echo "Installing Operators with Helm"
# tag::helm-install-operators[]
helm install --wait commons-operator {{ helm.repo_name }}/commons-operator --version {{ versions.commons }}
helm install --wait secret-operator {{ helm.repo_name }}/secret-operator --version {{ versions.secret }}
helm install --wait airflow-operator {{ helm.repo_name }}/airflow-operator --version {{ versions.airflow }}
# end::helm-install-operators[]
;;
"stackablectl")
echo "installing Operators with stackablectl"
# tag::stackablectl-install-operators[]
stackablectl operator install \
commons={{ versions.commons }} \
secret={{ versions.secret }} \
airflow={{ versions.airflow }}
# end::stackablectl-install-operators[]
;;
*)
echo "Need to give 'helm' or 'stackablectl' as an argument for which installation method to use!"
exit 1
;;
esac
echo "Creating credentials secret"
# tag::apply-airflow-credentials[]
kubectl apply -f airflow-credentials.yaml
# end::apply-airflow-credentials[]
echo "Creating Airflow cluster"
# tag::install-airflow[]
kubectl apply -f airflow.yaml
# end::install-airflow[]
sleep 5
echo "Awaiting Airflow rollout finish ..."
# tag::watch-airflow-rollout[]
kubectl rollout status --watch --timeout=5m statefulset/airflow-webserver-default
kubectl rollout status --watch --timeout=5m statefulset/airflow-worker-default
kubectl rollout status --watch --timeout=5m statefulset/airflow-scheduler-default
# end::watch-airflow-rollout[]
echo "Starting port-forwarding of port 8080"
# tag::port-forwarding[]
kubectl port-forward svc/airflow-webserver 8080 2>&1 >/dev/null &
# end::port-forwarding[]
PORT_FORWARD_PID=$!
trap "kill $PORT_FORWARD_PID" EXIT
sleep 5
echo "Checking if web interface is reachable ..."
return_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/login/)
if [ "$return_code" == 200 ]; then
echo "Webserver UI reachable!"
else
echo "Could not reach Webserver UI."
exit 1
fi
sleep 5
server_health() {
# tag::server-health[]
curl -s -XGET http://localhost:8080/api/v1/health
# end::server-health[]
}
echo "Checking webserver health ..."
health=$(server_health | jq -r '.scheduler.status')
if [ "$health" == "healthy" ]; then
echo "We have a healthy webserver!"
else
echo "Webserver does not have the expected status. Detected status: " "$health"
exit 1
fi
enable_dag() {
# tag::enable-dag[]
curl -s --user airflow:airflow -H 'Content-Type:application/json' \
-XPATCH http://localhost:8080/api/v1/dags/example_complex \
-d '{"is_paused": false}'
# end::enable-dag[]
}
echo "Triggering a DAG run. Enable DAG..."
enable_dag
run_dag() {
# tag::run-dag[]
curl -s --user airflow:airflow -H 'Content-Type:application/json' \
-XPOST http://localhost:8080/api/v1/dags/example_complex/dagRuns \
-d '{}' | jq -r '.dag_run_id'
# end::run-dag[]
}
dag_id=$(run_dag)
request_dag_status() {
# tag::check-dag[]
curl -s --user airflow:airflow -H 'Content-Type:application/json' \
-XGET http://localhost:8080/api/v1/dags/example_complex/dagRuns/"$dag_id" | jq -r '.state'
# end::check-dag[]
}
dag_state=$(request_dag_status)
while [[ "$dag_state" == "running" || "$dag_state" == "queued" ]]; do
echo "Awaiting DAG completion ..."
sleep 5
dag_state=$(request_dag_status)
done
echo "Checking DAG result ..."
if [ "$dag_state" == "success" ]; then
echo "DAG run successful for ID: " "$dag_id"
else
echo "The DAG was not successful. State: " "$dag_state"
exit 1
fi