Skip to content

Commit fb4717a

Browse files
xtutranxtutran
authored
Update README.md
1 parent 0c93526 commit fb4717a

File tree

1 file changed

+113
-108
lines changed

1 file changed

+113
-108
lines changed

‎airflow/README.md‎

Lines changed: 113 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
## Install & config apache airflow mysql
22
### 1. Install mysql <a href='https://dev.mysql.com/downloads/mysql/'>here</a>
33
- Homebrew in Macbook
4-
```bash
5-
brew install mysql
6-
```
7-
- Mannually install without Admin right, refer [here](../mysql)
4+
```bash
5+
brew install mysql
6+
```
7+
- Mannually install without Admin right, refer [here](../mysql)
88

99
### 2. Install mysql-python
1010
```bash
@@ -13,119 +13,124 @@ pip install mysqlclient
1313
```
1414

1515
- Create airflow db
16-
```bash
17-
chmod +x scripts/create_db.sh
18-
. scripts/create_db.sh
19-
```
16+
```bash
17+
chmod +x scripts/create_db.sh
18+
. scripts/create_db.sh
19+
```
2020

2121
- Test mysql-python connection
22-
```python
23-
from sqlalchemy import create_engine
24-
25-
eng = create_engine('mysql://root:@localhost:3306/mysql')
26-
with eng.connect() as con:
27-
28-
rs = con.execute('SELECT 6')
29-
30-
data = rs.fetchone()[0]
31-
32-
print "Data: %s" % data
33-
```
34-
35-
### 3. Install apache airflow, official site <a href='http://airflow.incubator.apache.org/installation.html'>here</a>
36-
37-
- Install airflow[celery,mysql]
38-
```bash
39-
pip install "pip install airflow[celery,mysql]"
40-
```
41-
42-
- Config airflow
43-
###### a. Initialize an airflow home
44-
```bash
45-
export AIRFLOW_HOME=~/airflow
46-
export PATH=/Users/txuantu/anaconda2/lib/python2.7/site-packages/airflow/bin:$PATH
47-
48-
airflow version
49-
```
50-
51-
###### b. Go to airflow home and config mysql connection
52-
```bash
53-
cd $AIRFLOW_HOME
54-
vi airflow.cfg
55-
```
56-
57-
- Default mysql
58-
```bash
59-
# The executor class that airflow should use. Choices include
60-
# SequentialExecutor, LocalExecutor, CeleryExecutor
61-
executor = CeleryExecutor
62-
63-
64-
# The SqlAlchemy connection string to the metadata database.
65-
# SqlAlchemy supports many different database engine, more information
66-
# their website
67-
sql_alchemy_conn = mysql://airflow:airflow@localhost/airflow
68-
69-
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
70-
# a sqlalchemy database. Refer to the Celery documentation for more
71-
# information.
72-
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
22+
```python
23+
from sqlalchemy import create_engine
7324

74-
# Another key Celery setting
75-
celery_result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
76-
```
25+
eng = create_engine('mysql://root:@localhost:3306/mysql')
26+
with eng.connect() as con:
7727

78-
- Mannual mysql
79-
```bash
80-
# The executor class that airflow should use. Choices include
81-
# SequentialExecutor, LocalExecutor, CeleryExecutor
82-
executor = CeleryExecutor
28+
rs = con.execute('SELECT 6')
8329

30+
data = rs.fetchone()[0]
8431

85-
# The SqlAlchemy connection string to the metadata database.
86-
# SqlAlchemy supports many different database engine, more information
87-
# their website
88-
sql_alchemy_conn = mysql+mysqldb://airflow:airflow@localhost/airflow?unix_socket=/path/to/thesock
32+
print "Data: %s" % data
33+
```
8934

90-
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
91-
# a sqlalchemy database. Refer to the Celery documentation for more
92-
# information.
93-
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow?unix_socket=/path/to/thesock
35+
### 3. Install apache airflow, official site <a href='http://airflow.incubator.apache.org/installation.html'>here</a>
9436

95-
# Another key Celery setting
96-
celery_result_backend = db+mysql://airflow:airflow@localhost:3306/airflow?unix_socket=/path/to/thesock
97-
```
37+
- Install airflow[celery,mysql]
38+
```bash
39+
pip install "pip install airflow[celery,mysql]"
40+
```
9841

99-
- Enable SSL
100-
```bash
101-
[webserver]
102-
# The base url of your website as airflow cannot guess what domain or
103-
# cname you are using. This is used in automated emails that
104-
# airflow sends to point links to the right web server
105-
base_url = http://localhost:443
106-
107-
# The ip specified when starting the web server
108-
web_server_host = 0.0.0.0
109-
110-
# The port on which to run the web server
111-
web_server_port = 443
112-
113-
# Paths to the SSL certificate and key for the web server. When both are
114-
# provided SSL will be enabled. This does not change the web server port.
115-
web_server_ssl_cert = ~/airflow/mycert.pem
116-
web_server_ssl_key = ~/airflow/mykey.key
117-
```
42+
- Config airflow
43+
###### a. Initialize an airflow home
44+
```bash
45+
export AIRFLOW_HOME=~/airflow
46+
export PATH=/Users/txuantu/anaconda2/lib/python2.7/site-packages/airflow/bin:$PATH
11847

119-
###### c. Initialise db connection
120-
```bash
121-
airflow initdb
122-
```
48+
airflow version
49+
```
12350

124-
###### d. Launch airflow webserver
125-
```bash
126-
airflow webserver
127-
```
51+
###### b. Go to airflow home and config mysql connection
52+
```bash
53+
cd $AIRFLOW_HOME
54+
vi airflow.cfg
55+
```
12856

129-
```bash
130-
ariflow worker
131-
```
57+
- Default mysql
58+
```bash
59+
# The executor class that airflow should use. Choices include
60+
# SequentialExecutor, LocalExecutor, CeleryExecutor
61+
executor = CeleryExecutor
62+
63+
64+
# The SqlAlchemy connection string to the metadata database.
65+
# SqlAlchemy supports many different database engine, more information
66+
# their website
67+
sql_alchemy_conn = mysql://airflow:airflow@localhost/airflow
68+
69+
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
70+
# a sqlalchemy database. Refer to the Celery documentation for more
71+
# information.
72+
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
73+
74+
# Another key Celery setting
75+
celery_result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
76+
```
77+
78+
- Mannual mysql
79+
```bash
80+
# The executor class that airflow should use. Choices include
81+
# SequentialExecutor, LocalExecutor, CeleryExecutor
82+
executor = CeleryExecutor
83+
84+
85+
# The SqlAlchemy connection string to the metadata database.
86+
# SqlAlchemy supports many different database engine, more information
87+
# their website
88+
sql_alchemy_conn = mysql+mysqldb://airflow:airflow@localhost/airflow?unix_socket=/path/to/thesock
89+
90+
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
91+
# a sqlalchemy database. Refer to the Celery documentation for more
92+
# information.
93+
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow?unix_socket=/path/to/thesock
94+
95+
# Another key Celery setting
96+
celery_result_backend = db+mysql://airflow:airflow@localhost:3306/airflow?unix_socket=/path/to/thesock
97+
```
98+
99+
- Enable SSL
100+
```bash
101+
# generate your own certificate
102+
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mykey.key -out mycert.pem
103+
```
104+
105+
```bash
106+
[webserver]
107+
# The base url of your website as airflow cannot guess what domain or
108+
# cname you are using. This is used in automated emails that
109+
# airflow sends to point links to the right web server
110+
base_url = http://localhost:443
111+
112+
# The ip specified when starting the web server
113+
web_server_host = 0.0.0.0
114+
115+
# The port on which to run the web server
116+
web_server_port = 443
117+
118+
# Paths to the SSL certificate and key for the web server. When both are
119+
# provided SSL will be enabled. This does not change the web server port.
120+
web_server_ssl_cert = ~/airflow/mycert.pem
121+
web_server_ssl_key = ~/airflow/mykey.key
122+
```
123+
124+
###### c. Initialise db connection
125+
```bash
126+
airflow initdb
127+
```
128+
129+
###### d. Launch airflow webserver
130+
```bash
131+
airflow webserver
132+
```
133+
134+
```bash
135+
ariflow worker
136+
```

0 commit comments

Comments
 (0)