134 lines (111 loc) · 6.24 KB
/
deploy.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: Build image for Production
on:
push:
branches:
- master
jobs:
deploy:
name: Deploy to EC2 on master branch push
runs-on: ubuntu-18.04
steps:
- name: Checkout Files
uses: actions/checkout@v2
- name: ssh to jump server
uses: appleboy/ssh-action@master
with:
host: ${{ secrets.HOST_DNS }}
username: ${{ secrets.USERNAME }}
key: ${{ secrets.EC2_SSH_KEY }}
script:
sudo yum install git -y &&
sudo amazon-linux-extras install epel -y &&
sudo yum install dpkg-devel dpkg-dev -y
- name: new ssh proxy
uses: appleboy/ssh-action@master
with:
proxy_key: ${{ secrets.EC2_SSH_KEY }}
proxy_host: ${{ secrets.HOST_DNS }}
proxy_username: ${{ secrets.USERNAME }}
host: ${{ secrets.TARGET_DNS }}
username: ${{ secrets.TARGET_USERNAME }}
key: ${{ secrets.EC2_SSH_KEY }}
script: |
FILE=/home/ubuntu/airflow_test/docker-compose.yaml
if ! sudo docker info > /dev/null 2>&1; then
sudo apt-get update -y
# https://docs.docker.com/engine/install/ubuntu/
sudo apt-get install \
ca-certificates \
curl \
gnupg \
lsb-release
# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
# echo \
# "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
# $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y
# sudo apt-get install docker-ce docker-ce-cli containerd.io -y
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y
docker --version
# https://docs.docker.com/compose/install/
# sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
# sudo chmod +x /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
docker-compose --version
sudo apt-get install postgresql-client -y
fi
if [ -f "$FILE" ]; then
sudo docker-compose -f /home/ubuntu/airflow_test/docker-compose.yaml down 2> /dev/null && sudo docker system prune -a -f
fi
whoami && sudo rm -rf airflow_test &&
pwd && sudo git clone https://github.com/psalmprax/airflow_test.git && ls -ltra &&
sudo docker-compose -f /home/ubuntu/airflow_test/docker-compose.yaml --compatibility up --build -d &&
sudo docker-compose -f /home/ubuntu/airflow_test/docker-compose.yaml down 2> /dev/null &&
sudo docker ps -a && sudo chmod -R 777 /home/ubuntu/airflow_test/logs/ &&
sudo chown -R ubuntu:ubuntu /home/ubuntu/airflow_test/logs/ &&
sudo chmod -R 777 /home/ubuntu/airflow_test/plugins/ &&
sudo chown -R ubuntu:ubuntu /home/ubuntu/airflow_test/plugins/ &&
sudo docker-compose -f /home/ubuntu/airflow_test/docker-compose.yaml --compatibility up --build -d &&
sleep 10s && ls -ltra /home/ubuntu/airflow_test &&
sudo sh /home/ubuntu/airflow_test/airflow_connections.sh
# sudo apt install tor -y
# cat << EOF > windscrb.exp
# #!/usr/bin/expect -f
# set timeout -1
# spawn windscribe login
#
# expect "Windscribe Username:"
# send "psalmprax\n"
#
# expect "Windscribe Password:"
# send "Single123.\n"
#
# expect eof
# EOF
# sudo chmod +x windscrb.exp
# ./windscrb.exp
# windscribe connect
# sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys FDC247B7 &&
# echo 'deb https://repo.windscribe.com/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/windscribe-repo.list &&
# sudo apt-get update -y && sudo apt-get install windscribe-cli -y &&
# sudo apt install expect -y &&
# sudo apt install iptables -y &&
# sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys FDC247B7 &&
# sudo dpkg -i /home/ubuntu/airflow_test/dags/windscribe_2.4.8_beta_amd64.deb &&
# gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys FDC247B7 &&
# echo 'deb https://repo.windscribe.com/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/windscribe-repo.list &&
# sudo apt-get update -y && sudo apt-get install windscribe-cli &&
# ls -ltra && sudo docker-compose -f /home/ubuntu/airflow_test/docker-compose.yaml down -v 2> /dev/null &&
# exit 0 && sudo docker system prune -a -f && ll &&
# && sudo docker-compose up --build -d
# sudo docker-compose down -v && sudo docker system prune -a -f &&
# && cd .. && rm -rf airflow_test 2> /dev/null
############# data warehouse troubleshoot #####################################
# https://aws.amazon.com/premiumsupport/knowledge-center/rds-connectivity-instance-subnet-vpc/
# https://aws.amazon.com/premiumsupport/knowledge-center/rds-connectivity-instance-subnet-vpc/