-
Notifications
You must be signed in to change notification settings - Fork 3
/
tasks.yaml
155 lines (142 loc) · 6.79 KB
/
tasks.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# Tasks related to stillalive
#
# Browse repo: https://gerrit.wikimedia.org/g/labs/countervandalism/stillalive
tasks stillalive:
- quit-a-bot:
- try:
- 'irc #cvn-bots'
- say 'MyBotName quit'
- else:
# Connect to the server where the bot used to be pooled
- ssh cvn-appX.cvn.eqiad.wmflabs
- 'ps aux | grep MyBotName'
# Find the process id, and terminate it
- sudo kill 123
- deploy-a-change:
# After you've made a change to the settings file in the Git repository, we need
# to deploy it on our cluster in wmflabs:
- ssh cvn-appX.cvn.eqiad.wmflabs
- cd /srv/cvn/git/stillalive
- git pull
# Set up a new bot
- add:
- do 'Follow install and setup instructions for the new bot and have it working on the new server'
- do 'Add a new task for MyBotName' to 'localSettings-cvn.yaml' in stillalive.git
# Edit with text editor on your local computer and push with Git
- deploy-a-change
- force-run
# Change on which node a process is pooled
- move:
- do 'Change the pool property from server PREV to NEXT for MyBotName' in 'localSettings-cvn.yaml' in stillalive.git
# Deploy the stillalive change to the previous server first (so that it won't restart there),
# then kill the active bot from IRC. Then copy the bot (while not running) to the new server.
- deploy-a-change@cvn-appX(PREVIOUS)
- quit-a-bot
- ssh cvn-appX(PREV).eqiad.wmflabs
- rsync -v -c -a --delete-delay --compress /srv/cvn/services/cvnbot/MyBotName /data/project/cvn-common/dropbox/
- ssh cvn-appX(NEXT).eqiad.wmflabs
- rsync -v -c -a --delete-delay --compress /data/project/cvn-common/dropbox/MyBotName /srv/cvn/services/cvnbot/
- deploy-a-change@cvn-appX(NEXT)
- force-run@cvn-appX(NEXT)
- remove:
- deploy-a-change
# stillalive only starts processes, it doesn't quit them because
# after you remove the setting, it no longer knows about it.
# Manually terminate the old bot process.
- quit-a-bot
- force-run:
- ssh cvn-appX.cvn.eqiad.wmflabs
- `sudo /srv/cvn/git/stillalive/bin/stillalive --pool=$(hostname)`
# Browse repo: https://gerrit.wikimedia.org/g/labs/countervandalism/CVNBot
tasks CVNBot:
- install:
- do Read https://gerrit.wikimedia.org/g/labs/countervandalism/CVNBot/+/HEAD/docs/install.md
- install:
- ssh cvn-appX.eqiad.wmflabs
- cd /srv/cvn/git/CVNBot
# Fetch the latest code and remove any local modifications or build artefacts
- 'git remote update && git reset --hard origin/main && git clean -dffx'
- do 'Follow CVNBot.git:/docs/install.md#installation'
- Compile with msbuild in Release mode.
- Move the generates files to /srv/cvn/services/cvnbot/MyBotName
- edit CVNBot.ini
# 'botnick' and 'botpass'. See https://meta.wikimedia.org/wiki/Countervandalism_Network/Management#Bots.
# e.g. botnick=CVNBot5 and botpass=(CVN-Bots2 password)
# 'partmsg': "https://meta.wikimedia.org/wiki/CVN"
# 'controlchannel': "#cvn-bots"
# 'feedchannel': "#cvn-..."
# 'broadcastchannel': "#cvn-broadcast"
# 'restartarg': Full in the full path to CVNBot.exe (e.g. "mono /srv/cvn/services/cvnbot/MyBotName/CVNBot.exe")
- if "You're migrating data from an old bot":
- cp ../oldbot/CVNBot.ini .
- cp ../oldbot/Lists.sqlite .
- cp ../oldbot/Projects.xml .
- do 'Set permissions and group ownership correctly (see CVNBot:/docs/install.md)'
- run-db-query:
- ssh cvn-appX.eqiad.wmflabs
- cd /srv/cvn/services/cvnbot/MyBotName
# The following command opens an interactive REPL.
# For the schema, see https://gerrit.wikimedia.org/g/labs/countervandalism/CVNBot/+/v1.21.0/src/CVNBot/ListManager.cs#51
- sqlite3 Lists.sqlite
- upgrade:
- ssh cvn-appX.eqiad.wmflabs
- cd /srv/cvn/git/CVNBot
# Fetch the latest code and remove any local modifications or build artefacts
- 'git remote update && git reset --hard origin/master && git clean -dffx'
- do 'Follow CVNBot.git:/docs/install.md#Upgrade'
- Compile with msbuild in Release mode.
- You MUST delete the newly generated 'CVNBot.ini' and 'Projects.xml' files.
- Quit the bot on IRC (or manually via 'ps' and 'kill' if the bot is unresponsive).
# $ msbuild …
# Usage:
# 1. Run 'msbuild …'
# 2. Copy 'cvnupgrade() {}' from below.
# 3. Run 'cvnupgrade MyCVNBOT1'
- bash: |
cvnupgrade() {
BOT=$1; BOTDIR="/srv/cvn/services/cvnbot/$BOT";
test -n "$BOT" && \
# Reset ownership and group with sudo (before copy)
sudo chown "$USER" "$BOTDIR/" "$BOTDIR/"* && \
sudo chgrp cvn.cvnservice "$BOTDIR/" "$BOTDIR/"* && \
rm -f /srv/cvn/git/CVNBot/src/CVNBot/bin/Release/CVNBot.ini /srv/cvn/git/CVNBot/src/CVNBot/bin/Release/Projects.xml && \
cp /srv/cvn/git/CVNBot/src/CVNBot/bin/Release/* "$BOTDIR/" && \
chgrp cvn.cvnservice "$BOTDIR/"* && \
chmod 660 "$BOTDIR/CVNBot.ini" && \
chmod 775 "$BOTDIR/CVNBot.exe" && \
sudo /srv/cvn/git/stillalive/bin/stillalive --pool=$(hostname);
}
- CVNBot.monitor
- check-chmod:
# To easily quick the permissions of files in a directory
# in a way that matches the parameters passed to chmod, use stat:
- stat -c '%a %n' *
# Produces:
# 775 CVNBot.exe
# 664 CVNBot.exe.config
# 660 CVNBot.ini
# 664 Console.msgs
# 775 Lists.sqlite
# 664 Projects.xml
- monitor:
- ssh cvn-appX.eqiad.wmflabs
# Open a log of recent logs from all CVNBot instances, updating in real time:
- sudo tail -n100 -f /var/log/syslog | grep CVNBot.exe
# Find recent warnings and errors from any CVNBot:
- sudo tail -n1000 -f /var/log/syslog | grep CVNBot.exe | grep -v ': INFO '
# Find errors from any CVNBot:
- sudo tail -n+0 -f /var/log/syslog | grep CVNBot | grep -E ': (ERROR|FATAL) '
# Find errors from any CVNBot (including yesterday):
- sudo tail -n+0 -f /var/log/syslog{.1,} | grep CVNBot | grep -E ': (ERROR|FATAL) '
# Show last 10 log messages grouped by bot
- alias cvnlog='for dir in `ls -d /srv/cvn/services/cvnbot/* | sort -V`; do name=$(basename "$dir"); echo; echo "# $name"; (sudo cat /var/log/syslog | grep -F "[$name]" | tail -n10); done;'
- cvnlog
- test:
# Run test on a cvn-app server because cvn-dev is subject to general
# IP connection limit of all of wmflabs (cvn-app servers have a dedicated IP)
- ssh cvn-appX.eqiad.wmflabs
- cd /srv/cvn/services/cvnbot/MyBotName
- mono CVNBot.exe
- assert "Bot MyBotName has joined #cvn-... channel and works as expected"
- do "Any additional setup if you need to (e.g. load or drop more wikis in memory)"
- do "kill the process from the command line (it was started from your shell so Ctrl^C will do)"