/
logagent-setup.sh
executable file
·472 lines (430 loc) · 12.7 KB
/
logagent-setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
#!/usr/bin/env bash
PLATFORM=$(uname)
SERVICE_NAME=logagent
INITD_SERVICE_FILE="/etc/init.d/logagent"
LAUNCHCTL_SERVICE_FILE="/Library/LaunchDaemons/com.sematext.logagent.plist"
SYSTEMD_SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service"
e=$'\e'
COLORblue="$e[0;36m"
COLORred="$e[0;31m"
COLORreset="$e[0m"
#nodeExecutable=`$(command -v node)||$(command -v nodejs)`
PATTERN="'/var/log/**/*.log'"
TOKEN=$1
while getopts ":i:u:g:j:c:l" opt; do
case $opt in
u)
export LOGSENE_RECEIVER_URL=$OPTARG
#shift $((OPTIND-1))
;;
c)
export CONFIG_FILE=${OPTARG}
;;
i)
export TOKEN=$OPTARG
;;
j)
export JOURNALD=yes
if [ ! -f /lib/systemd/system/systemd-journal-upload.service ]; then
echo
echo "journal-upload service not found. Attempting to install it..."
echo
if [ -x "$(command -v apt-get)" ]; then
apt-get update
apt-get -y install systemd-journal-remote
if [ ! $? eq 0 ]; then
echo "Failed to install the systemd-journal-remote. We need this package in order to continue."
exit 1
fi
else
if [ -x "$(command -v yum)" ]; then
yum clean all
yum -y install systemd-journal-gateway
if [ ! $? -eq 0 ]; then
echo "Failed to install the systemd-journal-remote. We need this package in order to continue."
exit 1
fi
else
echo "Can't find yum or apt. Please install journal-upload manually and try again."
exit 1
fi
fi
fi
;;
l) # stands for Linux :)
export JOURNALCTL=yes
;;
g)
export PATTERN="$OPTARG"
;;
\?)
echo "Invalid option: -$OPTARG" >&2
;;
esac
done
export LOGSENE_RECEIVER_URL=${LOGSENE_RECEIVER_URL:-https://logsene-receiver.sematext.com}
echo Set Logs receiver url: $LOGSENE_RECEIVER_URL
function generate_upstart()
{
echo -e "description \"Sematext Logagent\"
start on runlevel [2345]
stop on runlevel [06]
respawn
chdir /tmp
exec $1 --config $SPM_AGENT_CONFIG_FILE " > /etc/init/${SERVICE_NAME}.conf
runCommand "initctl reload-configuration"
stop $SERVICE_NAME 2> /dev/null
runCommand "start ${SERVICE_NAME}"
}
function generate_systemd()
{
echo -e \
"
[Unit]
Description=Sematext Logagent
After=network.target
[Service]
Restart=always\nRestartSec=10
ExecStart=$1 --config $SPM_AGENT_CONFIG_FILE
[Install]
WantedBy=multi-user.target" > $SYSTEMD_SERVICE_FILE
echo "Service file $SERVICE_FILE:"
cat $SYSTEMD_SERVICE_FILE
runCommand "systemctl enable $SERVICE_NAME" 1
runCommand "systemctl stop $SERVICE_NAME " 2
runCommand "systemctl start $SERVICE_NAME" 3
sleep 1
runCommand "systemctl status $SERVICE_NAME --no-pager" 4
runCommand "journalctl -n 10 -u $SERVICE_NAME --no-pager" 5
if [ ! -z "$JOURNALD" ]; then
runCommand "systemctl restart systemd-journal-upload.service" 6
fi
}
function generate_initd()
{
cat > /etc/init.d/logagent <<EOL
#!/bin/sh
### BEGIN INIT INFO
# Provides: logagent
# Required-Start: \$local_fs \$network \$named \$time \$syslog
# Required-Stop: \$local_fs \$network \$named \$time \$syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Description: logagent
### END INIT INFO
NAME=logagent
PIDFILE=/var/run/\$NAME.pid
DAEMON=$1
DAEMON_OPTS="--config $SPM_AGENT_CONFIG_FILE"
export PATH="\${PATH:+\$PATH:}/usr/sbin:/sbin"
[ -f /etc/default/\$NAME ] && . /etc/default/\$NAME
is_running() {
[ -f "\$PIDFILE" ] && ps \`cat \$PIDFILE\` > /dev/null 2>&1
}
start() {
if is_running; then
echo "Already started"
else
echo "Starting daemon: "\$NAME
start-stop-daemon --start -q --no-close -m --oknodo -b -p \$PIDFILE -x \$DAEMON -- \$DAEMON_OPTS >> /var/log/\$NAME.log 2>&1
fi
}
stop() {
if is_running; then
echo "Stopping daemon: \$NAME..."
start-stop-daemon --stop --quiet --oknodo -p \$PIDFILE --retry 3
else
echo "\$NAME is not running"
fi
}
case "\$1" in
start)
start
;;
stop)
stop
;;
restart)
echo "Restarting daemon: "\$NAME
stop
start
;;
status)
if is_running; then
echo "Running"
else
echo "Stopped"
exit 1
fi
;;
*)
echo "Usage: "\$1" {start|stop|restart|status}"
exit 1
esac
exit 0
EOL
runCommand "chmod +x $INITD_SERVICE_FILE" 1
runCommand "update-rc.d logagent defaults" 2
runCommand "service logagent start" 3
sleep 2
runCommand "tail -n 10 /var/log/logagent.log" 4
}
function runCommand ()
{
echo $2 $1
$1
}
function generate_la_cfg()
{
# make a backup of original config file
if [ -e "$SPM_AGENT_CONFIG_FILE" ]; then
echo Config file ${SPM_AGENT_CONFIG_FILE} exists already, creating backup file "${SPM_AGENT_CONFIG_FILE}.bak"
mv $SPM_AGENT_CONFIG_FILE "${SPM_AGENT_CONFIG_FILE}.bak"
fi
if [ -e "$3" ]; then
echo Config file ${3} exists already, creating backup file "${3}.bak"
mv "$3" "${3}.bak"
fi
if [ -n "$CONFIG_FILE" ]; then
echo "Copying $CONFIG_FILE to $SPM_AGENT_CONFIG_FILE"
cp $CONFIG_FILE $SPM_AGENT_CONFIG_FILE
else
if [ ! -z "$JOURNALD" ]; then
echo -e \
"
# Global options
options:
# print stats every 60 seconds
printStats: 60
# don't write parsed logs to stdout
suppress: true
# Enable/disable GeoIP lookups
# Startup of logagent might be slower, when downloading the GeoIP database
geoipEnabled: false
# Directory to store Logagent status and temporary files
# this is equals to LOGS_TMP_DIR env variable
diskBufferDir: /tmp/sematext-logagent
# the original line will duplicate everything, so we'll exclude it
includeOriginalLine: false
# journald input listens on localhost:5731
input:
journald-upload:
module: input-journald-upload
port: 5731
bindAddress: localhost
useIndexFromUrlPath: true
systemdUnitFilter:
include: !!js/regexp /.*/i
# here we parse journald logs and remove extra fields
outputFilter:
journald-format:
module: journald-format
# Run Logagent parser for the message field
parseMessageField: true
# JS regular expression to match log source name
matchSource: !!js/regexp journald
removeFields:
module: remove-fields
# JS regular expression to match log source name
matchSource: !!js/regexp journald
# Note: journald format converts to lower case
fields:
- __cursor
- __monotonic_timestamp
- _transport
output:
# index logs in Elasticsearch or Sematext Logs
elasticsearch:
module: elasticsearch
url: $LOGSENE_RECEIVER_URL
" > $SPM_AGENT_CONFIG_FILE
JOURNALD_CONF=/etc/systemd/journal-upload.conf
if [ -e "$JOURNALD_CONF" ]; then
echo Config file $JOURNALD_CONF exists already, creating backup file "${JOURNALD_CONF}.bak"
mv "$JOURNALD_CONF" "${JOURNALD_CONF}.bak"
fi
echo '[Upload]' > $JOURNALD_CONF
echo "URL=http://localhost:5731/$TOKEN" >> $JOURNALD_CONF
else
if [ ! -z "$JOURNALCTL" ]; then
ONE_WEEK_AGO=$(date --date='1 week ago' +"%Y-%m-%d %H:%M:%S")
echo -e \
"
# Global options
options:
# print stats every 60 seconds
printStats: 60
# don't write parsed logs to stdout
suppress: true
# Enable/disable GeoIP lookups
# Startup of logagent might be slower, when downloading the GeoIP database
geoipEnabled: false
# Directory to store Logagent status and temporary files
# this is equals to LOGS_TMP_DIR env variable
diskBufferDir: /tmp/sematext-logagent
input:
journald-json:
module: command
# note journalctl -u unitName can filter logs for systemd-units
command: journalctl -o json --since=\"\$QUERY_TIME\"
sourceName: journald
dateFormat: YYYY-MM-DD HH:mm:ss # date format for $QUERY_TIME and $NOW
restart: 1 # seconds to wait between runs
# where to persist last $QUERY_TIME
lastQueryTimeFile: /var/run/logagentLastQueryTimeFile
# pull logs from one week ago initially
initialQueryTime: \"$ONE_WEEK_AGO\"
# 100MB pipe buffer
maxBuffer: 100000000
# here we parse journald logs and remove extra fields
outputFilter:
journald-format:
module: journald-format
# Run Logagent parser for the message field
parseMessageField: true
# JS regular expression to match log source name
matchSource: !!js/regexp journald
lowercase-fields:
module: lowercase-fields
# JS regular expression to match log source name
matchSource: !!js/regexp journald
allFields: true
removeFields:
module: remove-fields
# JS regular expression to match log source name
matchSource: !!js/regexp journald
# Note: journald format converts to lower case
fields:
- __cursor
- __monotonic_timestamp
- _transport
output:
# index logs in Elasticsearch or Sematext Logs
elasticsearch:
module: elasticsearch
url: $LOGSENE_RECEIVER_URL
# default index (Logs token) to use for journald logs:
indices:
$TOKEN:
- journald
" > $SPM_AGENT_CONFIG_FILE
else
echo -e \
"
# Global options
options:
# print stats every 60 seconds
printStats: 60
# don't write parsed logs to stdout
suppress: true
# Enable/disable GeoIP lookups
# Startup of logagent might be slower, when downloading the GeoIP database
geoipEnabled: false
# Directory to store Logagent status and temporary files
# this is equals to LOGS_TMP_DIR env variable
diskBufferDir: /tmp/sematext-logagent
input:
# a list of glob patterns to watch files to tail
files:
- $PATTERN
output:
# index logs in Elasticsearch or Sematext Logs
elasticsearch:
module: elasticsearch
url: $LOGSENE_RECEIVER_URL
# default index (Logs token) to use:
index: $TOKEN
" > $SPM_AGENT_CONFIG_FILE
fi
fi
fi
}
function generate_launchctl()
{
echo -e \
"
<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">
<plist version=\"1.0\">
<dict>
<key>EnvironmentVariables</key>
<dict>
<key>PATH</key>
<string>/usr/local/bin/:$PATH</string>
</dict>
<key>Label</key>
<string>com.sematext.logagent</string>
<key>ProgramArguments</key>
<array>
<string>$1</string>
<string>--config</string>
<string>/etc/sematext/logagent.conf</string>
</array>
<key>StandardErrorPath</key>
<string>/Library/Logs/logagent.log</string>
<key>StandardOutPath</key>
<string>/Library/Logs/logagent.log</string>
<key>RunAtLoad</key>
<true/>
</dict>
</plist>" > $LAUNCHCTL_SERVICE_FILE
echo "Service file $LAUNCHCTL_SERVICE_FILE:"
# cat $LAUNCHCTL_SERVICE_FILE
runCommand "launchctl unload -w -F $LAUNCHCTL_SERVICE_FILE" 1
runCommand "launchctl load -w -F $LAUNCHCTL_SERVICE_FILE" 2
runCommand "launchctl start com.sematext.logagent" 3
runCommand "tail -n 10 /Library/Logs/logagent.log" 4
}
function install_script ()
{
export SPM_AGENT_CONFIG_FILE="/etc/sematext/logagent.conf"
mkdir -p /etc/sematext
echo "Create config file: $SPM_AGENT_CONFIG_FILE"
generate_la_cfg $2 $3
# echo "-s --logsene-tmp-dir /tmp -t $2 -g $3" > $SPM_AGENT_CONFIG_FILE
runCommand "chown root $SPM_AGENT_CONFIG_FILE"
runCommand "chmod 0600 $SPM_AGENT_CONFIG_FILE"
cat $SPM_AGENT_CONFIG_FILE
if [[ $PLATFORM = "Darwin" ]]; then
echo "Generate launchd script ${LAUNCHCTL_SERVICE_FILE}"
generate_launchctl $1 $2 $3
return
fi
if [[ `/sbin/init --version` =~ upstart ]]>/dev/null 2>&1; then
echo "Generate upstart script ${UPSTART_SERVICE_FILE}"
generate_upstart $1 $2 $3
return
fi
if [[ `systemctl` =~ -\.mount ]]>/dev/null 2>&1; then
echo "Generate systemd script "
generate_systemd $1 $2 $3
return
fi
if [[ `. /etc/os-release && echo $VERSION =~ wheezy` ]]; then
echo "Generate init.d script"
generate_initd $1 $2 $3
return
fi
}
LOGAGENT_COMMAND=$(command -v logagent)
echo $LOGAGENT_COMMAND
if [ -n "$TOKEN" ] ; then
install_script $LOGAGENT_COMMAND $TOKEN "${PATTERN}";
else
if [ -n "$CONFIG_FILE" ]; then
install_script $LOGAGENT_COMMAND "NO_TOKEN_PROVIDED_JUST_CONFIG"
else
echo "${COLORred}Missing paramaters. Usage:"
echo `basename $0` "-i LOGS_TOKEN -g '/var/log/**/*.log' -u https://logsene-receiver.sematext.com"
echo "Please obtain your Logs App token for US region from https://apps.sematext.com/"
echo "Please obtain your Logs App token for EU region from https://apps.eu.sematext.com/"
echo "For EU region use -u https://logsene-receiver.eu.sematext.com/"
echo "To set up Logagent to pull journal entries via journalctl, add -l"
echo "To set up Logagent as a local receiver for journald-upload, add -j true"
echo
echo "If you already have Logagent config and you want to install the service, use:"
echo " " `basename $0` "-c /PATH/TO/CONFIG_FILE $COLORreset"
fi
fi
echo
echo "Logagent documentation: https://sematext.com/docs/logagent"