From 91799bd435bbf87ca0da8fde7bdb526a292b8fc3 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Fri, 6 Jul 2018 15:08:37 +0100 Subject: [PATCH 01/24] First attempt at the monitoring directory structure --- code/monit/fill_templates.sh | 1 + code/monit/monit.sh | 15 ++ code/monit/monitrc | 311 +++++++++++++++++++++++++++ code/monit/scripts/monitalert.cfg | 1 + code/monit/scripts/monitconfig.cfg | 1 + code/monit/scripts/monittemplate.txt | 1 + 6 files changed, 330 insertions(+) create mode 100644 code/monit/fill_templates.sh create mode 100644 code/monit/monit.sh create mode 100755 code/monit/monitrc create mode 100644 code/monit/scripts/monitalert.cfg create mode 100644 code/monit/scripts/monitconfig.cfg create mode 100644 code/monit/scripts/monittemplate.txt diff --git a/code/monit/fill_templates.sh b/code/monit/fill_templates.sh new file mode 100644 index 000000000..f4b0ccf4c --- /dev/null +++ b/code/monit/fill_templates.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/code/monit/monit.sh b/code/monit/monit.sh new file mode 100644 index 000000000..a883f1419 --- /dev/null +++ b/code/monit/monit.sh @@ -0,0 +1,15 @@ +#!/bin/sh -xv +if [ "-bash" = $0 ]; then + dirpath="${BASH_SOURCE[0]}" +else + dirpath="$0" +fi + +eval ". $(dirname "$dirpath")/setenv.sh" + +if [ ! -f $KDBAPPCONFIG/monitrc ]; then # run fill_templates.sh if monitrc is no present + eval ". $(dirname "$dirpath")/fill_templates.sh" +fi + +$TORQBLKTREE/monit/bin/monit -c $KDBAPPCONFIG/monitrc $@ + diff --git a/code/monit/monitrc b/code/monit/monitrc new file mode 100755 index 000000000..4f04772bb --- /dev/null +++ b/code/monit/monitrc @@ -0,0 +1,311 @@ +############################################################################### +## Monit control file +############################################################################### +## +## Comments begin with a '#' and extend through the end of the line. Keywords +## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. +## +## Below you will find examples of some frequently used statements. For +## information about the control file and a complete list of statements and +## options, please have a look in the Monit manual. +## +## +############################################################################### +## Global section +############################################################################### +## +## Start Monit in the background (run as a daemon): +# + set daemon 30 # check services at 30 seconds intervals +# with start delay 240 # optional: delay the first check by 4-minutes (by +# # default Monit check immediately after Monit start) +# +# +## Set syslog logging. If you want to log to a standalone log file instead, +## specify the full path to the log file +# +#set logfile $TORQMONIT/monit.log + +# +# +## Set the location of the Monit lock file which stores the process id of the +## running Monit instance. By default this file is stored in $HOME/.monit.pid +# +#set pidfile $TORQMONIT/monit.pid +# +## Set the location of the Monit id file which stores the unique id for the +## Monit instance. The id is generated and stored on first Monit start. By +## default the file is placed in $HOME/.monit.id. +# +# set idfile /var/.monit.id + #set idfile $TORQMONIT/.monit.id +# +## Set the location of the Monit state file which saves monitoring states +## on each cycle. By default the file is placed in $HOME/.monit.state. If +## the state file is stored on a persistent filesystem, Monit will recover +## the monitoring state across reboots. If it is on temporary filesystem, the +## state will be lost on reboot which may be convenient in some situations. +# + #set statefile $TORQMONIT/.monit.state +# set statefile /var/lib/monit/state +# +# + +## Set limits for various tests. The following example shows the default values: +## +# set limits { +# programOutput: 512 B, # check program's output truncate limit +# sendExpectBuffer: 256 B, # limit for send/expect protocol test +# fileContentBuffer: 512 B, # limit for file content test +# httpContentBuffer: 1 MB, # limit for HTTP content test +# networkTimeout: 5 seconds # timeout for network I/O +# programTimeout: 300 seconds # timeout for check program +# stopTimeout: 30 seconds # timeout for service stop +# startTimeout: 30 seconds # timeout for service start +# restartTimeout: 30 seconds # timeout for service restart +# } + +## Set global SSL options (just most common options showed, see manual for +## full list). +# +# set ssl { +# verify : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED) +# selfsigned : allow # allow self signed SSL certificates (reject by default) +# } +# +# +## Set the list of mail servers for alert delivery. Multiple servers may be +## specified using a comma separator. If the first mail server fails, Monit +# will use the second mail server in the list and so on. By default Monit uses +# port 25 - it is possible to override this with the PORT option. +# +# set mailserver mail.bar.baz, # primary mailserver +# backup.bar.baz port 10025, # backup mailserver on port 10025 +# localhost # fallback relay + #set mailserver smtp.office365.com port 587 + # username "auto@aquaq.co.uk" + # password "Circuit20" + # using ssl + # with timeout 30 seconds + + + #set alert tradefeedr@aquaq.co.uk +# +# +## By default Monit will drop alert events if no mail servers are available. +## If you want to keep the alerts for later delivery retry, you can use the +## EVENTQUEUE statement. The base directory where undelivered alerts will be +## stored is specified by the BASEDIR option. You can limit the queue size +## by using the SLOTS option (if omitted, the queue is limited by space +## available in the back end filesystem). +# + #set eventqueue +# basedir /var/lib/monit/events # set the base directory where events will be stored + # basedir $TORQMONIT/events + # slots 100 # optionally limit the queue size +# +# +## Send status and events to M/Monit (for more informations about M/Monit +## see https://mmonit.com/). By default Monit registers credentials with +## M/Monit so M/Monit can smoothly communicate back to Monit and you don't +## have to register Monit credentials manually in M/Monit. It is possible to +## disable credential registration using the commented out option below. +## Though, if safety is a concern we recommend instead using https when +## communicating with M/Monit and send credentials encrypted. The password +## should be URL encoded if it contains URL-significant characters like +## \":\", \"?\", \"@\". +# +# set mmonit http://monit:monit@104.46.37.155:2810/collector +# # and register without credentials # Don't register credentials +# +# +## Monit by default uses the following format for alerts if the the mail-format +## statement is missing:: +## --8<-- +## set mail-format { +## from: Monit +## subject: monit alert -- $EVENT $SERVICE +## message: $EVENT Service $SERVICE +## Date: $DATE +## Action: $ACTION +## Host: $HOST +## Description: $DESCRIPTION +## +## Your faithful employee, +## Monit +## } +## --8<-- + # set mail-format { + # from: auto@aquaq.co.uk + # subject: [$HOST] monit alert -- $EVENT $SERVICE} +## +## You can override this message format or parts of it, such as subject +## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. +## are expanded at runtime. For example, to override the sender, use: +# +# set mail-format { from: monit@foo.bar } +# +# +## You can set alert recipients whom will receive alerts if/when a +## service defined in this file has errors. Alerts may be restricted on +## events by using a filter as in the second example below. +# +# set alert sysadm@foo.bar # receive all alerts +# +## Do not alert when Monit starts, stops or performs a user initiated action. +## This filter is recommended to avoid getting alerts for trivial cases. +# +# set alert your-name@your.domain not on { instance, action } +# +# +## Monit has an embedded HTTP interface which can be used to view status of +## services monitored and manage services from a web interface. The HTTP +## interface is also required if you want to issue Monit commands from the +## command line, such as 'monit status' or 'monit restart service' The reason +## for this is that the Monit client uses the HTTP interface to send these +## commands to a running Monit daemon. See the Monit Wiki if you want to +## enable SSL for the HTTP interface. +# +#set httpd port 2812 and +# use address localhost # only accept connection from localhost +# allow localhost # allow localhost to connect to the server and + # allow admin:monit # require user 'admin' with password 'monit' + +############################################################################### +## Services +############################################################################### +## +## Check general system resources such as load average, cpu and memory +## usage. Each test specifies a resource, conditions and the action to be +## performed should a test fail. +# +# check system $HOST +# if loadavg (1min) > 4 then alert +# if loadavg (5min) > 2 then alert +# if cpu usage > 95% for 10 cycles then alert +# if memory usage > 75% then alert +# if swap usage > 25% then alert +# +# +## Check if a file exists, checksum, permissions, uid and gid. In addition +## to alert recipients in the global section, customized alert can be sent to +## additional recipients by specifying a local alert handler. The service may +## be grouped using the GROUP option. More than one group can be specified by +## repeating the 'group name' statement. +# +# check file apache_bin with path /usr/local/apache/bin/httpd +# if failed checksum and +# expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor +# if failed permission 755 then unmonitor +# if failed uid root then unmonitor +# if failed gid root then unmonitor +# alert security@foo.bar on { +# checksum, permission, uid, gid, unmonitor +# } with the mail-format { subject: Alarm! } +# group server +# +# +## Check that a process is running, in this case Apache, and that it respond +## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory, +## and number of children. If the process is not running, Monit will restart +## it by default. In case the service is restarted very often and the +## problem remains, it is possible to disable monitoring using the TIMEOUT +## statement. This service depends on another service (apache_bin) which +## is defined above. +# +# check process apache with pidfile /usr/local/apache/logs/httpd.pid +# start program = \"/etc/init.d/httpd start\" with timeout 60 seconds +# stop program = \"/etc/init.d/httpd stop\" +# if cpu > 60% for 2 cycles then alert +# if cpu > 80% for 5 cycles then restart +# if totalmem > 200.0 MB for 5 cycles then restart +# if children > 250 then restart +# if loadavg(5min) greater than 10 for 8 cycles then stop +# if failed host www.tildeslash.com port 80 protocol http +# and request \"/somefile.html\" +# then restart +# if failed port 443 protocol https with timeout 15 seconds then restart +# if 3 restarts within 5 cycles then unmonitor +# depends on apache_bin +# group server +# +# +## Check filesystem permissions, uid, gid, space and inode usage. Other services, +## such as databases, may depend on this resource and an automatically graceful +## stop may be cascaded to them before the filesystem will become full and data +## lost. +# +# check filesystem datafs with path /dev/sdb1 +# start program = \"/bin/mount /data\" +# stop program = \"/bin/umount /data\" +# if failed permission 660 then unmonitor +# if failed uid root then unmonitor +# if failed gid disk then unmonitor +# if space usage > 80% for 5 times within 15 cycles then alert +# if space usage > 99% then stop +# if inode usage > 30000 then alert +# if inode usage > 99% then stop +# group server +# +# +## Check a file's timestamp. In this example, we test if a file is older +## than 15 minutes and assume something is wrong if its not updated. Also, +## if the file size exceed a given limit, execute a script +# +# check file database with path /data/mydatabase.db +# if failed permission 700 then alert +# if failed uid data then alert +# if failed gid data then alert +# if timestamp > 15 minutes then alert +# if size > 100 MB then exec \"/my/cleanup/script\" as uid dba and gid dba +# +# +## Check directory permission, uid and gid. An event is triggered if the +## directory does not belong to the user with uid 0 and gid 0. In addition, +## the permissions have to match the octal description of 755 (see chmod(1)). +# +# check directory bin with path /bin +# if failed permission 755 then unmonitor +# if failed uid 0 then unmonitor +# if failed gid 0 then unmonitor +# +# +## Check a remote host availability by issuing a ping test and check the +## content of a response from a web server. Up to three pings are sent and +## connection to a port and an application level network check is performed. +# +# check host myserver with address 192.168.1.1 +# if failed ping then alert +# if failed port 3306 protocol mysql with timeout 15 seconds then alert +# if failed port 80 protocol http +# and request /some/path with content = \"a string\" +# then alert +# +# +## Check a network link status (up/down), link capacity changes, saturation +## and bandwidth usage. +# +# check network public with interface eth0 +# if failed link then alert +# if changed link then alert +# if saturation > 90% then alert +# if download > 10 MB/s then alert +# if total uploaded > 1 GB in last hour then alert +# +# +## Check custom program status output. +# +# check program myscript with path /usr/local/bin/myscript.sh +# if status != 0 then alert +# +# +############################################################################### +## Includes +############################################################################### +## +## It is possible to include additional configuration parts from other files or +## directories. +# +# include /etc/monit.d/* +# +# include $TORQBLKTREE/scripts/monit/*.cfg diff --git a/code/monit/scripts/monitalert.cfg b/code/monit/scripts/monitalert.cfg new file mode 100644 index 000000000..b294ccee3 --- /dev/null +++ b/code/monit/scripts/monitalert.cfg @@ -0,0 +1 @@ +#text diff --git a/code/monit/scripts/monitconfig.cfg b/code/monit/scripts/monitconfig.cfg new file mode 100644 index 000000000..ebfcc8697 --- /dev/null +++ b/code/monit/scripts/monitconfig.cfg @@ -0,0 +1 @@ +#text diff --git a/code/monit/scripts/monittemplate.txt b/code/monit/scripts/monittemplate.txt new file mode 100644 index 000000000..ebfcc8697 --- /dev/null +++ b/code/monit/scripts/monittemplate.txt @@ -0,0 +1 @@ +#text From 695bca6c6dd3569f34d9251f3be267e96a323bd1 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Fri, 6 Jul 2018 16:42:36 +0100 Subject: [PATCH 02/24] Created the backbone of fills_templates.sh Created the backbone of monit.sh --- code/monit/fill_templates.sh | 1 - code/monit/{monitrc => monitrcexample} | 0 fill_templates.sh | 40 ++++++++++++++++++++++++++ code/monit/monit.sh => monit.sh | 6 ++-- 4 files changed, 43 insertions(+), 4 deletions(-) delete mode 100644 code/monit/fill_templates.sh rename code/monit/{monitrc => monitrcexample} (100%) create mode 100644 fill_templates.sh rename code/monit/monit.sh => monit.sh (50%) diff --git a/code/monit/fill_templates.sh b/code/monit/fill_templates.sh deleted file mode 100644 index f4b0ccf4c..000000000 --- a/code/monit/fill_templates.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/code/monit/monitrc b/code/monit/monitrcexample similarity index 100% rename from code/monit/monitrc rename to code/monit/monitrcexample diff --git a/fill_templates.sh b/fill_templates.sh new file mode 100644 index 000000000..225143c35 --- /dev/null +++ b/fill_templates.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +#FUNTION DECLARATION +createmonitrc(){ + #function to read all processes from the processes.csv + echo $1 + echo $2 + echo $3 +} + +createoutput(){ + #function to create the output file + +} + +createmonitrc(){ + #function to create the monitrc file + +} + +#SETTING ENVIRONMENT VARIABLES +if [ "-bash" = $0 ]; then + dirpath="${BASH_SOURCE[0]}" +else + dirpath="$0" +fi + +eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables +templates="${TORQHOME}/code/monit/templates" #set temmplates folder +config="${TORQHOME}/code/monit/" #set configs folder +monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf + +mkdir -p $config #ensure output directory is created + +#monittemplate="$(cat $templates/monittemplate.txt)" + +createmonitrc "$templates/monitconfig.cfg" "one" "two" + +#TESTING (to be removed) +#echo ${TORQHOME} diff --git a/code/monit/monit.sh b/monit.sh similarity index 50% rename from code/monit/monit.sh rename to monit.sh index a883f1419..5f963a50d 100644 --- a/code/monit/monit.sh +++ b/monit.sh @@ -1,4 +1,4 @@ -#!/bin/sh -xv +#!/bin/sh if [ "-bash" = $0 ]; then dirpath="${BASH_SOURCE[0]}" else @@ -7,9 +7,9 @@ fi eval ". $(dirname "$dirpath")/setenv.sh" -if [ ! -f $KDBAPPCONFIG/monitrc ]; then # run fill_templates.sh if monitrc is no present +if [ ! -f ${TORQHOME}/code/monit/monitrc ];then # run fill_templates.sh if monitrc is no present eval ". $(dirname "$dirpath")/fill_templates.sh" fi -$TORQBLKTREE/monit/bin/monit -c $KDBAPPCONFIG/monitrc $@ +monit -c ${TORQHOME}/code/monit/monitrc $@ From e95611d16205a57f6f464f7c06ec7fe9e7595633 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Mon, 9 Jul 2018 11:56:03 +0100 Subject: [PATCH 03/24] Added function createmonitrc which takes the procname and proctype from process.csv Added monitconfig.cfg which contains the location of the process.csv and the output file --- code/monit/templates/monitconfig.cfg | 1 + code/monit/templates/monittemplate.txt | 7 +++++ fill_templates.sh | 38 +++++++++++++++++++------- 3 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 code/monit/templates/monitconfig.cfg create mode 100644 code/monit/templates/monittemplate.txt diff --git a/code/monit/templates/monitconfig.cfg b/code/monit/templates/monitconfig.cfg new file mode 100644 index 000000000..ba3bff528 --- /dev/null +++ b/code/monit/templates/monitconfig.cfg @@ -0,0 +1 @@ +/home/$USER/TorQProdSupp/TorQDev/TorQ-Finance-Starter-Pack/appconfig/process.csv monitconfig.cfg diff --git a/code/monit/templates/monittemplate.txt b/code/monit/templates/monittemplate.txt new file mode 100644 index 000000000..b3e7ce480 --- /dev/null +++ b/code/monit/templates/monittemplate.txt @@ -0,0 +1,7 @@ +check process $procname + matching \"$KDBSTACKID -proctype $proctype -procname $procname\" + start program = \"/bin/bash $startscript $procname\" + with timeout 10 seconds + stop program = \"/bin/bash $stopscript $procname\" + every \"* * * * *\" + mode active diff --git a/fill_templates.sh b/fill_templates.sh index 225143c35..4048f5555 100644 --- a/fill_templates.sh +++ b/fill_templates.sh @@ -3,20 +3,35 @@ #FUNTION DECLARATION createmonitrc(){ #function to read all processes from the processes.csv - echo $1 - echo $2 - echo $3 + #and build the array + + cat $1|while read line; do + conf=($line) + procs=$(eval "echo \"${conf[0]}\"") + #echo $procs + output="$configs/${conf[1]}" + done + + proclist=`tail -n +2 /home/$USER/TorQProdSupp/TorQDev/TorQ-Finance-Starter-Pack/appconfig/process.csv | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` + echo "$proclist"|while read procs; do + array=($procs) + proctype=${array[0]} + procname=${array[1]} + "eval echo $2" + #eval "{$2}" >> test.txt + done + } -createoutput(){ +#createoutput(){ #function to create the output file -} +#} -createmonitrc(){ +#createmonitrc(){ #function to create the monitrc file -} +#} #SETTING ENVIRONMENT VARIABLES if [ "-bash" = $0 ]; then @@ -27,14 +42,17 @@ fi eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables templates="${TORQHOME}/code/monit/templates" #set temmplates folder -config="${TORQHOME}/code/monit/" #set configs folder +config="${TORQHOME}/code/monit/" #set configs folder monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf mkdir -p $config #ensure output directory is created -#monittemplate="$(cat $templates/monittemplate.txt)" +monittemplate="$(cat $templates/monittemplate.txt)" + +#echo ${monittemplate} -createmonitrc "$templates/monitconfig.cfg" "one" "two" +createmonitrc "$templates/monitconfig.cfg" "$monittemplate" #TESTING (to be removed) #echo ${TORQHOME} +#echo $templates \ No newline at end of file From 17e46de3ba9c633e7fd71e44fc1482566006f03a Mon Sep 17 00:00:00 2001 From: Aqualex Date: Mon, 9 Jul 2018 17:15:05 +0100 Subject: [PATCH 04/24] Modified monittemplate.txt to use the new TorQ start script Modified the location of the files Updated the monitrc to include the monitconfi.cfg --- code/monit/scripts/monitconfig.cfg | 1 - code/monit/scripts/monittemplate.txt | 1 - code/monit/templates/monitconfig.cfg | 1 - code/scripts/monitrc | 311 ++++++++++++++++++ code/{monit => scripts}/monitrcexample | 0 .../templates}/monitalert.cfg | 0 code/scripts/templates/monitconfig.cfg | 1 + .../templates/monittemplate.txt | 5 +- fill_templates.sh | 56 ++-- monit.sh | 2 +- setenv.sh | 2 +- 11 files changed, 341 insertions(+), 39 deletions(-) delete mode 100644 code/monit/scripts/monitconfig.cfg delete mode 100644 code/monit/scripts/monittemplate.txt delete mode 100644 code/monit/templates/monitconfig.cfg create mode 100755 code/scripts/monitrc rename code/{monit => scripts}/monitrcexample (100%) rename code/{monit/scripts => scripts/templates}/monitalert.cfg (100%) create mode 100644 code/scripts/templates/monitconfig.cfg rename code/{monit => scripts}/templates/monittemplate.txt (56%) diff --git a/code/monit/scripts/monitconfig.cfg b/code/monit/scripts/monitconfig.cfg deleted file mode 100644 index ebfcc8697..000000000 --- a/code/monit/scripts/monitconfig.cfg +++ /dev/null @@ -1 +0,0 @@ -#text diff --git a/code/monit/scripts/monittemplate.txt b/code/monit/scripts/monittemplate.txt deleted file mode 100644 index ebfcc8697..000000000 --- a/code/monit/scripts/monittemplate.txt +++ /dev/null @@ -1 +0,0 @@ -#text diff --git a/code/monit/templates/monitconfig.cfg b/code/monit/templates/monitconfig.cfg deleted file mode 100644 index ba3bff528..000000000 --- a/code/monit/templates/monitconfig.cfg +++ /dev/null @@ -1 +0,0 @@ -/home/$USER/TorQProdSupp/TorQDev/TorQ-Finance-Starter-Pack/appconfig/process.csv monitconfig.cfg diff --git a/code/scripts/monitrc b/code/scripts/monitrc new file mode 100755 index 000000000..f681d4254 --- /dev/null +++ b/code/scripts/monitrc @@ -0,0 +1,311 @@ +############################################################################### +## Monit control file +############################################################################### +## +## Comments begin with a '#' and extend through the end of the line. Keywords +## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. +## +## Below you will find examples of some frequently used statements. For +## information about the control file and a complete list of statements and +## options, please have a look in the Monit manual. +## +## +############################################################################### +## Global section +############################################################################### +## +## Start Monit in the background (run as a daemon): +# + set daemon 30 # check services at 30 seconds intervals +# with start delay 240 # optional: delay the first check by 4-minutes (by +# # default Monit check immediately after Monit start) +# +# +## Set syslog logging. If you want to log to a standalone log file instead, +## specify the full path to the log file +# +#set logfile $TORQMONIT/monit.log + +# +# +## Set the location of the Monit lock file which stores the process id of the +## running Monit instance. By default this file is stored in $HOME/.monit.pid +# +#set pidfile $TORQMONIT/monit.pid +# +## Set the location of the Monit id file which stores the unique id for the +## Monit instance. The id is generated and stored on first Monit start. By +## default the file is placed in $HOME/.monit.id. +# +# set idfile /var/.monit.id + #set idfile $TORQMONIT/.monit.id +# +## Set the location of the Monit state file which saves monitoring states +## on each cycle. By default the file is placed in $HOME/.monit.state. If +## the state file is stored on a persistent filesystem, Monit will recover +## the monitoring state across reboots. If it is on temporary filesystem, the +## state will be lost on reboot which may be convenient in some situations. +# + #set statefile $TORQMONIT/.monit.state +# set statefile /var/lib/monit/state +# +# + +## Set limits for various tests. The following example shows the default values: +## +# set limits { +# programOutput: 512 B, # check program's output truncate limit +# sendExpectBuffer: 256 B, # limit for send/expect protocol test +# fileContentBuffer: 512 B, # limit for file content test +# httpContentBuffer: 1 MB, # limit for HTTP content test +# networkTimeout: 5 seconds # timeout for network I/O +# programTimeout: 300 seconds # timeout for check program +# stopTimeout: 30 seconds # timeout for service stop +# startTimeout: 30 seconds # timeout for service start +# restartTimeout: 30 seconds # timeout for service restart +# } + +## Set global SSL options (just most common options showed, see manual for +## full list). +# +# set ssl { +# verify : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED) +# selfsigned : allow # allow self signed SSL certificates (reject by default) +# } +# +# +## Set the list of mail servers for alert delivery. Multiple servers may be +## specified using a comma separator. If the first mail server fails, Monit +# will use the second mail server in the list and so on. By default Monit uses +# port 25 - it is possible to override this with the PORT option. +# +# set mailserver mail.bar.baz, # primary mailserver +# backup.bar.baz port 10025, # backup mailserver on port 10025 +# localhost # fallback relay + #set mailserver smtp.office365.com port 587 + # username "auto@aquaq.co.uk" + # password "Circuit20" + # using ssl + # with timeout 30 seconds + + + #set alert tradefeedr@aquaq.co.uk +# +# +## By default Monit will drop alert events if no mail servers are available. +## If you want to keep the alerts for later delivery retry, you can use the +## EVENTQUEUE statement. The base directory where undelivered alerts will be +## stored is specified by the BASEDIR option. You can limit the queue size +## by using the SLOTS option (if omitted, the queue is limited by space +## available in the back end filesystem). +# + #set eventqueue +# basedir /var/lib/monit/events # set the base directory where events will be stored + # basedir $TORQMONIT/events + # slots 100 # optionally limit the queue size +# +# +## Send status and events to M/Monit (for more informations about M/Monit +## see https://mmonit.com/). By default Monit registers credentials with +## M/Monit so M/Monit can smoothly communicate back to Monit and you don't +## have to register Monit credentials manually in M/Monit. It is possible to +## disable credential registration using the commented out option below. +## Though, if safety is a concern we recommend instead using https when +## communicating with M/Monit and send credentials encrypted. The password +## should be URL encoded if it contains URL-significant characters like +## \":\", \"?\", \"@\". +# +# set mmonit http://monit:monit@104.46.37.155:2810/collector +# # and register without credentials # Don't register credentials +# +# +## Monit by default uses the following format for alerts if the the mail-format +## statement is missing:: +## --8<-- +## set mail-format { +## from: Monit +## subject: monit alert -- $EVENT $SERVICE +## message: $EVENT Service $SERVICE +## Date: $DATE +## Action: $ACTION +## Host: $HOST +## Description: $DESCRIPTION +## +## Your faithful employee, +## Monit +## } +## --8<-- + # set mail-format { + # from: auto@aquaq.co.uk + # subject: [$HOST] monit alert -- $EVENT $SERVICE} +## +## You can override this message format or parts of it, such as subject +## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. +## are expanded at runtime. For example, to override the sender, use: +# +# set mail-format { from: monit@foo.bar } +# +# +## You can set alert recipients whom will receive alerts if/when a +## service defined in this file has errors. Alerts may be restricted on +## events by using a filter as in the second example below. +# +# set alert sysadm@foo.bar # receive all alerts +# +## Do not alert when Monit starts, stops or performs a user initiated action. +## This filter is recommended to avoid getting alerts for trivial cases. +# +# set alert your-name@your.domain not on { instance, action } +# +# +## Monit has an embedded HTTP interface which can be used to view status of +## services monitored and manage services from a web interface. The HTTP +## interface is also required if you want to issue Monit commands from the +## command line, such as 'monit status' or 'monit restart service' The reason +## for this is that the Monit client uses the HTTP interface to send these +## commands to a running Monit daemon. See the Monit Wiki if you want to +## enable SSL for the HTTP interface. +# +#set httpd port 2812 and +# use address localhost # only accept connection from localhost +# allow localhost # allow localhost to connect to the server and + # allow admin:monit # require user 'admin' with password 'monit' + +############################################################################### +## Services +############################################################################### +## +## Check general system resources such as load average, cpu and memory +## usage. Each test specifies a resource, conditions and the action to be +## performed should a test fail. +# +# check system $HOST +# if loadavg (1min) > 4 then alert +# if loadavg (5min) > 2 then alert +# if cpu usage > 95% for 10 cycles then alert +# if memory usage > 75% then alert +# if swap usage > 25% then alert +# +# +## Check if a file exists, checksum, permissions, uid and gid. In addition +## to alert recipients in the global section, customized alert can be sent to +## additional recipients by specifying a local alert handler. The service may +## be grouped using the GROUP option. More than one group can be specified by +## repeating the 'group name' statement. +# +# check file apache_bin with path /usr/local/apache/bin/httpd +# if failed checksum and +# expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor +# if failed permission 755 then unmonitor +# if failed uid root then unmonitor +# if failed gid root then unmonitor +# alert security@foo.bar on { +# checksum, permission, uid, gid, unmonitor +# } with the mail-format { subject: Alarm! } +# group server +# +# +## Check that a process is running, in this case Apache, and that it respond +## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory, +## and number of children. If the process is not running, Monit will restart +## it by default. In case the service is restarted very often and the +## problem remains, it is possible to disable monitoring using the TIMEOUT +## statement. This service depends on another service (apache_bin) which +## is defined above. +# +# check process apache with pidfile /usr/local/apache/logs/httpd.pid +# start program = \"/etc/init.d/httpd start\" with timeout 60 seconds +# stop program = \"/etc/init.d/httpd stop\" +# if cpu > 60% for 2 cycles then alert +# if cpu > 80% for 5 cycles then restart +# if totalmem > 200.0 MB for 5 cycles then restart +# if children > 250 then restart +# if loadavg(5min) greater than 10 for 8 cycles then stop +# if failed host www.tildeslash.com port 80 protocol http +# and request \"/somefile.html\" +# then restart +# if failed port 443 protocol https with timeout 15 seconds then restart +# if 3 restarts within 5 cycles then unmonitor +# depends on apache_bin +# group server +# +# +## Check filesystem permissions, uid, gid, space and inode usage. Other services, +## such as databases, may depend on this resource and an automatically graceful +## stop may be cascaded to them before the filesystem will become full and data +## lost. +# +# check filesystem datafs with path /dev/sdb1 +# start program = \"/bin/mount /data\" +# stop program = \"/bin/umount /data\" +# if failed permission 660 then unmonitor +# if failed uid root then unmonitor +# if failed gid disk then unmonitor +# if space usage > 80% for 5 times within 15 cycles then alert +# if space usage > 99% then stop +# if inode usage > 30000 then alert +# if inode usage > 99% then stop +# group server +# +# +## Check a file's timestamp. In this example, we test if a file is older +## than 15 minutes and assume something is wrong if its not updated. Also, +## if the file size exceed a given limit, execute a script +# +# check file database with path /data/mydatabase.db +# if failed permission 700 then alert +# if failed uid data then alert +# if failed gid data then alert +# if timestamp > 15 minutes then alert +# if size > 100 MB then exec \"/my/cleanup/script\" as uid dba and gid dba +# +# +## Check directory permission, uid and gid. An event is triggered if the +## directory does not belong to the user with uid 0 and gid 0. In addition, +## the permissions have to match the octal description of 755 (see chmod(1)). +# +# check directory bin with path /bin +# if failed permission 755 then unmonitor +# if failed uid 0 then unmonitor +# if failed gid 0 then unmonitor +# +# +## Check a remote host availability by issuing a ping test and check the +## content of a response from a web server. Up to three pings are sent and +## connection to a port and an application level network check is performed. +# +# check host myserver with address 192.168.1.1 +# if failed ping then alert +# if failed port 3306 protocol mysql with timeout 15 seconds then alert +# if failed port 80 protocol http +# and request /some/path with content = \"a string\" +# then alert +# +# +## Check a network link status (up/down), link capacity changes, saturation +## and bandwidth usage. +# +# check network public with interface eth0 +# if failed link then alert +# if changed link then alert +# if saturation > 90% then alert +# if download > 10 MB/s then alert +# if total uploaded > 1 GB in last hour then alert +# +# +## Check custom program status output. +# +# check program myscript with path /usr/local/bin/myscript.sh +# if status != 0 then alert +# +# +############################################################################### +## Includes +############################################################################### +## +## It is possible to include additional configuration parts from other files or +## directories. +# +# include /etc/monit.d/* +# + include /home/arisnoveanu/TorQProdSupp/TorQDev/deploy/code/scripts/monit/*.cfg diff --git a/code/monit/monitrcexample b/code/scripts/monitrcexample similarity index 100% rename from code/monit/monitrcexample rename to code/scripts/monitrcexample diff --git a/code/monit/scripts/monitalert.cfg b/code/scripts/templates/monitalert.cfg similarity index 100% rename from code/monit/scripts/monitalert.cfg rename to code/scripts/templates/monitalert.cfg diff --git a/code/scripts/templates/monitconfig.cfg b/code/scripts/templates/monitconfig.cfg new file mode 100644 index 000000000..8166f1663 --- /dev/null +++ b/code/scripts/templates/monitconfig.cfg @@ -0,0 +1 @@ +${TORQHOME}/appconfig/process.csv ${TORQHOME}/torq.sh monitconfig.cfg diff --git a/code/monit/templates/monittemplate.txt b/code/scripts/templates/monittemplate.txt similarity index 56% rename from code/monit/templates/monittemplate.txt rename to code/scripts/templates/monittemplate.txt index b3e7ce480..b4d889611 100644 --- a/code/monit/templates/monittemplate.txt +++ b/code/scripts/templates/monittemplate.txt @@ -1,7 +1,8 @@ check process $procname matching \"$KDBSTACKID -proctype $proctype -procname $procname\" - start program = \"/bin/bash $startscript $procname\" + start program = \"/bin/bash $startstopsc start $procname\" with timeout 10 seconds - stop program = \"/bin/bash $stopscript $procname\" + stop program = \"/bin/bash $startstopsc stop $procname\" every \"* * * * *\" mode active + diff --git a/fill_templates.sh b/fill_templates.sh index 4048f5555..d7d397795 100644 --- a/fill_templates.sh +++ b/fill_templates.sh @@ -1,36 +1,30 @@ #!/bin/bash #FUNTION DECLARATION -createmonitrc(){ +createmonconfig(){ #function to read all processes from the processes.csv #and build the array - - cat $1|while read line; do - conf=($line) - procs=$(eval "echo \"${conf[0]}\"") - #echo $procs - output="$configs/${conf[1]}" - done - - proclist=`tail -n +2 /home/$USER/TorQProdSupp/TorQDev/TorQ-Finance-Starter-Pack/appconfig/process.csv | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` - echo "$proclist"|while read procs; do - array=($procs) - proctype=${array[0]} - procname=${array[1]} - "eval echo $2" - #eval "{$2}" >> test.txt + echo -n "Creating output..." + cat $1|while read line;do + conf=($line) + procs=$(eval "echo \"${conf[0]}\"") + startstopsc=$(eval "echo \"${conf[1]}\"") + output="$configs/${conf[2]}" + + proclist=`tail -n +2 ${procs} | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` + echo "$proclist"|while read procs;do + array=($procs) + proctype=${array[0]} + procname=${array[1]} + eval "echo $2" >> $output + echo "" >> $output + done done - + echo " DONE" } -#createoutput(){ - #function to create the output file - -#} - -#createmonitrc(){ - #function to create the monitrc file - +#createmonalert(){ + #} #SETTING ENVIRONMENT VARIABLES @@ -41,17 +35,15 @@ else fi eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables -templates="${TORQHOME}/code/monit/templates" #set temmplates folder -config="${TORQHOME}/code/monit/" #set configs folder +templates="${TORQHOME}/code/scripts/templates/" #set temmplates folder +configs="${TORQHOME}/code/scripts/monit/" #set configs folder monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf -mkdir -p $config #ensure output directory is created - -monittemplate="$(cat $templates/monittemplate.txt)" +mkdir -p $configs #ensure output directory is created -#echo ${monittemplate} +monittemplate="$(cat ${templates}monittemplate.txt)" -createmonitrc "$templates/monitconfig.cfg" "$monittemplate" +createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" "" #TESTING (to be removed) #echo ${TORQHOME} diff --git a/monit.sh b/monit.sh index 5f963a50d..957b44b87 100644 --- a/monit.sh +++ b/monit.sh @@ -11,5 +11,5 @@ if [ ! -f ${TORQHOME}/code/monit/monitrc ];then # eval ". $(dirname "$dirpath")/fill_templates.sh" fi -monit -c ${TORQHOME}/code/monit/monitrc $@ +monit -c ${TORQHOME}/code/scripts/monitrc $@ diff --git a/setenv.sh b/setenv.sh index dd7f972a0..cc8c6fc86 100644 --- a/setenv.sh +++ b/setenv.sh @@ -12,7 +12,7 @@ export KDBAPPCODE=${TORQAPPHOME}/code export KDBHDB=${TORQDATA}/hdb/database export KDBWDB=${TORQDATA}/wdbhdb -export KDBBASEPORT=6000 # set KDBBASEPORT to the default value for a TorQ Installation +export KDBBASEPORT=9500 # set KDBBASEPORT to the default value for a TorQ Installation export KDBSTACKID="-stackid ${KDBBASEPORT}" export TORQPROCESSES=${KDBAPPCONFIG}/process.csv # set TORQPROCESSES to the default process csv From 75bd90e34840c2274eeda9d038cc9c2e259597bb Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 10 Jul 2018 08:53:01 +0100 Subject: [PATCH 05/24] Removed unnecessary quotes after createmonconfig function --- fill_templates.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fill_templates.sh b/fill_templates.sh index d7d397795..6957be7d2 100644 --- a/fill_templates.sh +++ b/fill_templates.sh @@ -43,7 +43,7 @@ mkdir -p $configs monittemplate="$(cat ${templates}monittemplate.txt)" -createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" "" +createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" #TESTING (to be removed) #echo ${TORQHOME} From 355b13d1673a4e5b8266ee6fa6caba5516fa5c82 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 17 Jul 2018 12:25:40 +0100 Subject: [PATCH 06/24] Added new alerts in monitalert.cfg to monitor the host Added a new test alert in monitconfig.cfg which executest a test bash script Modified the e-mail format in /code/templates/monitrc to include more details about the event --- code/scripts/monitrcexample | 311 ----------------------- code/scripts/templates/monitalert.cfg | 17 +- code/scripts/{ => templates}/monitrc | 50 ++-- code/scripts/templates/monittemplate.txt | 4 +- code/scripts/templates/testmail.sh | 3 + config/settings/gateway.q | 2 +- fill_templates.sh | 41 +-- monit.sh | 6 +- setenv.sh | 4 +- 9 files changed, 84 insertions(+), 354 deletions(-) delete mode 100755 code/scripts/monitrcexample rename code/scripts/{ => templates}/monitrc (91%) create mode 100644 code/scripts/templates/testmail.sh mode change 100644 => 100755 fill_templates.sh mode change 100644 => 100755 monit.sh diff --git a/code/scripts/monitrcexample b/code/scripts/monitrcexample deleted file mode 100755 index 4f04772bb..000000000 --- a/code/scripts/monitrcexample +++ /dev/null @@ -1,311 +0,0 @@ -############################################################################### -## Monit control file -############################################################################### -## -## Comments begin with a '#' and extend through the end of the line. Keywords -## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. -## -## Below you will find examples of some frequently used statements. For -## information about the control file and a complete list of statements and -## options, please have a look in the Monit manual. -## -## -############################################################################### -## Global section -############################################################################### -## -## Start Monit in the background (run as a daemon): -# - set daemon 30 # check services at 30 seconds intervals -# with start delay 240 # optional: delay the first check by 4-minutes (by -# # default Monit check immediately after Monit start) -# -# -## Set syslog logging. If you want to log to a standalone log file instead, -## specify the full path to the log file -# -#set logfile $TORQMONIT/monit.log - -# -# -## Set the location of the Monit lock file which stores the process id of the -## running Monit instance. By default this file is stored in $HOME/.monit.pid -# -#set pidfile $TORQMONIT/monit.pid -# -## Set the location of the Monit id file which stores the unique id for the -## Monit instance. The id is generated and stored on first Monit start. By -## default the file is placed in $HOME/.monit.id. -# -# set idfile /var/.monit.id - #set idfile $TORQMONIT/.monit.id -# -## Set the location of the Monit state file which saves monitoring states -## on each cycle. By default the file is placed in $HOME/.monit.state. If -## the state file is stored on a persistent filesystem, Monit will recover -## the monitoring state across reboots. If it is on temporary filesystem, the -## state will be lost on reboot which may be convenient in some situations. -# - #set statefile $TORQMONIT/.monit.state -# set statefile /var/lib/monit/state -# -# - -## Set limits for various tests. The following example shows the default values: -## -# set limits { -# programOutput: 512 B, # check program's output truncate limit -# sendExpectBuffer: 256 B, # limit for send/expect protocol test -# fileContentBuffer: 512 B, # limit for file content test -# httpContentBuffer: 1 MB, # limit for HTTP content test -# networkTimeout: 5 seconds # timeout for network I/O -# programTimeout: 300 seconds # timeout for check program -# stopTimeout: 30 seconds # timeout for service stop -# startTimeout: 30 seconds # timeout for service start -# restartTimeout: 30 seconds # timeout for service restart -# } - -## Set global SSL options (just most common options showed, see manual for -## full list). -# -# set ssl { -# verify : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED) -# selfsigned : allow # allow self signed SSL certificates (reject by default) -# } -# -# -## Set the list of mail servers for alert delivery. Multiple servers may be -## specified using a comma separator. If the first mail server fails, Monit -# will use the second mail server in the list and so on. By default Monit uses -# port 25 - it is possible to override this with the PORT option. -# -# set mailserver mail.bar.baz, # primary mailserver -# backup.bar.baz port 10025, # backup mailserver on port 10025 -# localhost # fallback relay - #set mailserver smtp.office365.com port 587 - # username "auto@aquaq.co.uk" - # password "Circuit20" - # using ssl - # with timeout 30 seconds - - - #set alert tradefeedr@aquaq.co.uk -# -# -## By default Monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for later delivery retry, you can use the -## EVENTQUEUE statement. The base directory where undelivered alerts will be -## stored is specified by the BASEDIR option. You can limit the queue size -## by using the SLOTS option (if omitted, the queue is limited by space -## available in the back end filesystem). -# - #set eventqueue -# basedir /var/lib/monit/events # set the base directory where events will be stored - # basedir $TORQMONIT/events - # slots 100 # optionally limit the queue size -# -# -## Send status and events to M/Monit (for more informations about M/Monit -## see https://mmonit.com/). By default Monit registers credentials with -## M/Monit so M/Monit can smoothly communicate back to Monit and you don't -## have to register Monit credentials manually in M/Monit. It is possible to -## disable credential registration using the commented out option below. -## Though, if safety is a concern we recommend instead using https when -## communicating with M/Monit and send credentials encrypted. The password -## should be URL encoded if it contains URL-significant characters like -## \":\", \"?\", \"@\". -# -# set mmonit http://monit:monit@104.46.37.155:2810/collector -# # and register without credentials # Don't register credentials -# -# -## Monit by default uses the following format for alerts if the the mail-format -## statement is missing:: -## --8<-- -## set mail-format { -## from: Monit -## subject: monit alert -- $EVENT $SERVICE -## message: $EVENT Service $SERVICE -## Date: $DATE -## Action: $ACTION -## Host: $HOST -## Description: $DESCRIPTION -## -## Your faithful employee, -## Monit -## } -## --8<-- - # set mail-format { - # from: auto@aquaq.co.uk - # subject: [$HOST] monit alert -- $EVENT $SERVICE} -## -## You can override this message format or parts of it, such as subject -## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. -## are expanded at runtime. For example, to override the sender, use: -# -# set mail-format { from: monit@foo.bar } -# -# -## You can set alert recipients whom will receive alerts if/when a -## service defined in this file has errors. Alerts may be restricted on -## events by using a filter as in the second example below. -# -# set alert sysadm@foo.bar # receive all alerts -# -## Do not alert when Monit starts, stops or performs a user initiated action. -## This filter is recommended to avoid getting alerts for trivial cases. -# -# set alert your-name@your.domain not on { instance, action } -# -# -## Monit has an embedded HTTP interface which can be used to view status of -## services monitored and manage services from a web interface. The HTTP -## interface is also required if you want to issue Monit commands from the -## command line, such as 'monit status' or 'monit restart service' The reason -## for this is that the Monit client uses the HTTP interface to send these -## commands to a running Monit daemon. See the Monit Wiki if you want to -## enable SSL for the HTTP interface. -# -#set httpd port 2812 and -# use address localhost # only accept connection from localhost -# allow localhost # allow localhost to connect to the server and - # allow admin:monit # require user 'admin' with password 'monit' - -############################################################################### -## Services -############################################################################### -## -## Check general system resources such as load average, cpu and memory -## usage. Each test specifies a resource, conditions and the action to be -## performed should a test fail. -# -# check system $HOST -# if loadavg (1min) > 4 then alert -# if loadavg (5min) > 2 then alert -# if cpu usage > 95% for 10 cycles then alert -# if memory usage > 75% then alert -# if swap usage > 25% then alert -# -# -## Check if a file exists, checksum, permissions, uid and gid. In addition -## to alert recipients in the global section, customized alert can be sent to -## additional recipients by specifying a local alert handler. The service may -## be grouped using the GROUP option. More than one group can be specified by -## repeating the 'group name' statement. -# -# check file apache_bin with path /usr/local/apache/bin/httpd -# if failed checksum and -# expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor -# if failed permission 755 then unmonitor -# if failed uid root then unmonitor -# if failed gid root then unmonitor -# alert security@foo.bar on { -# checksum, permission, uid, gid, unmonitor -# } with the mail-format { subject: Alarm! } -# group server -# -# -## Check that a process is running, in this case Apache, and that it respond -## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory, -## and number of children. If the process is not running, Monit will restart -## it by default. In case the service is restarted very often and the -## problem remains, it is possible to disable monitoring using the TIMEOUT -## statement. This service depends on another service (apache_bin) which -## is defined above. -# -# check process apache with pidfile /usr/local/apache/logs/httpd.pid -# start program = \"/etc/init.d/httpd start\" with timeout 60 seconds -# stop program = \"/etc/init.d/httpd stop\" -# if cpu > 60% for 2 cycles then alert -# if cpu > 80% for 5 cycles then restart -# if totalmem > 200.0 MB for 5 cycles then restart -# if children > 250 then restart -# if loadavg(5min) greater than 10 for 8 cycles then stop -# if failed host www.tildeslash.com port 80 protocol http -# and request \"/somefile.html\" -# then restart -# if failed port 443 protocol https with timeout 15 seconds then restart -# if 3 restarts within 5 cycles then unmonitor -# depends on apache_bin -# group server -# -# -## Check filesystem permissions, uid, gid, space and inode usage. Other services, -## such as databases, may depend on this resource and an automatically graceful -## stop may be cascaded to them before the filesystem will become full and data -## lost. -# -# check filesystem datafs with path /dev/sdb1 -# start program = \"/bin/mount /data\" -# stop program = \"/bin/umount /data\" -# if failed permission 660 then unmonitor -# if failed uid root then unmonitor -# if failed gid disk then unmonitor -# if space usage > 80% for 5 times within 15 cycles then alert -# if space usage > 99% then stop -# if inode usage > 30000 then alert -# if inode usage > 99% then stop -# group server -# -# -## Check a file's timestamp. In this example, we test if a file is older -## than 15 minutes and assume something is wrong if its not updated. Also, -## if the file size exceed a given limit, execute a script -# -# check file database with path /data/mydatabase.db -# if failed permission 700 then alert -# if failed uid data then alert -# if failed gid data then alert -# if timestamp > 15 minutes then alert -# if size > 100 MB then exec \"/my/cleanup/script\" as uid dba and gid dba -# -# -## Check directory permission, uid and gid. An event is triggered if the -## directory does not belong to the user with uid 0 and gid 0. In addition, -## the permissions have to match the octal description of 755 (see chmod(1)). -# -# check directory bin with path /bin -# if failed permission 755 then unmonitor -# if failed uid 0 then unmonitor -# if failed gid 0 then unmonitor -# -# -## Check a remote host availability by issuing a ping test and check the -## content of a response from a web server. Up to three pings are sent and -## connection to a port and an application level network check is performed. -# -# check host myserver with address 192.168.1.1 -# if failed ping then alert -# if failed port 3306 protocol mysql with timeout 15 seconds then alert -# if failed port 80 protocol http -# and request /some/path with content = \"a string\" -# then alert -# -# -## Check a network link status (up/down), link capacity changes, saturation -## and bandwidth usage. -# -# check network public with interface eth0 -# if failed link then alert -# if changed link then alert -# if saturation > 90% then alert -# if download > 10 MB/s then alert -# if total uploaded > 1 GB in last hour then alert -# -# -## Check custom program status output. -# -# check program myscript with path /usr/local/bin/myscript.sh -# if status != 0 then alert -# -# -############################################################################### -## Includes -############################################################################### -## -## It is possible to include additional configuration parts from other files or -## directories. -# -# include /etc/monit.d/* -# -# include $TORQBLKTREE/scripts/monit/*.cfg diff --git a/code/scripts/templates/monitalert.cfg b/code/scripts/templates/monitalert.cfg index b294ccee3..b93f6ccde 100644 --- a/code/scripts/templates/monitalert.cfg +++ b/code/scripts/templates/monitalert.cfg @@ -1 +1,16 @@ -#text +check system homer.aquaq.co.uk + if loadavg (5min) > 3 for 4 cycles then alert + if loadavg (15min) > 1 for 4 cycles then alert + if memory usage > 80% for 4 cycles then alert + if swap usage > 20% for 4 cycles then alert + if cpu usage (system) > 20% for 4 cycles then alert + +check host google.com with address gwadawdawdwad.com + if failed url http://gawdawdawdaw.com + then alert + +check file alerttest with path /.nonexistent + alert rdanutalexandru@gmail.com with reminder on 500 cycles + +check file alerttest2 with path /.nonexistent + if does not exist then exec "/home/arisnoveanu/TorQProdSupp/TorQDev/deploy/code/scripts/templates/testmail.sh" diff --git a/code/scripts/monitrc b/code/scripts/templates/monitrc similarity index 91% rename from code/scripts/monitrc rename to code/scripts/templates/monitrc index f681d4254..9e7e5bea0 100755 --- a/code/scripts/monitrc +++ b/code/scripts/templates/monitrc @@ -16,15 +16,15 @@ ## ## Start Monit in the background (run as a daemon): # - set daemon 30 # check services at 30 seconds intervals -# with start delay 240 # optional: delay the first check by 4-minutes (by + set daemon 30 # check services at 30 seconds intervals + #with start delay 240 # optional: delay the first check by 4-minutes (by # # default Monit check immediately after Monit start) # # ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # -#set logfile $TORQMONIT/monit.log + set logfile $TORQMONIT/monit.log # # @@ -46,8 +46,8 @@ ## the monitoring state across reboots. If it is on temporary filesystem, the ## state will be lost on reboot which may be convenient in some situations. # - #set statefile $TORQMONIT/.monit.state -# set statefile /var/lib/monit/state + set statefile $TORQMONIT/.monit.state + #set statefile /var/lib/monit/state # # @@ -82,14 +82,13 @@ # set mailserver mail.bar.baz, # primary mailserver # backup.bar.baz port 10025, # backup mailserver on port 10025 # localhost # fallback relay - #set mailserver smtp.office365.com port 587 - # username "auto@aquaq.co.uk" - # password "Circuit20" - # using ssl - # with timeout 30 seconds + set mailserver smtp.gmail.com port 587 + username "\"torqmonit@gmail.com\"" password "\"torqmonit2018\"" + using ssl + with timeout 30 seconds - #set alert tradefeedr@aquaq.co.uk + set alert rdanutalexandru@gmail.com # # ## By default Monit will drop alert events if no mail servers are available. @@ -99,10 +98,10 @@ ## by using the SLOTS option (if omitted, the queue is limited by space ## available in the back end filesystem). # - #set eventqueue + set eventqueue # basedir /var/lib/monit/events # set the base directory where events will be stored - # basedir $TORQMONIT/events - # slots 100 # optionally limit the queue size + basedir $TORQMONIT/events + slots 100 # optionally limit the queue size # # ## Send status and events to M/Monit (for more informations about M/Monit @@ -135,9 +134,20 @@ ## Monit ## } ## --8<-- - # set mail-format { - # from: auto@aquaq.co.uk - # subject: [$HOST] monit alert -- $EVENT $SERVICE} + set mail-format { + from: torqmonit@gmail.com + subject: [\$SERVICE] monit alert -- \$EVENT at \$DATE + message: Monit Report: + ACTION: \$ACTION + SERVICE: \$SERVICE + DATE: \$DATE + HOST: \$HOST + DESCRIPTION: \$DESCRIPTION + + Powered by Monit + + This message has been generated automatically! + } ## ## You can override this message format or parts of it, such as subject ## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. @@ -166,10 +176,10 @@ ## commands to a running Monit daemon. See the Monit Wiki if you want to ## enable SSL for the HTTP interface. # -#set httpd port 2812 and + set httpd port 2812 and # use address localhost # only accept connection from localhost # allow localhost # allow localhost to connect to the server and - # allow admin:monit # require user 'admin' with password 'monit' + allow admin:monit # require user 'admin' with password 'monit' ############################################################################### ## Services @@ -308,4 +318,4 @@ # # include /etc/monit.d/* # - include /home/arisnoveanu/TorQProdSupp/TorQDev/deploy/code/scripts/monit/*.cfg + include ${TORQHOME}/code/scripts/monit/*.cfg diff --git a/code/scripts/templates/monittemplate.txt b/code/scripts/templates/monittemplate.txt index b4d889611..731eae405 100644 --- a/code/scripts/templates/monittemplate.txt +++ b/code/scripts/templates/monittemplate.txt @@ -1,6 +1,6 @@ check process $procname - matching \"$KDBSTACKID -proctype $proctype -procname $procname\" - start program = \"/bin/bash $startstopsc start $procname\" + matching \"$KDBBASEPORT -proctype $proctype -procname $procname\" + start program = \"/bin/bash -c '$startstopsc start $procname'\" with timeout 10 seconds stop program = \"/bin/bash $startstopsc stop $procname\" every \"* * * * *\" diff --git a/code/scripts/templates/testmail.sh b/code/scripts/templates/testmail.sh new file mode 100644 index 000000000..cd462bed3 --- /dev/null +++ b/code/scripts/templates/testmail.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +mail.mailutils -s "TORQMONIT" rdanutalexandru@gmail.com <<< "Hi! Monit here! Process Failed" diff --git a/config/settings/gateway.q b/config/settings/gateway.q index 2f7375671..4b092b310 100644 --- a/config/settings/gateway.q +++ b/config/settings/gateway.q @@ -11,5 +11,5 @@ clearinactivetime:0D01:00 // the time to keep inactive handle data // Server connection details \d .servers -CONNECTIONS:`rdb`hdb // list of connections to make at start up +CONNECTIONS:`rdb`hdb`wdb // list of connections to make at start up RETRY:0D00:01 // period on which to retry dead connections. If 0, no reconnection attempts diff --git a/fill_templates.sh b/fill_templates.sh old mode 100644 new mode 100755 index 6957be7d2..ea7f3cb0a --- a/fill_templates.sh +++ b/fill_templates.sh @@ -1,6 +1,5 @@ #!/bin/bash - -#FUNTION DECLARATION +#FUNTION DECLARATION ############################################################################### createmonconfig(){ #function to read all processes from the processes.csv #and build the array @@ -23,28 +22,38 @@ createmonconfig(){ echo " DONE" } -#createmonalert(){ +createmonalert(){ + if [ -f ${configs}monitalert.cfg ];then + echo -n "Deleting monitalert.cfg..." + rm ${configs}monitalert.cfg + echo " DONE" + fi -#} + echo -n "Copying monitalert from ${templates}..." + cp ${templates}monitalert.cfg ${configs} + echo " DONE" +} -#SETTING ENVIRONMENT VARIABLES +#SETTING DEFAULT ENVIRONMENT VARIABLES ############################################################# if [ "-bash" = $0 ]; then dirpath="${BASH_SOURCE[0]}" else dirpath="$0" fi - -eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables -templates="${TORQHOME}/code/scripts/templates/" #set temmplates folder -configs="${TORQHOME}/code/scripts/monit/" #set configs folder -monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf - -mkdir -p $configs #ensure output directory is created - + +eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables +templates="${TORQHOME}/code/scripts/templates/" #set temmplates folder +configs="${TORQHOME}/code/scripts/monit/" #set configs folder +monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf monittemplate="$(cat ${templates}monittemplate.txt)" +mkdir -p $configs #creating the output directory createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" +createmonalert + +echo -n "Creating monitrc..." +controltemplate="$(cat ${templates}monitrc)" +eval "echo \"${controltemplate}\"" > ${monit_control} +chmod 700 ${monit_control} +echo " DONE" -#TESTING (to be removed) -#echo ${TORQHOME} -#echo $templates \ No newline at end of file diff --git a/monit.sh b/monit.sh old mode 100644 new mode 100755 index 957b44b87..a67b70170 --- a/monit.sh +++ b/monit.sh @@ -1,4 +1,6 @@ #!/bin/sh +#SETTING DEFAULT VALUES ############################################################################ + if [ "-bash" = $0 ]; then dirpath="${BASH_SOURCE[0]}" else @@ -7,9 +9,9 @@ fi eval ". $(dirname "$dirpath")/setenv.sh" -if [ ! -f ${TORQHOME}/code/monit/monitrc ];then # run fill_templates.sh if monitrc is no present +if [ ! -f ${TORQHOME}/config/monitrc ];then # run fill_templates.sh if monitrc is no present eval ". $(dirname "$dirpath")/fill_templates.sh" fi -monit -c ${TORQHOME}/code/scripts/monitrc $@ +monit -c ${TORQHOME}/config/monitrc $@ diff --git a/setenv.sh b/setenv.sh index cc8c6fc86..c704acba4 100644 --- a/setenv.sh +++ b/setenv.sh @@ -12,8 +12,10 @@ export KDBAPPCODE=${TORQAPPHOME}/code export KDBHDB=${TORQDATA}/hdb/database export KDBWDB=${TORQDATA}/wdbhdb -export KDBBASEPORT=9500 # set KDBBASEPORT to the default value for a TorQ Installation +export KDBBASEPORT=6000 # set KDBBASEPORT to the default value for a TorQ Installation export KDBSTACKID="-stackid ${KDBBASEPORT}" export TORQPROCESSES=${KDBAPPCONFIG}/process.csv # set TORQPROCESSES to the default process csv export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KDBLIB/l32 + +export TORQMONIT=${TORQHOME}/logs/monit # set the folder for monit outputs From ee1fc5cebf180f2db9da0f3d3fc88d310c509631 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 17 Jul 2018 17:38:59 +0100 Subject: [PATCH 07/24] Attempted to add email alerts using mail.mailutils --- code/scripts/templates/testmail.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/scripts/templates/testmail.sh b/code/scripts/templates/testmail.sh index cd462bed3..b03248bcf 100644 --- a/code/scripts/templates/testmail.sh +++ b/code/scripts/templates/testmail.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/bash -mail.mailutils -s "TORQMONIT" rdanutalexandru@gmail.com <<< "Hi! Monit here! Process Failed" +echo -e "TORQMONIT\r\rThis is an automated message\r\rProcess SERVICE is down"|mail.mailutils -s \"SUBJECT\" rdanutalexandrugmail.com From dc0b17d44812a3013b9867b175c10454873fb85b Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 7 Aug 2018 16:48:48 +0100 Subject: [PATCH 08/24] Added function checkst in monit.sh and fill_templates.sh to facilitate status output Modified torq.sh to setenv using a BASEDIR defined in the .bashrc_torq Enabled the log file output in monitrc Modified monit.sh to create a .bashrc_torq which holds the environment variables necessary for starting torq from root --- code/scripts/templates/monitalert.cfg | 2 - code/scripts/templates/monitrc | 6 +-- code/scripts/templates/monittemplate.txt | 7 ++-- fill_templates.sh | 38 +++++++++++++----- monit.sh | 50 ++++++++++++++++++++++++ setenv.sh | 4 +- torq.sh | 2 +- 7 files changed, 88 insertions(+), 21 deletions(-) diff --git a/code/scripts/templates/monitalert.cfg b/code/scripts/templates/monitalert.cfg index b93f6ccde..4061ff50e 100644 --- a/code/scripts/templates/monitalert.cfg +++ b/code/scripts/templates/monitalert.cfg @@ -12,5 +12,3 @@ check host google.com with address gwadawdawdwad.com check file alerttest with path /.nonexistent alert rdanutalexandru@gmail.com with reminder on 500 cycles -check file alerttest2 with path /.nonexistent - if does not exist then exec "/home/arisnoveanu/TorQProdSupp/TorQDev/deploy/code/scripts/templates/testmail.sh" diff --git a/code/scripts/templates/monitrc b/code/scripts/templates/monitrc index 9e7e5bea0..664501c19 100755 --- a/code/scripts/templates/monitrc +++ b/code/scripts/templates/monitrc @@ -24,7 +24,7 @@ ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # - set logfile $TORQMONIT/monit.log + set logfile $TORQHOME/monit.log # # @@ -46,7 +46,7 @@ ## the monitoring state across reboots. If it is on temporary filesystem, the ## state will be lost on reboot which may be convenient in some situations. # - set statefile $TORQMONIT/.monit.state + #set statefile $TORQMONIT/.monit.state #set statefile /var/lib/monit/state # # @@ -176,7 +176,7 @@ ## commands to a running Monit daemon. See the Monit Wiki if you want to ## enable SSL for the HTTP interface. # - set httpd port 2812 and + set httpd port 11000 and # use address localhost # only accept connection from localhost # allow localhost # allow localhost to connect to the server and allow admin:monit # require user 'admin' with password 'monit' diff --git a/code/scripts/templates/monittemplate.txt b/code/scripts/templates/monittemplate.txt index 731eae405..a0fe13022 100644 --- a/code/scripts/templates/monittemplate.txt +++ b/code/scripts/templates/monittemplate.txt @@ -1,8 +1,7 @@ check process $procname - matching \"$KDBBASEPORT -proctype $proctype -procname $procname\" + matching \"$KDBBASEPORT -proctype $proctype -procname $procname\" start program = \"/bin/bash -c '$startstopsc start $procname'\" - with timeout 10 seconds - stop program = \"/bin/bash $startstopsc stop $procname\" + with timeout 10 seconds + stop program = \"/bin/bash -c '$startstopsc stop $procname'\" every \"* * * * *\" mode active - diff --git a/fill_templates.sh b/fill_templates.sh index ea7f3cb0a..126e414ed 100755 --- a/fill_templates.sh +++ b/fill_templates.sh @@ -1,14 +1,37 @@ #!/bin/bash #FUNTION DECLARATION ############################################################################### + +checkst(){ +#function to check if file exists +case $3 in + exist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;32mOK\033[0m ] $2" + else + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + fi + ;; + nexist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + else + echo -e "[ \033[01;32mOK\033[0m ] $2" + fi + ;; + *) + echo "Not yet implemented" + ;; +esac +} + createmonconfig(){ #function to read all processes from the processes.csv #and build the array - echo -n "Creating output..." cat $1|while read line;do conf=($line) procs=$(eval "echo \"${conf[0]}\"") startstopsc=$(eval "echo \"${conf[1]}\"") - output="$configs/${conf[2]}" + output="$configs${conf[2]}" proclist=`tail -n +2 ${procs} | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` echo "$proclist"|while read procs;do @@ -19,19 +42,17 @@ createmonconfig(){ echo "" >> $output done done - echo " DONE" + checkst "$configs${conf[2]}monitconfig.cfg" "Output file created" "exist" } createmonalert(){ if [ -f ${configs}monitalert.cfg ];then - echo -n "Deleting monitalert.cfg..." rm ${configs}monitalert.cfg - echo " DONE" + checkst "${configs}monitalert.cfg" "Deleting monitalert.cfg" "nexist" fi - echo -n "Copying monitalert from ${templates}..." cp ${templates}monitalert.cfg ${configs} - echo " DONE" + checkst "${configs}monitalert.cfg" "Copying monitalert from ${templates}" "exist" } #SETTING DEFAULT ENVIRONMENT VARIABLES ############################################################# @@ -51,9 +72,8 @@ mkdir -p $configs createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" createmonalert -echo -n "Creating monitrc..." controltemplate="$(cat ${templates}monitrc)" eval "echo \"${controltemplate}\"" > ${monit_control} chmod 700 ${monit_control} -echo " DONE" +checkst "$monit_control" "Creating monitrc" "exist" diff --git a/monit.sh b/monit.sh index a67b70170..4802a29b6 100755 --- a/monit.sh +++ b/monit.sh @@ -1,6 +1,56 @@ #!/bin/sh #SETTING DEFAULT VALUES ############################################################################ +checkst(){ +#function to check if file exists +case $3 in + exist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;32mOK\033[0m ] $2" + else + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + fi + ;; + nexist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + else + echo -e "[ \033[01;32mOK\033[0m ] $2" + fi + ;; + npres) + if [[ $4 -eq `eval "$1"` ]]; then + echo -e "[ \033[01;32mOK\033[0m ] $2" + else + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + fi + ;; + *) + echo "Not yet implemented" + ;; +esac +} + +if [[ ! -e ~/.bashrc_torq ]]; then + touch $HOME/.bashrc_torq + checkst "$HOME/.bashrc_torq" "Creating .bashrc_torq" "exist" + echo "export TORQHOME=${PWD}" >> $HOME/.bashrc_torq + echo "export BASEDIR=${PWD}" >> $HOME/.bashrc_torq + checkst "cat $HOME/.bashrc_torq | grep 'TORQHOME\|BASEDIR'|wc -l" "Environment variables are set" "npres" "2" + echo "source $HOME/.bashrc_torq" >> $HOME/.bashrc + source $HOME/.bashrc + checkst "echo $TORQHOME|wc -l" "TORQHOME present" "npres" "1" + checkst "echo $BASEDIR|wc -l" "BASEDIR present" "npres" "1" +else + echo "export TORQHOME=${PWD}" >> $HOME/.bashrc_torq + echo "export BASEDIR=${PWD}" >> $HOME/.bashrc_torq + checkst "cat $HOME/.bashrc_torq | grep 'TORQHOME\|BASEDIR'|wc -l" "Environment variables are set" "npres" "2" + echo "source $HOME/.bashrc_torq" >> $HOME/.bashrc + source $HOME/.bashrc + checkst "echo $TORQHOME|wc -l" "TORQHOME present" "npres" "1" + checkst "echo $BASEDIR|wc -l" "BASEDIR present" "npres" "1" +fi + if [ "-bash" = $0 ]; then dirpath="${BASH_SOURCE[0]}" else diff --git a/setenv.sh b/setenv.sh index c704acba4..77cf1251f 100644 --- a/setenv.sh +++ b/setenv.sh @@ -1,5 +1,5 @@ -export TORQHOME=${PWD} # if running the kdb+tick example, change these to full paths -export TORQDATA=${PWD} # some of the kdb+tick processes will change directory, and these will no longer be valid +export TORQHOME=${PWD} # if running the kdb+tick example, change these to full paths +export TORQDATA=${PWD} # some of the kdb+tick processes will change directory, and these will no longer be valid export TORQAPPHOME=${PWD} export KDBLOG=${TORQDATA}/logs diff --git a/torq.sh b/torq.sh index b34b062ae..b65064e9b 100755 --- a/torq.sh +++ b/torq.sh @@ -222,7 +222,7 @@ usage() { } if [[ -z $SETENV ]]; then - SETENV=${PWD}/setenv.sh; # set the environment if not predefined + SETENV=${BASEDIR}/setenv.sh; # set the environment if not predefined fi if [ -f $SETENV ]; then # check script exists From 9a8be8fc82398663d997c360a1ff998a24ab2a2e Mon Sep 17 00:00:00 2001 From: Aqualex Date: Wed, 8 Aug 2018 14:53:21 +0100 Subject: [PATCH 09/24] Replaced ${TORQHOME} with ${PWD} in templates/monitconfig.cfg and set the path to torq.sh and process.csv from fill_templates.sh Modified the way the path is set in templates/monitrc to include the full path not only the path relative to the current directory Modified e-mail settings in monitrc from ssl to tslv1 and fixed the issues with sending alert e-mails Added path to events folder in templates/monitrc Fixed e-mail format in templates/monitrc Changed port for HTTP interface Fixed path to torq.sh in fill_templates Modified monit.sh to run fill_templates to run if any of the necessary files are missing Modified fill_templates to check if the necessary files exist and create them only it they are missing Modified path to setenv.sh in torq.sh to be set based on the current directory Added small script in torq.sh, monit.sh to determine the path based on the current directory --- code/scripts/templates/monitconfig.cfg | 2 +- code/scripts/templates/monitrc | 24 ++++++------ fill_templates.sh | 39 ++++++++++++------- monit.sh | 54 +------------------------- torq.sh | 10 ++++- 5 files changed, 47 insertions(+), 82 deletions(-) diff --git a/code/scripts/templates/monitconfig.cfg b/code/scripts/templates/monitconfig.cfg index 8166f1663..14b411c5c 100644 --- a/code/scripts/templates/monitconfig.cfg +++ b/code/scripts/templates/monitconfig.cfg @@ -1 +1 @@ -${TORQHOME}/appconfig/process.csv ${TORQHOME}/torq.sh monitconfig.cfg +appconfig/process.csv torq.sh monitconfig.cfg diff --git a/code/scripts/templates/monitrc b/code/scripts/templates/monitrc index 664501c19..50b389047 100755 --- a/code/scripts/templates/monitrc +++ b/code/scripts/templates/monitrc @@ -24,7 +24,7 @@ ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # - set logfile $TORQHOME/monit.log + set logfile ${PWD}/monit.log # # @@ -46,7 +46,7 @@ ## the monitoring state across reboots. If it is on temporary filesystem, the ## state will be lost on reboot which may be convenient in some situations. # - #set statefile $TORQMONIT/.monit.state + set statefile ${PWD}/monit.state #set statefile /var/lib/monit/state # # @@ -84,7 +84,7 @@ # localhost # fallback relay set mailserver smtp.gmail.com port 587 username "\"torqmonit@gmail.com\"" password "\"torqmonit2018\"" - using ssl + using tlsv1 with timeout 30 seconds @@ -100,7 +100,7 @@ # set eventqueue # basedir /var/lib/monit/events # set the base directory where events will be stored - basedir $TORQMONIT/events + basedir ${PWD}/events slots 100 # optionally limit the queue size # # @@ -137,16 +137,16 @@ set mail-format { from: torqmonit@gmail.com subject: [\$SERVICE] monit alert -- \$EVENT at \$DATE - message: Monit Report: - ACTION: \$ACTION - SERVICE: \$SERVICE - DATE: \$DATE - HOST: \$HOST - DESCRIPTION: \$DESCRIPTION + message: Monit Report: +ACTION: \$ACTION +SERVICE: \$SERVICE +DATE: \$DATE +HOST: \$HOST +DESCRIPTION: \$DESCRIPTION - Powered by Monit +Powered by Monit - This message has been generated automatically! +This message has been generated automatically! } ## ## You can override this message format or parts of it, such as subject diff --git a/fill_templates.sh b/fill_templates.sh index 126e414ed..f5e0169fa 100755 --- a/fill_templates.sh +++ b/fill_templates.sh @@ -2,7 +2,7 @@ #FUNTION DECLARATION ############################################################################### checkst(){ -#function to check if file exists +#function to check if file exists case $3 in exist) if [[ -e $1 ]]; then @@ -29,30 +29,33 @@ createmonconfig(){ #and build the array cat $1|while read line;do conf=($line) - procs=$(eval "echo \"${conf[0]}\"") - startstopsc=$(eval "echo \"${conf[1]}\"") + procs=$(eval "echo \"${PWD}/${conf[0]}\"") + startstopsc=$(eval "echo \"${PWD}/${conf[1]}\"") + echo "$startstopsc" output="$configs${conf[2]}" - + proclist=`tail -n +2 ${procs} | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` echo "$proclist"|while read procs;do array=($procs) proctype=${array[0]} procname=${array[1]} - eval "echo $2" >> $output - echo "" >> $output + if [[ ! "$procname" == "killtick" ]]; then # exclude killtick from the list of monitored processes + eval "echo $2" >> $output + echo "" >> $output + fi done done - checkst "$configs${conf[2]}monitconfig.cfg" "Output file created" "exist" + checkst "$configs${conf[2]}monitconfig.cfg" "Output file created..." "exist" } createmonalert(){ if [ -f ${configs}monitalert.cfg ];then rm ${configs}monitalert.cfg - checkst "${configs}monitalert.cfg" "Deleting monitalert.cfg" "nexist" + checkst "${configs}monitalert.cfg" "Deleting monitalert.cfg..." "nexist" fi cp ${templates}monitalert.cfg ${configs} - checkst "${configs}monitalert.cfg" "Copying monitalert from ${templates}" "exist" + checkst "${configs}monitalert.cfg" "Copying monitalert from ${templates}..." "exist" } #SETTING DEFAULT ENVIRONMENT VARIABLES ############################################################# @@ -69,11 +72,17 @@ monit_control="${TORQHOME}/config/monitrc" monittemplate="$(cat ${templates}monittemplate.txt)" mkdir -p $configs #creating the output directory -createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" -createmonalert +if [ ! -f ${configs}/monitconfig.cfg ]; then + createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" +fi -controltemplate="$(cat ${templates}monitrc)" -eval "echo \"${controltemplate}\"" > ${monit_control} -chmod 700 ${monit_control} -checkst "$monit_control" "Creating monitrc" "exist" +if [ ! -f ${configs}/monitalert.cfg ]; then + createmonalert +fi +if [ ! -f ${monit_control} ]; then + controltemplate="$(cat ${templates}monitrc)" + eval "echo \"${controltemplate}\"" > ${monit_control} + chmod 700 ${monit_control} + checkst "$monit_control" "Creating monitrc" "exist" +fi diff --git a/monit.sh b/monit.sh index 4802a29b6..fdbb67e7b 100755 --- a/monit.sh +++ b/monit.sh @@ -1,56 +1,6 @@ #!/bin/sh #SETTING DEFAULT VALUES ############################################################################ -checkst(){ -#function to check if file exists -case $3 in - exist) - if [[ -e $1 ]]; then - echo -e "[ \033[01;32mOK\033[0m ] $2" - else - echo -e "[ \033[01;31mFAILED\033[0m ] $2" - fi - ;; - nexist) - if [[ -e $1 ]]; then - echo -e "[ \033[01;31mFAILED\033[0m ] $2" - else - echo -e "[ \033[01;32mOK\033[0m ] $2" - fi - ;; - npres) - if [[ $4 -eq `eval "$1"` ]]; then - echo -e "[ \033[01;32mOK\033[0m ] $2" - else - echo -e "[ \033[01;31mFAILED\033[0m ] $2" - fi - ;; - *) - echo "Not yet implemented" - ;; -esac -} - -if [[ ! -e ~/.bashrc_torq ]]; then - touch $HOME/.bashrc_torq - checkst "$HOME/.bashrc_torq" "Creating .bashrc_torq" "exist" - echo "export TORQHOME=${PWD}" >> $HOME/.bashrc_torq - echo "export BASEDIR=${PWD}" >> $HOME/.bashrc_torq - checkst "cat $HOME/.bashrc_torq | grep 'TORQHOME\|BASEDIR'|wc -l" "Environment variables are set" "npres" "2" - echo "source $HOME/.bashrc_torq" >> $HOME/.bashrc - source $HOME/.bashrc - checkst "echo $TORQHOME|wc -l" "TORQHOME present" "npres" "1" - checkst "echo $BASEDIR|wc -l" "BASEDIR present" "npres" "1" -else - echo "export TORQHOME=${PWD}" >> $HOME/.bashrc_torq - echo "export BASEDIR=${PWD}" >> $HOME/.bashrc_torq - checkst "cat $HOME/.bashrc_torq | grep 'TORQHOME\|BASEDIR'|wc -l" "Environment variables are set" "npres" "2" - echo "source $HOME/.bashrc_torq" >> $HOME/.bashrc - source $HOME/.bashrc - checkst "echo $TORQHOME|wc -l" "TORQHOME present" "npres" "1" - checkst "echo $BASEDIR|wc -l" "BASEDIR present" "npres" "1" -fi - if [ "-bash" = $0 ]; then dirpath="${BASH_SOURCE[0]}" else @@ -59,8 +9,8 @@ fi eval ". $(dirname "$dirpath")/setenv.sh" -if [ ! -f ${TORQHOME}/config/monitrc ];then # run fill_templates.sh if monitrc is no present - eval ". $(dirname "$dirpath")/fill_templates.sh" +if [ ! -f ${TORQHOME}/config/monitrc ] || [ ! -f ${TORQHOME}/code/scripts/monit/monitalert.cfg ] || [ ! -f ${TORQHOME}/code/scripts/monit/monitconfig.cfg ];then # run fill_templates.sh if monitrc is no present + eval ". $(dirname "$dirpath")/fill_templates.sh" fi monit -c ${TORQHOME}/config/monitrc $@ diff --git a/torq.sh b/torq.sh index b65064e9b..9bfa68666 100755 --- a/torq.sh +++ b/torq.sh @@ -221,8 +221,14 @@ usage() { exit 1 } -if [[ -z $SETENV ]]; then - SETENV=${BASEDIR}/setenv.sh; # set the environment if not predefined +if [ "-bash" = $0 ]; then + dirpath="${BASH_SOURCE[0]}" +else + dirpath="$0" +fi + +if [[ -z $SETENV ]]; then + SETENV=$(dirname $dirpath)/setenv.sh; # set the environment if not predefined fi if [ -f $SETENV ]; then # check script exists From 2dcc6c1547fe3ac421ee9cd35caafa6196d101db Mon Sep 17 00:00:00 2001 From: Aqualex Date: Wed, 8 Aug 2018 17:12:16 +0100 Subject: [PATCH 10/24] Modified setenv.sh to set the path for Torq direcotry based on the directory the script is executed from --- setenv.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/setenv.sh b/setenv.sh index 77cf1251f..1d780f802 100644 --- a/setenv.sh +++ b/setenv.sh @@ -1,6 +1,12 @@ -export TORQHOME=${PWD} # if running the kdb+tick example, change these to full paths -export TORQDATA=${PWD} # some of the kdb+tick processes will change directory, and these will no longer be valid -export TORQAPPHOME=${PWD} +if [ "-bash" = $0 ]; then + dirpath="${BASH_SOURCE[0]}" +else + dirpath="$0" +fi + +export TORQHOME=$(dirname $dirpath) # if running the kdb+tick example, change these to full paths +export TORQDATA=$(dirname $dirpath) # some of the kdb+tick processes will change directory, and these will no longer be valid +export TORQAPPHOME=$(dirname $dirpath) export KDBLOG=${TORQDATA}/logs export KDBHTML=${TORQHOME}/html From 1aeb20439673fa12d452602917e46e90dc64018b Mon Sep 17 00:00:00 2001 From: Aqualex Date: Wed, 8 Aug 2018 17:14:37 +0100 Subject: [PATCH 11/24] Fixed spacing for comment --- setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setenv.sh b/setenv.sh index 1d780f802..c8d29f523 100644 --- a/setenv.sh +++ b/setenv.sh @@ -18,7 +18,7 @@ export KDBAPPCODE=${TORQAPPHOME}/code export KDBHDB=${TORQDATA}/hdb/database export KDBWDB=${TORQDATA}/wdbhdb -export KDBBASEPORT=6000 # set KDBBASEPORT to the default value for a TorQ Installation +export KDBBASEPORT=6000 # set KDBBASEPORT to the default value for a TorQ Installation export KDBSTACKID="-stackid ${KDBBASEPORT}" export TORQPROCESSES=${KDBAPPCONFIG}/process.csv # set TORQPROCESSES to the default process csv From ff5d14e09171967de32e100e49ea4c2759e39618 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Wed, 8 Aug 2018 17:17:51 +0100 Subject: [PATCH 12/24] Removed unnecessary testmai.sh file --- code/scripts/templates/testmail.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 code/scripts/templates/testmail.sh diff --git a/code/scripts/templates/testmail.sh b/code/scripts/templates/testmail.sh deleted file mode 100644 index b03248bcf..000000000 --- a/code/scripts/templates/testmail.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo -e "TORQMONIT\r\rThis is an automated message\r\rProcess SERVICE is down"|mail.mailutils -s \"SUBJECT\" rdanutalexandrugmail.com From cb93dd258d9ae75f4d894e92a019cea832abc395 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Thu, 9 Aug 2018 08:37:12 +0100 Subject: [PATCH 13/24] Removed personal e-mail, username and password from the template monitrc Fixed format for e-mail template Removed personal e-mail from monitalert template --- code/scripts/templates/monitalert.cfg | 2 +- code/scripts/templates/monitrc | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/code/scripts/templates/monitalert.cfg b/code/scripts/templates/monitalert.cfg index 4061ff50e..f56ee75c4 100644 --- a/code/scripts/templates/monitalert.cfg +++ b/code/scripts/templates/monitalert.cfg @@ -10,5 +10,5 @@ check host google.com with address gwadawdawdwad.com then alert check file alerttest with path /.nonexistent - alert rdanutalexandru@gmail.com with reminder on 500 cycles + then alert diff --git a/code/scripts/templates/monitrc b/code/scripts/templates/monitrc index 50b389047..8e56eb13c 100755 --- a/code/scripts/templates/monitrc +++ b/code/scripts/templates/monitrc @@ -83,12 +83,12 @@ # backup.bar.baz port 10025, # backup mailserver on port 10025 # localhost # fallback relay set mailserver smtp.gmail.com port 587 - username "\"torqmonit@gmail.com\"" password "\"torqmonit2018\"" + username "USERNAME@MAILSERVICE.COM" password "PASSWORD" using tlsv1 with timeout 30 seconds - set alert rdanutalexandru@gmail.com + set alert EMAIL # # ## By default Monit will drop alert events if no mail servers are available. @@ -138,11 +138,11 @@ from: torqmonit@gmail.com subject: [\$SERVICE] monit alert -- \$EVENT at \$DATE message: Monit Report: -ACTION: \$ACTION -SERVICE: \$SERVICE -DATE: \$DATE -HOST: \$HOST -DESCRIPTION: \$DESCRIPTION +ACTION: \$ACTION +SERVICE: \$SERVICE +DATE: \$DATE +HOST: \$HOST +DESCRIPTION: \$DESCRIPTION Powered by Monit From d3e7f856d275e82132a209ad2eac94e232da9774 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Thu, 9 Aug 2018 17:34:34 +0100 Subject: [PATCH 14/24] Modified monit.sh to be more flexible with how the config files are created and how monit is started --- fill_templates.sh | 88 ----------- monit/bin/monit.sh | 139 ++++++++++++++++++ monit.sh => monit/bin/monit_old.sh | 0 .../templates/monitalert.cfg | 4 +- .../templates/monitconfig.cfg | 0 {code/scripts => monit}/templates/monitrc | 12 +- .../templates/monittemplate.txt | 0 7 files changed, 146 insertions(+), 97 deletions(-) delete mode 100755 fill_templates.sh create mode 100755 monit/bin/monit.sh rename monit.sh => monit/bin/monit_old.sh (100%) rename {code/scripts => monit}/templates/monitalert.cfg (86%) rename {code/scripts => monit}/templates/monitconfig.cfg (100%) rename {code/scripts => monit}/templates/monitrc (97%) rename {code/scripts => monit}/templates/monittemplate.txt (100%) diff --git a/fill_templates.sh b/fill_templates.sh deleted file mode 100755 index f5e0169fa..000000000 --- a/fill_templates.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -#FUNTION DECLARATION ############################################################################### - -checkst(){ -#function to check if file exists -case $3 in - exist) - if [[ -e $1 ]]; then - echo -e "[ \033[01;32mOK\033[0m ] $2" - else - echo -e "[ \033[01;31mFAILED\033[0m ] $2" - fi - ;; - nexist) - if [[ -e $1 ]]; then - echo -e "[ \033[01;31mFAILED\033[0m ] $2" - else - echo -e "[ \033[01;32mOK\033[0m ] $2" - fi - ;; - *) - echo "Not yet implemented" - ;; -esac -} - -createmonconfig(){ - #function to read all processes from the processes.csv - #and build the array - cat $1|while read line;do - conf=($line) - procs=$(eval "echo \"${PWD}/${conf[0]}\"") - startstopsc=$(eval "echo \"${PWD}/${conf[1]}\"") - echo "$startstopsc" - output="$configs${conf[2]}" - - proclist=`tail -n +2 ${procs} | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` - echo "$proclist"|while read procs;do - array=($procs) - proctype=${array[0]} - procname=${array[1]} - if [[ ! "$procname" == "killtick" ]]; then # exclude killtick from the list of monitored processes - eval "echo $2" >> $output - echo "" >> $output - fi - done - done - checkst "$configs${conf[2]}monitconfig.cfg" "Output file created..." "exist" -} - -createmonalert(){ - if [ -f ${configs}monitalert.cfg ];then - rm ${configs}monitalert.cfg - checkst "${configs}monitalert.cfg" "Deleting monitalert.cfg..." "nexist" - fi - - cp ${templates}monitalert.cfg ${configs} - checkst "${configs}monitalert.cfg" "Copying monitalert from ${templates}..." "exist" -} - -#SETTING DEFAULT ENVIRONMENT VARIABLES ############################################################# -if [ "-bash" = $0 ]; then - dirpath="${BASH_SOURCE[0]}" -else - dirpath="$0" -fi - -eval ". $(dirname "$dirpath")/setenv.sh" #set environment variables -templates="${TORQHOME}/code/scripts/templates/" #set temmplates folder -configs="${TORQHOME}/code/scripts/monit/" #set configs folder -monit_control="${TORQHOME}/config/monitrc" #set output file for main monit conf -monittemplate="$(cat ${templates}monittemplate.txt)" -mkdir -p $configs #creating the output directory - -if [ ! -f ${configs}/monitconfig.cfg ]; then - createmonconfig "${templates}monitconfig.cfg" "\"${monittemplate}\"" -fi - -if [ ! -f ${configs}/monitalert.cfg ]; then - createmonalert -fi - -if [ ! -f ${monit_control} ]; then - controltemplate="$(cat ${templates}monitrc)" - eval "echo \"${controltemplate}\"" > ${monit_control} - chmod 700 ${monit_control} - checkst "$monit_control" "Creating monitrc" "exist" -fi diff --git a/monit/bin/monit.sh b/monit/bin/monit.sh new file mode 100755 index 000000000..80d24b6be --- /dev/null +++ b/monit/bin/monit.sh @@ -0,0 +1,139 @@ +#!/bin/bash +#FUNTION DECLARATION ############################################################################### + + cd ../.. && export BASEDIR="${PWD}" && cd - > /dev/null; # set BASEDIR to the root of the torq directory + +checkst(){ +#function to check if file exists +case $3 in + exist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;32mOK\033[0m ] $2" + else + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + fi + ;; + nexist) + if [[ -e $1 ]]; then + echo -e "[ \033[01;31mFAILED\033[0m ] $2" + else + echo -e "[ \033[01;32mOK\033[0m ] $2" + fi + ;; + *) + echo "Not yet implemented" + ;; +esac +} + +createmonconfig(){ + #function to read all processes from the processes.csv + #and build the array + #generating the monitconfig.cfg + procs=$1 + startstopsc=$2 + procs="${procs:-${BASEDIR}/appconfig/process.csv}" # sets procs to default only if unset already + startstopsc="${startstopsc:-${BASEDIR}/torq.sh}" + output="$configs/monitconfig.cfg" + + proclist=`tail -n +2 ${procs} | awk -F "\"*,\"*" '{print $3 " " $4}'|cut -d" " -f1,2` + echo "$proclist"|while read procs;do + array=($procs) + proctype=${array[0]} + procname=${array[1]} + if [[ ! "$procname" == "killtick" ]];then # exclude killtick from the list of monitored processes + #eval "echo $2" >> $output + eval "echo \"${monittemplate}\"" >> $output + echo "" >> $output + fi + #done + done + checkst "$configs/monitconfig.cfg" "Output file created..." "exist" +} + +createmonalert(){ + #generating the monitalert file from template + if [ -f ${configs}/monitalert.cfg ];then + rm ${configs}/monitalert.cfg + checkst "${configs}/monitalert.cfg" "Deleting monitalert.cfg..." "nexist" + fi + + cp ${templates}/monitalert.cfg ${configs} + checkst "${configs}/monitalert.cfg" "Copying monitalert from ${templates}..." "exist" +} + +generate(){ + eval ". $BASEDIR/setenv.sh" # set environment variables + templates="${BASEDIR}/monit/templates" # set temmplates folder + configs="${BASEDIR}/monit/config" # set configs folder + monit_control="${BASEDIR}/monit/config/monitrc" # set output file for main monit conf + monittemplate="$(cat ${templates}/monittemplate.txt)" + mkdir -p $configs + + case $1 in + monitalert) + if [ ! -f ${configs}/monitalert.cfg ];then + createmonalert + fi + ;; + monitconfig) + if [ ! -f ${configs}/monitconfig.cfg ];then + createmonconfig "$2" "$3" + fi + ;; + monitrc) + #I need some way to define the location of the monitrc template + if [ ! -f ${monit_control} ];then + if [ -z $2 ]; then + controltemplate="$(cat ${templates}/monitrc)" + else + controltemplate="$(cat $2)" + fi + eval "echo \"${controltemplate}\"" > ${monit_control} + chmod 700 ${monit_control} + checkst "$monit_control" "Creating monitrc" "exist" + fi + ;; + all) + #create monitalert + if [ ! -f ${configs}/monitalert.cfg ];then + createmonalert + fi + + #create monitconfig + if [ ! -f ${configs}/monitconfig.cfg ];then + createmonconfig "$2" "$3" + fi + + #create monitrc + if [ ! -f ${monit_control} ];then + if [ -z $4 ]; then + controltemplate="$(cat ${templates}/monitrc)" + else + controltemplate="$(cat $4)" + fi + eval "echo \"${controltemplate}\"" > ${monit_control} + chmod 700 ${monit_control} + checkst "$monit_control" "Creating monitrc" "exist" + fi + ;; + *) + echo "Not yet implemented" + ;; + esac + } + + start(){ + #this function just starts monit and specifies the location of the monitrc + if [ -z $1 ];then + monit -c ${BASEDIR}/monit/config/monitrc + else + monit -c $1 + fi + } + + usage(){ + echo "USAGE FUNCTION - Not yet implemented" + } + +"$@" diff --git a/monit.sh b/monit/bin/monit_old.sh similarity index 100% rename from monit.sh rename to monit/bin/monit_old.sh diff --git a/code/scripts/templates/monitalert.cfg b/monit/templates/monitalert.cfg similarity index 86% rename from code/scripts/templates/monitalert.cfg rename to monit/templates/monitalert.cfg index f56ee75c4..8c9a559e2 100644 --- a/code/scripts/templates/monitalert.cfg +++ b/monit/templates/monitalert.cfg @@ -8,7 +8,5 @@ check system homer.aquaq.co.uk check host google.com with address gwadawdawdwad.com if failed url http://gawdawdawdaw.com then alert - -check file alerttest with path /.nonexistent - then alert + diff --git a/code/scripts/templates/monitconfig.cfg b/monit/templates/monitconfig.cfg similarity index 100% rename from code/scripts/templates/monitconfig.cfg rename to monit/templates/monitconfig.cfg diff --git a/code/scripts/templates/monitrc b/monit/templates/monitrc similarity index 97% rename from code/scripts/templates/monitrc rename to monit/templates/monitrc index 8e56eb13c..56817a8c1 100755 --- a/code/scripts/templates/monitrc +++ b/monit/templates/monitrc @@ -24,7 +24,7 @@ ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # - set logfile ${PWD}/monit.log + set logfile ${PWD}/monit/logs/monit.log # # @@ -46,7 +46,7 @@ ## the monitoring state across reboots. If it is on temporary filesystem, the ## state will be lost on reboot which may be convenient in some situations. # - set statefile ${PWD}/monit.state + set statefile ${PWD}/monit/logs/monit.state #set statefile /var/lib/monit/state # # @@ -83,12 +83,12 @@ # backup.bar.baz port 10025, # backup mailserver on port 10025 # localhost # fallback relay set mailserver smtp.gmail.com port 587 - username "USERNAME@MAILSERVICE.COM" password "PASSWORD" + username "\"torqmonit@gmail.com\"" password "\"torqmonit2018\"" using tlsv1 with timeout 30 seconds - set alert EMAIL + set alert rdanutalexandru@gmail.com # # ## By default Monit will drop alert events if no mail servers are available. @@ -100,7 +100,7 @@ # set eventqueue # basedir /var/lib/monit/events # set the base directory where events will be stored - basedir ${PWD}/events + basedir ${PWD}/monit/events slots 100 # optionally limit the queue size # # @@ -318,4 +318,4 @@ This message has been generated automatically! # # include /etc/monit.d/* # - include ${TORQHOME}/code/scripts/monit/*.cfg + include ${PWD}/monit/config/*.cfg diff --git a/code/scripts/templates/monittemplate.txt b/monit/templates/monittemplate.txt similarity index 100% rename from code/scripts/templates/monittemplate.txt rename to monit/templates/monittemplate.txt From 45ea5a3e886db9685cd8dd3a32a4c6009c715a4f Mon Sep 17 00:00:00 2001 From: Aqualex Date: Fri, 10 Aug 2018 12:42:44 +0100 Subject: [PATCH 15/24] Removed old monit.sh Added usage function in monit.sh Modified monitrc to used the BASEDIR instead of PWD --- monit/bin/monit.sh | 29 ++++++++++++++++++++++++++--- monit/bin/monit_old.sh | 17 ----------------- monit/templates/monitrc | 8 ++++---- 3 files changed, 30 insertions(+), 24 deletions(-) delete mode 100755 monit/bin/monit_old.sh diff --git a/monit/bin/monit.sh b/monit/bin/monit.sh index 80d24b6be..9319d047c 100755 --- a/monit/bin/monit.sh +++ b/monit/bin/monit.sh @@ -46,7 +46,6 @@ createmonconfig(){ eval "echo \"${monittemplate}\"" >> $output echo "" >> $output fi - #done done checkst "$configs/monitconfig.cfg" "Output file created..." "exist" } @@ -126,14 +125,38 @@ generate(){ start(){ #this function just starts monit and specifies the location of the monitrc if [ -z $1 ];then + echo "Argument not provided monit will default to the following monitrc file: ${BASEDIR}/monit/config/monitrc" monit -c ${BASEDIR}/monit/config/monitrc - else + else monit -c $1 fi } usage(){ - echo "USAGE FUNCTION - Not yet implemented" + echo "" + echo "NOTE: if any of the arguments are missing the default locations will be used" + echo "" + echo "----------------------------------------------------------------------------" + printf "%-20s | %-30s | %-30s\n" "FILE" "DEFAULT TEMPLATE PATH" "DEFAULT CONFIG PATH" + echo "----------------------------------------------------------------------------" + printf "%-20s | %-30s | %-30s\n" "monitconfig.cfg" "deploy/monit/templates" "deploy/monit/config" + printf "%-20s | %-30s | %-30s\n" "monitalert.cfg" "deploy/monit/templates" "deploy/monit/config" + printf "%-20s | %-30s | %-30s\n" "monitrc" "deploy/monit/templates" "deploy/monit/config" + printf "%-20s | %-30s | %-30s\n" "monit.log" "NA" "deploy/monit/logs" + printf "%-20s | %-30s | %-30s\n" "monit.state" "NA" "deploy/monit/logs" + echo "----------------------------------------------------------------------------" + echo "" + echo "" + echo "----------------------------------------------------------------------------------------------------------------------------------------------" + printf "%-10s | %-15s | %-40s | %-75s\n" "FUNCTION" "OPTION" "COMMENTS" "ARGUMENTS" + echo "----------------------------------------------------------------------------------------------------------------------------------------------" + printf "%-10s | %-15s | %-40s | %-75s\n" "generate" "monitalert" "generates the monitalert.cfg" "no arguments" + printf "%-10s | %-15s | %-40s | %-75s\n" "generate" "monitconfig" "generates the monitconfig.cfg" "\"\" & \"\"" + printf "%-10s | %-15s | %-40s | %-75s\n" "generate" "monitrc" "generates the monitrc.cfg" "\"\"" + printf "%-10s | %-15s | %-40s | %-75s\n" "generate" "all" "generates all *.cfg files & monitrc" "\"\" & \"\" & \"\"" + printf "%-10s | %-15s | %-40s | %-75s\n" "start" "NA" "starts monit" "\"\"" + echo "----------------------------------------------------------------------------------------------------------------------------------------------" + echo "" } "$@" diff --git a/monit/bin/monit_old.sh b/monit/bin/monit_old.sh deleted file mode 100755 index fdbb67e7b..000000000 --- a/monit/bin/monit_old.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -#SETTING DEFAULT VALUES ############################################################################ - -if [ "-bash" = $0 ]; then - dirpath="${BASH_SOURCE[0]}" -else - dirpath="$0" -fi - -eval ". $(dirname "$dirpath")/setenv.sh" - -if [ ! -f ${TORQHOME}/config/monitrc ] || [ ! -f ${TORQHOME}/code/scripts/monit/monitalert.cfg ] || [ ! -f ${TORQHOME}/code/scripts/monit/monitconfig.cfg ];then # run fill_templates.sh if monitrc is no present - eval ". $(dirname "$dirpath")/fill_templates.sh" -fi - -monit -c ${TORQHOME}/config/monitrc $@ - diff --git a/monit/templates/monitrc b/monit/templates/monitrc index 56817a8c1..f7aa167f2 100755 --- a/monit/templates/monitrc +++ b/monit/templates/monitrc @@ -24,7 +24,7 @@ ## Set syslog logging. If you want to log to a standalone log file instead, ## specify the full path to the log file # - set logfile ${PWD}/monit/logs/monit.log + set logfile ${BASEDIR}/monit/logs/monit.log # # @@ -46,7 +46,7 @@ ## the monitoring state across reboots. If it is on temporary filesystem, the ## state will be lost on reboot which may be convenient in some situations. # - set statefile ${PWD}/monit/logs/monit.state + set statefile ${BASEDIR}/monit/logs/monit.state #set statefile /var/lib/monit/state # # @@ -100,7 +100,7 @@ # set eventqueue # basedir /var/lib/monit/events # set the base directory where events will be stored - basedir ${PWD}/monit/events + basedir ${BASEDIR}/monit/events slots 100 # optionally limit the queue size # # @@ -318,4 +318,4 @@ This message has been generated automatically! # # include /etc/monit.d/* # - include ${PWD}/monit/config/*.cfg + include ${BASEDIR}/monit/config/*.cfg From 9d82a9618db2bff671026ca3d30f023c20191541 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Fri, 10 Aug 2018 12:48:40 +0100 Subject: [PATCH 16/24] Reverted a change in gateway.q --- config/settings/gateway.q | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/gateway.q b/config/settings/gateway.q index 4b092b310..2f7375671 100644 --- a/config/settings/gateway.q +++ b/config/settings/gateway.q @@ -11,5 +11,5 @@ clearinactivetime:0D01:00 // the time to keep inactive handle data // Server connection details \d .servers -CONNECTIONS:`rdb`hdb`wdb // list of connections to make at start up +CONNECTIONS:`rdb`hdb // list of connections to make at start up RETRY:0D00:01 // period on which to retry dead connections. If 0, no reconnection attempts From dbd21ff04aa60eb03cef0a489afd1b8d2dffb66d Mon Sep 17 00:00:00 2001 From: Aqualex Date: Fri, 10 Aug 2018 12:52:25 +0100 Subject: [PATCH 17/24] Removed unnecessary monitconfig.cfg --- monit/templates/monitconfig.cfg | 1 - 1 file changed, 1 deletion(-) delete mode 100644 monit/templates/monitconfig.cfg diff --git a/monit/templates/monitconfig.cfg b/monit/templates/monitconfig.cfg deleted file mode 100644 index 14b411c5c..000000000 --- a/monit/templates/monitconfig.cfg +++ /dev/null @@ -1 +0,0 @@ -appconfig/process.csv torq.sh monitconfig.cfg From b468b3b7f085a7d29dde37d5d3e698e4333cd5cc Mon Sep 17 00:00:00 2001 From: Aqualex Date: Thu, 16 Aug 2018 15:52:32 +0100 Subject: [PATCH 18/24] Removed homer.aquaq.co.uk and google.com from the PR --- monit/templates/monitalert.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monit/templates/monitalert.cfg b/monit/templates/monitalert.cfg index 8c9a559e2..62800e6a3 100644 --- a/monit/templates/monitalert.cfg +++ b/monit/templates/monitalert.cfg @@ -1,11 +1,11 @@ -check system homer.aquaq.co.uk +check system examplehost if loadavg (5min) > 3 for 4 cycles then alert if loadavg (15min) > 1 for 4 cycles then alert if memory usage > 80% for 4 cycles then alert if swap usage > 20% for 4 cycles then alert if cpu usage (system) > 20% for 4 cycles then alert -check host google.com with address gwadawdawdwad.com +check host examplehost.com with address gwadawdawdwad.com if failed url http://gawdawdawdaw.com then alert From add361b2aaae216591cc9a7c2d3b6c4cc0a3f523 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Thu, 16 Aug 2018 15:54:03 +0100 Subject: [PATCH 19/24] Fixed typo --- monit/bin/monit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit/bin/monit.sh b/monit/bin/monit.sh index 9319d047c..7ddc5e553 100755 --- a/monit/bin/monit.sh +++ b/monit/bin/monit.sh @@ -1,5 +1,5 @@ #!/bin/bash -#FUNTION DECLARATION ############################################################################### +#FUNCTION DECLARATION ############################################################################### cd ../.. && export BASEDIR="${PWD}" && cd - > /dev/null; # set BASEDIR to the root of the torq directory From 8d0c4333ccd00ec46bb98058e14a789c6a1b0af6 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Mon, 3 Sep 2018 21:49:22 +0100 Subject: [PATCH 20/24] FIXED error with certificates in setenv.sh ADDED an empty documentation file --- docs/monit.md | 2 ++ monit/bin/monit.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 docs/monit.md diff --git a/docs/monit.md b/docs/monit.md new file mode 100644 index 000000000..95069caa5 --- /dev/null +++ b/docs/monit.md @@ -0,0 +1,2 @@ +Monitoring +========== diff --git a/monit/bin/monit.sh b/monit/bin/monit.sh index 7ddc5e553..78d540b72 100755 --- a/monit/bin/monit.sh +++ b/monit/bin/monit.sh @@ -2,6 +2,7 @@ #FUNCTION DECLARATION ############################################################################### cd ../.. && export BASEDIR="${PWD}" && cd - > /dev/null; # set BASEDIR to the root of the torq directory + mkdir -p ../logs checkst(){ #function to check if file exists @@ -62,7 +63,7 @@ createmonalert(){ } generate(){ - eval ". $BASEDIR/setenv.sh" # set environment variables + eval "cd $BASEDIR && . setenv.sh && cd - > /dev/null" # set environment variables templates="${BASEDIR}/monit/templates" # set temmplates folder configs="${BASEDIR}/monit/config" # set configs folder monit_control="${BASEDIR}/monit/config/monitrc" # set output file for main monit conf From 3c3646915bff4d006cd528d95e9fbfbe2258d994 Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 4 Sep 2018 00:06:52 +0100 Subject: [PATCH 21/24] Added documentation for monit Removed unnecessary comment --- docs/monit.md | 76 ++++++++++++++++++++++++++++++++++++++++++++-- monit/bin/monit.sh | 1 - 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/monit.md b/docs/monit.md index 95069caa5..9fd2d90c2 100644 --- a/docs/monit.md +++ b/docs/monit.md @@ -1,2 +1,74 @@ -Monitoring -========== +Monitoring TorQ +=============== + +**Monit** is a small open source utility for monitoring and managing UNIX systems. Monit's ease of use makes it the perfect tool for tracking the status of TorQ processes. + +Installation +------------ +Monit does not require any specific installation instructions. It has been added as an auxiliary utility to TorQ and can be used using the configuration files which are provided by AquaQ or by specifing the location of user defined configuration files. + +The basic monit directory which has been added to TorQ can be seen below: +``` +${TORQHOME} +|---monit + |---bin + | |---monit.sh + |---templates + |---monitalert.cfg + |---monitrc + |---monittemplate.txt +``` + +It is important to mention that AquaQ will not offer support for **monitalert.cfg** and **monitrc**. Those two files have been added as an example on how **monit** can be configured to monitor your system and to offer an out-of-the-box configuration that you can use to test that **monit** works. + +Features +-------- +Monit is only available for UNIX and it comes with a bash script that you can use to generate the configuration and start the processes. More details on how yo use this script can be found below. + +We have also included a standard **monitrc** which will: ++ Set the check interval to 30 seconds ++ Set the location of the **monit.log** file ++ Set the location of **monit.state** fsile ++ Define the **mail alert** basic configuration ++ Define the **e-mail format** ++ Set the **interface port** (11000) **user** and **password** ++ Set the location of the ***.cfg** files + +The **monitalert.cfg** it is only an example on how you can configure your own alerts for monitoring your UNIX system. There are no TorQ specific examples in this file. + +The only file which will be updated with future TorQ releases is the **monittemplate.txt** which generates the **monitconfig.cfg** which contains the configuration for monitoring TorQ processes. An example is included below: + +``` +check process tickerplant1 + matching "15000 -proctype tickerplant -procname tickerplant1" + start program = "/bin/bash -c '/home/USER/torqprodsupp/torqdev/deploy/torq.sh start tickerplant1'" + with timeout 10 seconds + stop program = "/bin/bash -c '/home/USER/torqprodsupp/torqdev/deploy/torq.sh stop tickerplant1'" + every "* * * * *" + mode active +``` + +Usage Guide +----------- +If you want to use **monit** to monitor your UNIX system and TorQ processes you must first generate the configuration files and then start **monit**. We will assume that you start with a fresh copy of TorQ. +1. Clone TorQ from https://github.com/AquaQAnalytics/TorQ.git +2. Clone TorQ-Finance-Starter-Pack from https://github.com/AquaQAnalytics/TorQ-Finance-Starter-Pack.git (optional) +3. Create the **deploy** folder and copy TorQ and TorQ-Finance-Starter-Pack (optional) in it +4. Modify the **${KDBBASEPORT}** if necessary +5. Navigate to **${TORQHOME}/monit/bin/** +6. Execute: +a. bash monit.sh generate all - to generate all the config files +b. bash monit.sh generate alert - to generate the alert configuration file +c. bash monit.sh generate monitconfig - to generate the monitconfig.cfg +d. bash monit.sh generate monitrc - to generate the monitrc file + +However, you can also use your own configuration files by either creating a new directory in monit called **config** and moving all the *.cfg files and the **monitrc** file in there or by modifying the last line in the monitrc to point to the folder where the *.cfg files can be found. + +7. Start monit by executing bash monit.sh start + +The start function also take a parameter **("string")** whch can specify the location of the **monitrc**. + + + + + diff --git a/monit/bin/monit.sh b/monit/bin/monit.sh index 78d540b72..66530fc8c 100755 --- a/monit/bin/monit.sh +++ b/monit/bin/monit.sh @@ -82,7 +82,6 @@ generate(){ fi ;; monitrc) - #I need some way to define the location of the monitrc template if [ ! -f ${monit_control} ];then if [ -z $2 ]; then controltemplate="$(cat ${templates}/monitrc)" From cb05f05352b63e50156cf5183d6b70a0bc65aa2d Mon Sep 17 00:00:00 2001 From: Aqualex Date: Tue, 4 Sep 2018 00:11:11 +0100 Subject: [PATCH 22/24] Fixed indentation for nested list --- docs/monit.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/monit.md b/docs/monit.md index 9fd2d90c2..243b11c0e 100644 --- a/docs/monit.md +++ b/docs/monit.md @@ -57,10 +57,10 @@ If you want to use **monit** to monitor your UNIX system and TorQ processes you 4. Modify the **${KDBBASEPORT}** if necessary 5. Navigate to **${TORQHOME}/monit/bin/** 6. Execute: -a. bash monit.sh generate all - to generate all the config files -b. bash monit.sh generate alert - to generate the alert configuration file -c. bash monit.sh generate monitconfig - to generate the monitconfig.cfg -d. bash monit.sh generate monitrc - to generate the monitrc file + * bash monit.sh generate all - to generate all the config files + * bash monit.sh generate alert - to generate the alert configuration file + * bash monit.sh generate monitconfig - to generate the monitconfig.cfg + * bash monit.sh generate monitrc - to generate the monitrc file However, you can also use your own configuration files by either creating a new directory in monit called **config** and moving all the *.cfg files and the **monitrc** file in there or by modifying the last line in the monitrc to point to the folder where the *.cfg files can be found. From 54dbcaac46bace64f2bc5b37cc9cd6026e890cda Mon Sep 17 00:00:00 2001 From: Jonny Press Date: Tue, 4 Sep 2018 12:42:36 +0100 Subject: [PATCH 23/24] Minor modifications to wording --- docs/monit.md | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/docs/monit.md b/docs/monit.md index 243b11c0e..fff639c91 100644 --- a/docs/monit.md +++ b/docs/monit.md @@ -5,7 +5,7 @@ Monitoring TorQ Installation ------------ -Monit does not require any specific installation instructions. It has been added as an auxiliary utility to TorQ and can be used using the configuration files which are provided by AquaQ or by specifing the location of user defined configuration files. +Monit is included in most Unix distributions but can also be downloaded from [here](https://mmonit.com/monit/#download). This monit addition to TorQ allows the monit config files to be easily generated, based on the contents of the process.csv file. The basic monit directory which has been added to TorQ can be seen below: ``` @@ -19,11 +19,11 @@ ${TORQHOME} |---monittemplate.txt ``` -It is important to mention that AquaQ will not offer support for **monitalert.cfg** and **monitrc**. Those two files have been added as an example on how **monit** can be configured to monitor your system and to offer an out-of-the-box configuration that you can use to test that **monit** works. +It is important to mention that AquaQ will not offer support for **monitalert.cfg** and **monitrc**. Those two files have been added as an example on how **monit** can be configured to monitor your system and to offer an out-of-the-box configuration that you can use to test that **monit** works. If the monit installation contains an updated version of monitrc, this should be used instead. Features -------- -Monit is only available for UNIX and it comes with a bash script that you can use to generate the configuration and start the processes. More details on how yo use this script can be found below. +Monit is only available for UNIX and it comes with a bash script that you can use to generate the configuration and start the processes. More details on how you use this script can be found below. We have also included a standard **monitrc** which will: + Set the check interval to 30 seconds @@ -36,7 +36,7 @@ We have also included a standard **monitrc** which will: The **monitalert.cfg** it is only an example on how you can configure your own alerts for monitoring your UNIX system. There are no TorQ specific examples in this file. -The only file which will be updated with future TorQ releases is the **monittemplate.txt** which generates the **monitconfig.cfg** which contains the configuration for monitoring TorQ processes. An example is included below: +The only file which will be updated with future TorQ releases is the **monittemplate.txt** which generates the **monitconfig.cfg**. An example is included below: ``` check process tickerplant1 @@ -51,12 +51,9 @@ check process tickerplant1 Usage Guide ----------- If you want to use **monit** to monitor your UNIX system and TorQ processes you must first generate the configuration files and then start **monit**. We will assume that you start with a fresh copy of TorQ. -1. Clone TorQ from https://github.com/AquaQAnalytics/TorQ.git -2. Clone TorQ-Finance-Starter-Pack from https://github.com/AquaQAnalytics/TorQ-Finance-Starter-Pack.git (optional) -3. Create the **deploy** folder and copy TorQ and TorQ-Finance-Starter-Pack (optional) in it -4. Modify the **${KDBBASEPORT}** if necessary -5. Navigate to **${TORQHOME}/monit/bin/** -6. Execute: +1. Install TorQ and the any optional customisations (e.g. the TorQ Finanace Starter Pack) +2. Navigate to **${TORQHOME}/monit/bin/** +3. Execute: * bash monit.sh generate all - to generate all the config files * bash monit.sh generate alert - to generate the alert configuration file * bash monit.sh generate monitconfig - to generate the monitconfig.cfg @@ -64,11 +61,6 @@ If you want to use **monit** to monitor your UNIX system and TorQ processes you However, you can also use your own configuration files by either creating a new directory in monit called **config** and moving all the *.cfg files and the **monitrc** file in there or by modifying the last line in the monitrc to point to the folder where the *.cfg files can be found. -7. Start monit by executing bash monit.sh start - -The start function also take a parameter **("string")** whch can specify the location of the **monitrc**. - - - - +4. Start monit by executing bash monit.sh start +The start function also take a parameter **("string")** whch can specify the location of the **monitrc**. From 396ee3051cbe7531655a6b1a9063b0ed0286b307 Mon Sep 17 00:00:00 2001 From: Jonny Press Date: Tue, 4 Sep 2018 12:44:13 +0100 Subject: [PATCH 24/24] added monit to menu --- mkdocs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index c971330ac..ef8718088 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,9 +9,10 @@ pages: - Connection Management: conn.md - Processes: Processes.md - Analytics Library: analyticslib.md + - Monitoring: monit.md - Visualisation: visualisation.md - TorQ Blog Posts: blog.md -copyright: 'Copyright © 2016 AquaQ Analytics Limited. Kx ® and kdb+ are registered trademarks of Kx Systems Inc.' +copyright: 'Copyright © 2018 AquaQ Analytics Limited. Kx ® and kdb+ are registered trademarks of Kx Systems Inc.' theme: 'material' repo_name: 'GitHub'