diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 170ae4b4..bfd37803 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -32,7 +32,7 @@ loggly_username: ops ## cores and logs ## -app_log_dir: /var/log +app_log_dir: /var/log/runnable core_file_dir: /var/log/core ## diff --git a/ansible/roles/bash_aliases/tasks/main.yml b/ansible/roles/bash_aliases/tasks/main.yml new file mode 100644 index 00000000..d0fd92ba --- /dev/null +++ b/ansible/roles/bash_aliases/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: copy bash aliases to ubuntu + tags: [ loggly, bash_aliases ] + template: + src=dot_bash_aliases.sh.j2 + dest=/home/ubuntu/.bash_aliases + owner=ubuntu + group=ubuntu + mode=0700 + +- name: copy bash aliases to root + tags: [ loggly, bash_aliases ] + become: true + template: + src=dot_bash_aliases_root.sh.j2 + dest=/root/.bash_aliases + owner=root + group=root + mode=0700 diff --git a/ansible/roles/bash_aliases/templates/dot_bash_aliases.sh.j2 b/ansible/roles/bash_aliases/templates/dot_bash_aliases.sh.j2 new file mode 100644 index 00000000..59fb4b81 --- /dev/null +++ b/ansible/roles/bash_aliases/templates/dot_bash_aliases.sh.j2 @@ -0,0 +1,19 @@ +# Follows the logfile for a given app_name interpolating the datetime string into the logpath (/var/log/runnable/YYYY/MM/DD/HH/.log) +# Usage: logtail +function logtail() { + local app_name="$1" + local datetime=`date +%Y/%m/%d/%H` + local app_log_dir="{{ app_log_dir }}" + local logfile="${app_log_dir}/${datetime}/${app_name}.log" + tail -f ${logfile} | bunyan +} + +# Outputs contents of an npm start log for , if it exists, into a pager for reading. +# Usage: npmlog +function npmlog() { + local app_name="$1" + local app_log_dir="/var/log" + local logfile="${app_log_dir}/${app_name}.log" + less ${logfile} +} + diff --git a/ansible/roles/bash_aliases/templates/dot_bash_aliases_root.sh.j2 b/ansible/roles/bash_aliases/templates/dot_bash_aliases_root.sh.j2 new file mode 100644 index 00000000..59fb4b81 --- /dev/null +++ b/ansible/roles/bash_aliases/templates/dot_bash_aliases_root.sh.j2 @@ -0,0 +1,19 @@ +# Follows the logfile for a given app_name interpolating the datetime string into the logpath (/var/log/runnable/YYYY/MM/DD/HH/.log) +# Usage: logtail +function logtail() { + local app_name="$1" + local datetime=`date +%Y/%m/%d/%H` + local app_log_dir="{{ app_log_dir }}" + local logfile="${app_log_dir}/${datetime}/${app_name}.log" + tail -f ${logfile} | bunyan +} + +# Outputs contents of an npm start log for , if it exists, into a pager for reading. +# Usage: npmlog +function npmlog() { + local app_name="$1" + local app_log_dir="/var/log" + local logfile="${app_log_dir}/${app_name}.log" + less ${logfile} +} + diff --git a/ansible/roles/loggly/templates/rsyslog.conf.j2 b/ansible/roles/loggly/files/rsyslog.conf similarity index 76% rename from ansible/roles/loggly/templates/rsyslog.conf.j2 rename to ansible/roles/loggly/files/rsyslog.conf index 5621cf3b..9a76afee 100644 --- a/ansible/roles/loggly/templates/rsyslog.conf.j2 +++ b/ansible/roles/loggly/files/rsyslog.conf @@ -35,6 +35,11 @@ $KLogPermitNonKernelFacility on # $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat +# Runnable JSON logging format +# Creates an "output formatter" template that accepts as input JSON and prints it out without any further processing ("raw JSON"). +# The formatting around the %msg% string is as such: start printing at the second character "2" until the end of the line "$" using the raw JSON format type. +$template RunnableJSON,"%msg:2:$:jsonr%\n" + # Filter duplicated messages $RepeatedMsgReduction on @@ -43,11 +48,11 @@ $RepeatedMsgReduction on # $FileOwner syslog $FileGroup adm -$FileCreateMode 0640 +$FileCreateMode 0644 $DirCreateMode 0755 $Umask 0022 $PrivDropToUser syslog -$PrivDropToGroup syslog +$PrivDropToGroup adm # # Where to place spool and state files diff --git a/ansible/roles/loggly/meta/main.yml b/ansible/roles/loggly/meta/main.yml new file mode 100644 index 00000000..1d1930df --- /dev/null +++ b/ansible/roles/loggly/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: bash_aliases } diff --git a/ansible/roles/loggly/tasks/main.yml b/ansible/roles/loggly/tasks/main.yml index e427b2aa..078a570f 100644 --- a/ansible/roles/loggly/tasks/main.yml +++ b/ansible/roles/loggly/tasks/main.yml @@ -46,24 +46,6 @@ owner=syslog group=syslog -- name: create runnable bin directory - tags: loggly - become: true - file: - path=/opt/runnable/bin - state=directory - mode=0755 - owner=ubuntu - group=ubuntu - -- name: copy rotate util script - tags: loggly - become: true - template: - src=rotate-logs.sh.j2 - dest=/opt/runnable/bin/rotate-{{ name }}-logs.sh - mode=0755 - - name: copy app config tags: loggly become: true @@ -76,8 +58,8 @@ - name: copy rsyslog config tags: loggly become: true - template: - src=rsyslog.conf.j2 + copy: + src=rsyslog.conf dest=/etc/rsyslog.conf owner=syslog group=syslog @@ -94,27 +76,16 @@ path=/var/spool/rsyslog/stat-{{ name }} state=absent -- name: check for current log file - tags: [loggly, deploy] - stat: path="{{ app_log_dir }}/{{ name }}-daemon.log" - register: log_file - -- name: remove old log file - when: log_file.stat.exists - tags: [loggly, deploy] - become: true - file: - path="{{ app_log_dir }}/{{ name }}-daemon.log" - state=absent - -- name: touch the log file - tags: [loggly, deploy] +- name: ensure log path + tags: loggly become: true file: - path="{{ app_log_dir }}/{{ name }}-daemon.log" - state=touch + path="{{ app_log_dir }}" + state=directory owner=syslog group=adm + mode=0755 + recurse=yes - name: restart rsyslog tags: [loggly, deploy] diff --git a/ansible/roles/loggly/templates/21-output-syslog.conf.j2 b/ansible/roles/loggly/templates/21-output-syslog.conf.j2 index 94a449bc..3a0e8697 100644 --- a/ansible/roles/loggly/templates/21-output-syslog.conf.j2 +++ b/ansible/roles/loggly/templates/21-output-syslog.conf.j2 @@ -1,14 +1,10 @@ $WorkDirectory /var/spool/rsyslog -# start log rotation via outchannel -# outchannel definition -$outchannel log_rotation_{{ name }},{{ app_log_dir }}/{{ name }}-daemon.log,524288000,/opt/runnable/bin/rotate-{{ name }}-logs.sh -# activate the channel and log everything to it -if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then :omfile:$log_rotation_{{ name }} -# end log rotation via outchannel +# Rotate per hour +$template RotateHourly_{{ name }},"{{ app_log_dir }}/%$YEAR%/%$MONTH%/%$DAY%/%$HOUR%/{{ name }}.log" +if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then { action (type="omfile" DynaFile="RotateHourly_{{ name }}" template="RunnableJSON") } -# {{ name }} access file: -#Add a tag for {{ name }} events +# Loggly: Add a tag for {{ name }} events $template LogglyFormat_{{ name }},"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [{{ loggly_token }}@41058 tag=\"runnable\" tag=\"{{ node_env }}\"] %msg%\n" -if $msg contains '{{ name }}' then @@logs-01.loggly.com:6514;LogglyFormat_{{ name }} -if $msg contains '{{ name }}' then stop +if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then @@logs-01.loggly.com:6514;LogglyFormat_{{ name }} +if $msg contains '{{ name }}' and $syslogfacility-text == 'local7' then stop diff --git a/ansible/roles/loggly/templates/rotate-logs.sh.j2 b/ansible/roles/loggly/templates/rotate-logs.sh.j2 deleted file mode 100644 index 69c3f8b9..00000000 --- a/ansible/roles/loggly/templates/rotate-logs.sh.j2 +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# literally truncate the file -cat /dev/null > {{ app_log_dir }}/{{ name }}-daemon.log 2>&1 -# exit code? don't care - -exit 0