Fix syncing logs when shutting down a server

playbooks/roles/common_vars/defaults/main.yml

@@ -21,6 +21,7 @@ COMMON_OBJECT_STORE_LOG_SYNC: False
 COMMON_OBJECT_STORE_LOG_SYNC_BUCKET: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}"
 COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/"
 COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store"
+COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit"
 
 COMMON_BASE_DIR: /edx
 COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"

playbooks/roles/vhost/tasks/main.yml

@@ -31,6 +31,15 @@
     mode: "{{ item.value.mode }}"
   with_dict: "{{ vhost_dirs }}"
 
+- name: Add script for syncing logs on exit
+  template:
+    src: sync-logs-on-exit.j2
+    dest: "{{ COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT }}"
+    owner: root
+    group: root
+    mode: 0744
+  when: COMMON_OBJECT_STORE_LOG_SYNC
+
 - name: Force logrotate on supervisor stop
   template:
     src: etc/init/sync-on-stop.conf.j2
@@ -39,6 +48,27 @@
     group: root
     mode: 0644
   when: COMMON_OBJECT_STORE_LOG_SYNC
+        and (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty')
+
+# With systemd, log rotation should be run before the `rsyslog` service exits
+# to ensure that all logs are compressed and synced before shutting down the server.
+- name: Add systemd service for syncing logs on exit
+  template:
+    src: etc/systemd/system/sync-logs-on-exit.service.j2
+    dest: /etc/systemd/system/sync-logs-on-exit.service
+    owner: root
+    group: root
+    mode: 0644
+  when: COMMON_OBJECT_STORE_LOG_SYNC
+        and not (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty')
+
+- name: Enable systemd service for syncing logs on exit
+  systemd:
+    name: sync-logs-on-exit
+    enabled: yes
+    daemon_reload: yes
+  when: COMMON_OBJECT_STORE_LOG_SYNC
+        and not (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty')
 
 - name: Update /etc/dhcp/dhclient.conf
   template:

playbooks/roles/vhost/templates/etc/init/sync-on-stop.conf.j2

@@ -1,6 +1,5 @@
 start on stopped supervisor
 description     "sync tracking logs on supervisor shutdown"
 script
-        /usr/sbin/logrotate -f /etc/logrotate.d/hourly/tracking.log
-        /usr/sbin/logrotate -f /etc/logrotate.d/hourly/edx-services
+        "{{ COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT }}"
 end script

playbooks/roles/vhost/templates/etc/systemd/system/sync-logs-on-exit.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Synchronizes logs on exit
+After=rsyslog.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStop=/edx/bin/sync-logs-on-exit
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/vhost/templates/sync-logs-on-exit.j2

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+/usr/sbin/logrotate -f /etc/logrotate.d/hourly/tracking.log
+/usr/sbin/logrotate -f /etc/logrotate.d/hourly/edx-services