Add ELK support for Ceph

Add logstash template file for ceph. Add a beaver ceph.conf file for ceph containers and hosts. Due to the permissions on /var/log/ceph.log and /var/log/ceph-audit.log it isn't possible to capture these at this point, and Ceph offers no otpions to adjust this, so we will need an alternative solution for this. This applies to the mons only.
rcbops · Aug 28, 2015 · 571fab5 · 571fab5
1 parent 63e78af
commit 571fab5
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 0 deletions.
diff --git a/rpcd/etc/openstack_deploy/env.d/ceph.yml b/rpcd/etc/openstack_deploy/env.d/ceph.yml
@@ -30,6 +30,7 @@ container_skel:
       - mons
     properties:
       container_release: trusty
+      service_name: ceph
   ceph_osd_container:
     belongs_to:
       - osds_containers
@@ -38,6 +39,7 @@ container_skel:
     properties:
       is_metal: true
       container_release: trusty
+      service_name: ceph
 
 
 physical_skel:

diff --git a/rpcd/playbooks/beaver.yml b/rpcd/playbooks/beaver.yml
@@ -89,3 +89,10 @@
       - name: "auth"
         log_file: "/var/log/auth.log"
         tags: "auth,infrastructure"
+      - name: "ceph-mon"
+        log_file: "/var/log/ceph/ceph-mon.{{ inventory_hostname }}.log"
+        multiline_regex_before: '^[a-z_]*\s'
+        tags: "ceph-mon,ceph,infrastructure"
+      - name: "ceph-osd"
+        log_file: "/var/log/ceph/ceph-osd.*.log"
+        tags: "ceph-osd,ceph,infrastructure"
diff --git a/rpcd/playbooks/ceph-osd.yml b/rpcd/playbooks/ceph-osd.yml
@@ -19,6 +19,28 @@
 - name: Deploy osds
   hosts: osds
   user: root
+  pre_tasks:
+    - name: Create log dir
+      file:
+        path: "{{ item.path }}"
+        state: directory
+      with_items:
+        - { path: "/openstack/log/{{ inventory_hostname }}-ceph" }
+      when: is_metal | bool
+      tags:
+        - ceph-logs
+        - ceph-log-dirs
+    - name: Create log aggregation links
+      file:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        state: "{{ item.state }}"
+        force: "yes"
+      with_items:
+        - { src: "/openstack/log/{{ inventory_hostname }}-ceph", dest: "/var/log/ceph", state: "link" }
+      when: is_metal | bool
+      tags:
+        - ceph-logs
   roles:
     - ceph-osd
   vars:

diff --git a/rpcd/playbooks/roles/logstash/tasks/logstash_post_install.yml b/rpcd/playbooks/roles/logstash/tasks/logstash_post_install.yml
@@ -70,6 +70,7 @@
     - 15-beaver.conf
     - 16-elasticsearch.conf
     - 17-rabbitmq.conf
+    - 18-ceph.conf
     - 99-output.conf
   notify: Restart Logstash
   tags:

diff --git a/rpcd/playbooks/roles/logstash/templates/18-ceph.conf b/rpcd/playbooks/roles/logstash/templates/18-ceph.conf
@@ -0,0 +1,12 @@
+filter {
+  if "ceph" in [tags] {
+    grok {
+      match => { "message" => "%{TIMESTAMP_ISO8601:date} %{NOTSPACE:osd_epoch} ?%{SPACE}?%{NOTSPACE:error_bool} %{GREEDYDATA:logmessage}" }
+    }
+  }
+  if "ceph-osd" in [tags] {
+    grok {
+      match => { "message" => "-- (?<src_host>(%{IPORHOST}\:%{POSINT}/%{POSINT})) (?:[<|>]){1,2} (?<dst_host>(%{IPORHOST}\:%{POSINT}/%{POSINT}))" }
+    }
+  }
+}