From 5ca29df29e81142da4f766d542203bcb834686f2 Mon Sep 17 00:00:00 2001 From: Mariusz Olejnik Date: Sat, 28 Dec 2019 01:28:18 +0100 Subject: [PATCH 1/3] Index & deployment name customization --- .../cf/add-firehose-to-syslog-uaa-clients.yml | 2 +- deployment/operations/cloudfoundry.yml | 5 ++-- jobs/ingestor_syslog/spec | 20 ++++++++++++- .../templates/config/filters_post.conf.erb | 30 +++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml b/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml index 2e306dad..933674cf 100644 --- a/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml +++ b/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml @@ -3,7 +3,7 @@ # UAA client for firehose-to-syslog - type: replace - path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/firehose-to-syslog? + path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/((uaa_clients_firehose_client_id))? value: override: true authorized-grant-types: client_credentials diff --git a/deployment/operations/cloudfoundry.yml b/deployment/operations/cloudfoundry.yml index ce5933fa..20d40575 100644 --- a/deployment/operations/cloudfoundry.yml +++ b/deployment/operations/cloudfoundry.yml @@ -77,7 +77,7 @@ ingestor: {from: ingestor_link} properties: cloudfoundry: - firehose_client_id: firehose-to-syslog + firehose_client_id: "((firehose_client_id))" firehose_client_secret: "((firehose_client_secret))" skip_ssl_validation: true @@ -113,6 +113,7 @@ properties: kibana-auth: cloudfoundry: + client_id: "((kibana_oauth2_client_id))" client_secret: "((kibana_oauth2_client_secret))" skip_ssl_validation: true @@ -138,7 +139,7 @@ port: 80 registration_interval: 60s uris: - - "logs.((system_domain))" + - "((kibana_hostname)).((system_domain))" - type: replace path: /instance_groups/name=kibana/jobs/name=kibana/properties/kibana/plugins? diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec index effc1d6d..beb60ff9 100644 --- a/jobs/ingestor_syslog/spec +++ b/jobs/ingestor_syslog/spec @@ -158,9 +158,27 @@ properties: logstash_parser.elasticsearch.document_id: description: "Use a specific, dynamic ID rather than an auto-generated identifier." default: ~ + logstash_parser.elasticsearch.index_name.platform.base: + description: "The platform index base name" + default: "platform" + logstash_parser.elasticsearch.index_name.platform.period: + description: "The platform index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html" + default: "YYYY.MM.dd" + logstash_parser.elasticsearch.index_name.app.base: + description: "The app index base name" + default: "app" + logstash_parser.elasticsearch.index_name.app.append_org_to_base: + description: "Whether to add the CF Org to the index base name" + default: true + logstash_parser.elasticsearch.index_name.app.append_space_to_base: + description: "Whether to add the CF Space to the index base name" + default: false + logstash_parser.elasticsearch.index_name.app.period: + description: "The app index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html" + default: "xxxx'w'ww" logstash_parser.elasticsearch.index: description: "The specific, dynamic index name to write events to." - default: "logstash-%{+YYYY.MM.dd}" + default: "logs-%{[@metadata][index_base]}-%{[@metadata][index_period]}" logstash_parser.elasticsearch.index_type: description: "The specific, dynamic index type name to write events to." default: "%{@type}" diff --git a/jobs/ingestor_syslog/templates/config/filters_post.conf.erb b/jobs/ingestor_syslog/templates/config/filters_post.conf.erb index 4570fcc2..470dd44e 100644 --- a/jobs/ingestor_syslog/templates/config/filters_post.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_post.conf.erb @@ -20,3 +20,33 @@ } } + + mutate { + add_field => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.platform.base')%>" } + add_field => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.platform.period')%>}" } # YYYY.MM.dd + } + + # Custom Override @metadata.index + if [@index_type] == "app" { + mutate { + replace => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.app.base')%>" } + replace => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.app.period')%>}" } # YYYY.MM, xxxx.ww, YYYY.MM.dd + } + <% if p('logstash_parser.elasticsearch.index_name.app.append_org_to_base') %> + if [@cf][org] { + mutate { + replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][org]}" } + } + } + <% end %> + <% if p('logstash_parser.elasticsearch.index_name.app.append_space_to_base') %> + if [@cf][space] { + mutate { + replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][space]}" } + } + } + <% end %> + mutate { + lowercase => [ "[@metadata][index_base]" ] + } + } From 280b89b8fb2b0503df9e6dff280e6f9c9d510d44 Mon Sep 17 00:00:00 2001 From: Mariusz Olejnik Date: Sat, 28 Dec 2019 02:43:56 +0100 Subject: [PATCH 2/3] custom firehose_subscription_id --- deployment/operations/cloudfoundry.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/operations/cloudfoundry.yml b/deployment/operations/cloudfoundry.yml index 20d40575..58ac5559 100644 --- a/deployment/operations/cloudfoundry.yml +++ b/deployment/operations/cloudfoundry.yml @@ -80,6 +80,7 @@ firehose_client_id: "((firehose_client_id))" firehose_client_secret: "((firehose_client_secret))" skip_ssl_validation: true + firehose_subscription_id: "((firehose_client_id))" - type: replace path: /instance_groups/name=ingestor/jobs/name=syslog_forwarder/properties/syslog_forwarder/config/- From 77ebbd2f3b9ac3432292bcc09c978522f7b09122 Mon Sep 17 00:00:00 2001 From: Mariusz Olejnik Date: Tue, 31 Dec 2019 08:12:18 +0100 Subject: [PATCH 3/3] restore default period --- jobs/ingestor_syslog/spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec index beb60ff9..5f0040f8 100644 --- a/jobs/ingestor_syslog/spec +++ b/jobs/ingestor_syslog/spec @@ -175,7 +175,7 @@ properties: default: false logstash_parser.elasticsearch.index_name.app.period: description: "The app index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html" - default: "xxxx'w'ww" + default: "YYYY.MM.dd" logstash_parser.elasticsearch.index: description: "The specific, dynamic index name to write events to." default: "logs-%{[@metadata][index_base]}-%{[@metadata][index_period]}"