diff --git a/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml b/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml index 2e306dad..933674cf 100644 --- a/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml +++ b/deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml @@ -3,7 +3,7 @@ # UAA client for firehose-to-syslog - type: replace - path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/firehose-to-syslog? + path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/((uaa_clients_firehose_client_id))? value: override: true authorized-grant-types: client_credentials diff --git a/deployment/operations/cloudfoundry.yml b/deployment/operations/cloudfoundry.yml index ce5933fa..58ac5559 100644 --- a/deployment/operations/cloudfoundry.yml +++ b/deployment/operations/cloudfoundry.yml @@ -77,9 +77,10 @@ ingestor: {from: ingestor_link} properties: cloudfoundry: - firehose_client_id: firehose-to-syslog + firehose_client_id: "((firehose_client_id))" firehose_client_secret: "((firehose_client_secret))" skip_ssl_validation: true + firehose_subscription_id: "((firehose_client_id))" - type: replace path: /instance_groups/name=ingestor/jobs/name=syslog_forwarder/properties/syslog_forwarder/config/- @@ -113,6 +114,7 @@ properties: kibana-auth: cloudfoundry: + client_id: "((kibana_oauth2_client_id))" client_secret: "((kibana_oauth2_client_secret))" skip_ssl_validation: true @@ -138,7 +140,7 @@ port: 80 registration_interval: 60s uris: - - "logs.((system_domain))" + - "((kibana_hostname)).((system_domain))" - type: replace path: /instance_groups/name=kibana/jobs/name=kibana/properties/kibana/plugins? diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec index effc1d6d..5f0040f8 100644 --- a/jobs/ingestor_syslog/spec +++ b/jobs/ingestor_syslog/spec @@ -158,9 +158,27 @@ properties: logstash_parser.elasticsearch.document_id: description: "Use a specific, dynamic ID rather than an auto-generated identifier." default: ~ + logstash_parser.elasticsearch.index_name.platform.base: + description: "The platform index base name" + default: "platform" + logstash_parser.elasticsearch.index_name.platform.period: + description: "The platform index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html" + default: "YYYY.MM.dd" + logstash_parser.elasticsearch.index_name.app.base: + description: "The app index base name" + default: "app" + logstash_parser.elasticsearch.index_name.app.append_org_to_base: + description: "Whether to add the CF Org to the index base name" + default: true + logstash_parser.elasticsearch.index_name.app.append_space_to_base: + description: "Whether to add the CF Space to the index base name" + default: false + logstash_parser.elasticsearch.index_name.app.period: + description: "The app index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html" + default: "YYYY.MM.dd" logstash_parser.elasticsearch.index: description: "The specific, dynamic index name to write events to." - default: "logstash-%{+YYYY.MM.dd}" + default: "logs-%{[@metadata][index_base]}-%{[@metadata][index_period]}" logstash_parser.elasticsearch.index_type: description: "The specific, dynamic index type name to write events to." default: "%{@type}" diff --git a/jobs/ingestor_syslog/templates/config/filters_post.conf.erb b/jobs/ingestor_syslog/templates/config/filters_post.conf.erb index 4570fcc2..470dd44e 100644 --- a/jobs/ingestor_syslog/templates/config/filters_post.conf.erb +++ b/jobs/ingestor_syslog/templates/config/filters_post.conf.erb @@ -20,3 +20,33 @@ } } + + mutate { + add_field => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.platform.base')%>" } + add_field => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.platform.period')%>}" } # YYYY.MM.dd + } + + # Custom Override @metadata.index + if [@index_type] == "app" { + mutate { + replace => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.app.base')%>" } + replace => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.app.period')%>}" } # YYYY.MM, xxxx.ww, YYYY.MM.dd + } + <% if p('logstash_parser.elasticsearch.index_name.app.append_org_to_base') %> + if [@cf][org] { + mutate { + replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][org]}" } + } + } + <% end %> + <% if p('logstash_parser.elasticsearch.index_name.app.append_space_to_base') %> + if [@cf][space] { + mutate { + replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][space]}" } + } + } + <% end %> + mutate { + lowercase => [ "[@metadata][index_base]" ] + } + }