From db5b0407acd457cecaaa0377f23319198e55fbfa Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 21:39:38 -0400 Subject: [PATCH 1/6] Add running-on-cloudfoundry for filebeat. --- deploy/cloudfoundry/README.md | 11 +++ deploy/cloudfoundry/filebeat/filebeat.yml | 54 +++++++++++++ deploy/cloudfoundry/filebeat/manifest.yml | 10 +++ deploy/cloudfoundry/metricbeat/manifest.yml | 10 +++ deploy/cloudfoundry/metricbeat/metricbeat.yml | 54 +++++++++++++ filebeat/docs/getting-started.asciidoc | 5 ++ .../docs/running-on-cloudfoundry.asciidoc | 80 +++++++++++++++++++ filebeat/docs/setting-up-running.asciidoc | 4 + 8 files changed, 228 insertions(+) create mode 100644 deploy/cloudfoundry/README.md create mode 100644 deploy/cloudfoundry/filebeat/filebeat.yml create mode 100644 deploy/cloudfoundry/filebeat/manifest.yml create mode 100644 deploy/cloudfoundry/metricbeat/manifest.yml create mode 100644 deploy/cloudfoundry/metricbeat/metricbeat.yml create mode 100644 filebeat/docs/running-on-cloudfoundry.asciidoc diff --git a/deploy/cloudfoundry/README.md b/deploy/cloudfoundry/README.md new file mode 100644 index 00000000000..f3c35b7771b --- /dev/null +++ b/deploy/cloudfoundry/README.md @@ -0,0 +1,11 @@ +# Beats Cloud Foundry manifests examples + +## Getting started + +This is the list of officially supported Beats, with example manifests to run +them in Cloud Foundry: + +Beat | Description +---- | ---- +[filebeat](filebeat) | Ships logs from loggregator +[metricbeat](metricbeat) | Ships metrics from loggregator diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml new file mode 100644 index 00000000000..a6ddfaca04b --- /dev/null +++ b/deploy/cloudfoundry/filebeat/filebeat.yml @@ -0,0 +1,54 @@ +#=========================== Filebeat inputs ============================= + +# Configure the input to access loggregator to forward the log events. + +filebeat.inputs: + - &cloudfoundry + type: cloudfoundry + client_id: ${CLIENT_ID:filebeat} + client_secret: ${CLIENT_SECRET:changeme} + api_address: ${API_ADDRESS} + #doppler_address: ${DOPPLER_ADDRESS} + #uaa_address: ${UAA_ADDRESS} + #rlp_address: ${RLP_ADDRESS} + #shard_id: ${SHARD_ID} + + +#================================ Outputs ===================================== + +# Configure the Elasticsearch output either to a specific host or using +# Elastic Cloud. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["${ELASTICSEARCH_OUTPUT}:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +#----------------------------- Elastic Cloud ---------------------------------- + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Processors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_cloudfoundry_metadata: + <<: *cloudfoundry diff --git a/deploy/cloudfoundry/filebeat/manifest.yml b/deploy/cloudfoundry/filebeat/manifest.yml new file mode 100644 index 00000000000..26ec5a0b958 --- /dev/null +++ b/deploy/cloudfoundry/filebeat/manifest.yml @@ -0,0 +1,10 @@ +applications: +- name: filebeat + memory: 256M + instances: 1 + buildpacks: + - binary_buildpack + command: ./filebeat -e -c ~/filebeat.yml + stack: cflinuxfs3 + health-check-type: process + no-route: true diff --git a/deploy/cloudfoundry/metricbeat/manifest.yml b/deploy/cloudfoundry/metricbeat/manifest.yml new file mode 100644 index 00000000000..767c8b8fa08 --- /dev/null +++ b/deploy/cloudfoundry/metricbeat/manifest.yml @@ -0,0 +1,10 @@ +applications: + - name: metricbeat + memory: 256M + instances: 1 + buildpacks: + - binary_buildpack + command: ./metricbeat -e -c ~/metricbeat.yml + stack: cflinuxfs3 + health-check-type: process + no-route: true \ No newline at end of file diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml new file mode 100644 index 00000000000..7beb7858872 --- /dev/null +++ b/deploy/cloudfoundry/metricbeat/metricbeat.yml @@ -0,0 +1,54 @@ +#=========================== Metricbeat modules ============================= + +# Configure the module to forward metrics from loggregator. + +metricbeat.modules: + - &cloudfoundry + module: cloudfoundry + client_id: ${CLIENT_ID} + client_secret: ${CLIENT_SECRET} + api_address: ${API_ADDRESS} + #doppler_address: ${DOPPLER_ADDRESS} + #uaa_address: ${UAA_ADDRESS} + #rlp_address: ${RLP_ADDRESS} + #shard_id: ${SHARD_ID} + + +#================================ Outputs ===================================== + +# Configure the Elasticsearch output either to a specific host or using +# Elastic Cloud. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["${ELASTICSEARCH_OUTPUT}:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +#----------------------------- Elastic Cloud ---------------------------------- + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Processors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_cloudfoundry_metadata: + <<: *cloudfoundry diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 8aa03b5595e..5b3972ad671 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -109,6 +109,11 @@ See <> for deploying Docker containers. See <> for deploying with Kubernetes. +[[cloudfoundry]] +*cloudfoundry:* + +See <> for deploying with Cloud Foundry. + [[win]] *win:* diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc new file mode 100644 index 00000000000..59c9fd81873 --- /dev/null +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -0,0 +1,80 @@ +[[running-on-cloudfoundry]] +=== Running {beatname_uc} on Cloud Foundry + +You can use {beatname_uc} on Cloud Foundry to retrieve and ship container logs. + +ifeval::["{release-state}"=="unreleased"] + +However, version {version} of {beatname_uc} has not yet been +released, so build is currently available for this version. + +endif::[] + + +[float] +==== Cloud Foundry credentials + +{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The uaac +command will create the required credentials for connecting to loggregator. + +["source", "sh"] +------------------------------------------------ +uaac client add filebeat --name filebeat --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +------------------------------------------------ + +[WARNING] +======================================= +*Use a unique secret:* The uaac command above is just an example and the secret should be changed and the +`filebeat.yml` should be updated with your choosen secret. +======================================= + + +[float] +==== Cloud Foundry deploy manifests + +You deploy {beatname_uc} as an application with no route. + +Cloud Foundry requires that 3 files exist inside of a directory to allow {beatname_uc} to be pushed. The commands +below provide the basic steps for getting it up and running. + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +cd {beatname_lc}-{version}-linux-x86_64 +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/{beatname_lc}.yml +# Update api_address, client_id, client_secret in downloaded {beatname_lc}. +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/manifest.yml +------------------------------------------------ + + +[float] +==== Deploy + +To deploy {beatname_uc} to Cloud Foundry, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +cf push +------------------------------------------------ + +To check the status, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ cf apps + +name requested state instances memory disk urls +filebeat started 1/1 256M 1G +------------------------------------------------ + +Log events should start flowing to Elasticsearch. The events are annotated with +metadata added by the <> processor. + + +[WARNING] +======================================= +*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that +{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` +will result in duplicate events being pushed to Elasticsearch. +======================================= diff --git a/filebeat/docs/setting-up-running.asciidoc b/filebeat/docs/setting-up-running.asciidoc index 61f952b94c3..5968b3f571d 100644 --- a/filebeat/docs/setting-up-running.asciidoc +++ b/filebeat/docs/setting-up-running.asciidoc @@ -26,6 +26,8 @@ This section includes additional information on how to set up and run * <> +* <> + * <> @@ -41,6 +43,8 @@ include::./running-on-docker.asciidoc[] include::./running-on-kubernetes.asciidoc[] +include::./running-on-cloudfoundry.asciidoc[] + include::{libbeat-dir}/shared-systemd.asciidoc[] include::{libbeat-dir}/shared-shutdown.asciidoc[] From c891c629d9c4e637a6dcb3992c41242ad227c662 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 21:56:05 -0400 Subject: [PATCH 2/6] Fix typo. --- filebeat/docs/running-on-cloudfoundry.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc index 59c9fd81873..d4ca0514a40 100644 --- a/filebeat/docs/running-on-cloudfoundry.asciidoc +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -6,7 +6,7 @@ You can use {beatname_uc} on Cloud Foundry to retrieve and ship container logs. ifeval::["{release-state}"=="unreleased"] However, version {version} of {beatname_uc} has not yet been -released, so build is currently available for this version. +released, no build is currently available for this version. endif::[] From 63b65d35bcd3fce2db7f2b487e6d42e90d16d243 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 22:00:51 -0400 Subject: [PATCH 3/6] Add running on cloudfoundry for metricbeat. --- .../docs/running-on-cloudfoundry.asciidoc | 2 +- metricbeat/docs/gettingstarted.asciidoc | 5 ++ .../docs/running-on-cloudfoundry.asciidoc | 80 +++++++++++++++++++ metricbeat/docs/setting-up-running.asciidoc | 4 + 4 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/running-on-cloudfoundry.asciidoc diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc index d4ca0514a40..b661c2857ac 100644 --- a/filebeat/docs/running-on-cloudfoundry.asciidoc +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -1,7 +1,7 @@ [[running-on-cloudfoundry]] === Running {beatname_uc} on Cloud Foundry -You can use {beatname_uc} on Cloud Foundry to retrieve and ship container logs. +You can use {beatname_uc} on Cloud Foundry to retrieve and ship logs. ifeval::["{release-state}"=="unreleased"] diff --git a/metricbeat/docs/gettingstarted.asciidoc b/metricbeat/docs/gettingstarted.asciidoc index d185dd69d64..b32cd1280be 100644 --- a/metricbeat/docs/gettingstarted.asciidoc +++ b/metricbeat/docs/gettingstarted.asciidoc @@ -119,6 +119,11 @@ See <> for deploying Docker containers. See <> for deploying with Kubernetes. +[[cloudfoundry]] +*cloudfoundry:* + +See <> for deploying with Cloud Foundry. + [[win]] *win:* diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc new file mode 100644 index 00000000000..731c8abf78d --- /dev/null +++ b/metricbeat/docs/running-on-cloudfoundry.asciidoc @@ -0,0 +1,80 @@ +[[running-on-cloudfoundry]] +=== Running {beatname_uc} on Cloud Foundry + +You can use {beatname_uc} on Cloud Foundry to retrieve and ship metrics. + +ifeval::["{release-state}"=="unreleased"] + +However, version {version} of {beatname_uc} has not yet been +released, no build is currently available for this version. + +endif::[] + + +[float] +==== Cloud Foundry credentials + +{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The uaac +command will create the required credentials for connecting to loggregator. + +["source", "sh"] +------------------------------------------------ +uaac client add filebeat --name filebeat --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +------------------------------------------------ + +[WARNING] +======================================= +*Use a unique secret:* The uaac command above is just an example and the secret should be changed and the +`filebeat.yml` should be updated with your choosen secret. +======================================= + + +[float] +==== Cloud Foundry deploy manifests + +You deploy {beatname_uc} as an application with no route. + +Cloud Foundry requires that 3 files exist inside of a directory to allow {beatname_uc} to be pushed. The commands +below provide the basic steps for getting it up and running. + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +cd {beatname_lc}-{version}-linux-x86_64 +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/{beatname_lc}.yml +# Update api_address, client_id, client_secret in downloaded {beatname_lc}. +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/manifest.yml +------------------------------------------------ + + +[float] +==== Deploy + +To deploy {beatname_uc} to Cloud Foundry, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +cf push +------------------------------------------------ + +To check the status, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ cf apps + +name requested state instances memory disk urls +metricbeat started 1/1 256M 1G +------------------------------------------------ + +Metrics should start flowing to Elasticsearch. The events are annotated with +metadata added by the <> processor. + + +[WARNING] +======================================= +*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that +{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` +will result in duplicate events being pushed to Elasticsearch. +======================================= diff --git a/metricbeat/docs/setting-up-running.asciidoc b/metricbeat/docs/setting-up-running.asciidoc index 305d802fa11..002cd8bd4b1 100644 --- a/metricbeat/docs/setting-up-running.asciidoc +++ b/metricbeat/docs/setting-up-running.asciidoc @@ -26,6 +26,8 @@ This section includes additional information on how to set up and run * <> +* <> + * <> //MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too. @@ -40,6 +42,8 @@ include::./running-on-docker.asciidoc[] include::./running-on-kubernetes.asciidoc[] +include::./running-on-cloudfoundry.asciidoc[] + include::{libbeat-dir}/shared-systemd.asciidoc[] include::{libbeat-dir}/shared-shutdown.asciidoc[] From c9b838b90a64a5416b249f6fb78795bd1a6a5bc8 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 22:15:40 -0400 Subject: [PATCH 4/6] Add changelog. --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6649c2ea36c..626b8d855e6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -206,6 +206,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add pattern for Cisco ASA / FTD Message 734001 {issue}16212[16212] {pull}16612[16612] - Added new module `o365` for ingesting Office 365 management activity API events. {issue}16196[16196] {pull}16386[16386] - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155] +- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] *Heartbeat* @@ -271,6 +272,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release Oracle module as GA. {issue}14279[14279] {pull}16833[16833] - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] - Add Storage metricsets to GCP module {pull}15598[15598] +- Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275] *Packetbeat* From 7852d2b2ff48b28993c99b8905d41acc93c97da3 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 22:18:14 -0400 Subject: [PATCH 5/6] Add missing newline at end of metricbeat manifest. --- deploy/cloudfoundry/metricbeat/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/cloudfoundry/metricbeat/manifest.yml b/deploy/cloudfoundry/metricbeat/manifest.yml index 767c8b8fa08..40f0459b8da 100644 --- a/deploy/cloudfoundry/metricbeat/manifest.yml +++ b/deploy/cloudfoundry/metricbeat/manifest.yml @@ -7,4 +7,4 @@ applications: command: ./metricbeat -e -c ~/metricbeat.yml stack: cflinuxfs3 health-check-type: process - no-route: true \ No newline at end of file + no-route: true From 26db486ee9de067350502686158ed36a54fca483 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Thu, 26 Mar 2020 22:26:25 -0400 Subject: [PATCH 6/6] Fix filebeat being referenced in metricbeat docs. --- filebeat/docs/running-on-cloudfoundry.asciidoc | 4 ++-- metricbeat/docs/running-on-cloudfoundry.asciidoc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc index b661c2857ac..34c225ed831 100644 --- a/filebeat/docs/running-on-cloudfoundry.asciidoc +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -19,13 +19,13 @@ command will create the required credentials for connecting to loggregator. ["source", "sh"] ------------------------------------------------ -uaac client add filebeat --name filebeat --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +uaac client add {beatname_lc} --name {beatname_lc} --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only ------------------------------------------------ [WARNING] ======================================= *Use a unique secret:* The uaac command above is just an example and the secret should be changed and the -`filebeat.yml` should be updated with your choosen secret. +`{beatname_lc}.yml` should be updated with your choosen secret. ======================================= diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc index 731c8abf78d..e6c25d02587 100644 --- a/metricbeat/docs/running-on-cloudfoundry.asciidoc +++ b/metricbeat/docs/running-on-cloudfoundry.asciidoc @@ -19,13 +19,13 @@ command will create the required credentials for connecting to loggregator. ["source", "sh"] ------------------------------------------------ -uaac client add filebeat --name filebeat --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +uaac client add {beatname_lc} --name {beatname_lc} --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only ------------------------------------------------ [WARNING] ======================================= *Use a unique secret:* The uaac command above is just an example and the secret should be changed and the -`filebeat.yml` should be updated with your choosen secret. +`{beatname_lc}.yml` should be updated with your choosen secret. =======================================