From 2ff51254a3c0ec2ddc01758e8cafe30d47f72d71 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 31 Mar 2020 09:49:23 -0400 Subject: [PATCH] Add documentation for deploying filebeat and metricbeat in Cloud Foundry. (#17275) (#17299) * Add running-on-cloudfoundry for filebeat. * Fix typo. * Add running on cloudfoundry for metricbeat. * Add changelog. * Add missing newline at end of metricbeat manifest. * Fix filebeat being referenced in metricbeat docs. (cherry picked from commit e29d92912123aa1dcc352ca3eaf42c953a3f47ef) --- CHANGELOG.next.asciidoc | 2 + deploy/cloudfoundry/README.md | 11 +++ deploy/cloudfoundry/filebeat/filebeat.yml | 54 +++++++++++++ deploy/cloudfoundry/filebeat/manifest.yml | 10 +++ deploy/cloudfoundry/metricbeat/manifest.yml | 10 +++ deploy/cloudfoundry/metricbeat/metricbeat.yml | 54 +++++++++++++ filebeat/docs/getting-started.asciidoc | 5 ++ .../docs/running-on-cloudfoundry.asciidoc | 80 +++++++++++++++++++ filebeat/docs/setting-up-running.asciidoc | 4 + metricbeat/docs/gettingstarted.asciidoc | 5 ++ .../docs/running-on-cloudfoundry.asciidoc | 80 +++++++++++++++++++ metricbeat/docs/setting-up-running.asciidoc | 4 + 12 files changed, 319 insertions(+) create mode 100644 deploy/cloudfoundry/README.md create mode 100644 deploy/cloudfoundry/filebeat/filebeat.yml create mode 100644 deploy/cloudfoundry/filebeat/manifest.yml create mode 100644 deploy/cloudfoundry/metricbeat/manifest.yml create mode 100644 deploy/cloudfoundry/metricbeat/metricbeat.yml create mode 100644 filebeat/docs/running-on-cloudfoundry.asciidoc create mode 100644 metricbeat/docs/running-on-cloudfoundry.asciidoc diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1ba2d962637..3f7154d67e8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -287,6 +287,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Filebeat Okta module. {pull}16362[16362] - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155] - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313] +- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] *Heartbeat* @@ -357,6 +358,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119] - Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319] +- Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275] *Packetbeat* diff --git a/deploy/cloudfoundry/README.md b/deploy/cloudfoundry/README.md new file mode 100644 index 00000000000..f3c35b7771b --- /dev/null +++ b/deploy/cloudfoundry/README.md @@ -0,0 +1,11 @@ +# Beats Cloud Foundry manifests examples + +## Getting started + +This is the list of officially supported Beats, with example manifests to run +them in Cloud Foundry: + +Beat | Description +---- | ---- +[filebeat](filebeat) | Ships logs from loggregator +[metricbeat](metricbeat) | Ships metrics from loggregator diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml new file mode 100644 index 00000000000..a6ddfaca04b --- /dev/null +++ b/deploy/cloudfoundry/filebeat/filebeat.yml @@ -0,0 +1,54 @@ +#=========================== Filebeat inputs ============================= + +# Configure the input to access loggregator to forward the log events. + +filebeat.inputs: + - &cloudfoundry + type: cloudfoundry + client_id: ${CLIENT_ID:filebeat} + client_secret: ${CLIENT_SECRET:changeme} + api_address: ${API_ADDRESS} + #doppler_address: ${DOPPLER_ADDRESS} + #uaa_address: ${UAA_ADDRESS} + #rlp_address: ${RLP_ADDRESS} + #shard_id: ${SHARD_ID} + + +#================================ Outputs ===================================== + +# Configure the Elasticsearch output either to a specific host or using +# Elastic Cloud. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["${ELASTICSEARCH_OUTPUT}:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +#----------------------------- Elastic Cloud ---------------------------------- + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Processors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_cloudfoundry_metadata: + <<: *cloudfoundry diff --git a/deploy/cloudfoundry/filebeat/manifest.yml b/deploy/cloudfoundry/filebeat/manifest.yml new file mode 100644 index 00000000000..26ec5a0b958 --- /dev/null +++ b/deploy/cloudfoundry/filebeat/manifest.yml @@ -0,0 +1,10 @@ +applications: +- name: filebeat + memory: 256M + instances: 1 + buildpacks: + - binary_buildpack + command: ./filebeat -e -c ~/filebeat.yml + stack: cflinuxfs3 + health-check-type: process + no-route: true diff --git a/deploy/cloudfoundry/metricbeat/manifest.yml b/deploy/cloudfoundry/metricbeat/manifest.yml new file mode 100644 index 00000000000..40f0459b8da --- /dev/null +++ b/deploy/cloudfoundry/metricbeat/manifest.yml @@ -0,0 +1,10 @@ +applications: + - name: metricbeat + memory: 256M + instances: 1 + buildpacks: + - binary_buildpack + command: ./metricbeat -e -c ~/metricbeat.yml + stack: cflinuxfs3 + health-check-type: process + no-route: true diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml new file mode 100644 index 00000000000..7beb7858872 --- /dev/null +++ b/deploy/cloudfoundry/metricbeat/metricbeat.yml @@ -0,0 +1,54 @@ +#=========================== Metricbeat modules ============================= + +# Configure the module to forward metrics from loggregator. + +metricbeat.modules: + - &cloudfoundry + module: cloudfoundry + client_id: ${CLIENT_ID} + client_secret: ${CLIENT_SECRET} + api_address: ${API_ADDRESS} + #doppler_address: ${DOPPLER_ADDRESS} + #uaa_address: ${UAA_ADDRESS} + #rlp_address: ${RLP_ADDRESS} + #shard_id: ${SHARD_ID} + + +#================================ Outputs ===================================== + +# Configure the Elasticsearch output either to a specific host or using +# Elastic Cloud. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["${ELASTICSEARCH_OUTPUT}:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +#----------------------------- Elastic Cloud ---------------------------------- + +# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Processors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_cloudfoundry_metadata: + <<: *cloudfoundry diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 8aa03b5595e..5b3972ad671 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -109,6 +109,11 @@ See <> for deploying Docker containers. See <> for deploying with Kubernetes. +[[cloudfoundry]] +*cloudfoundry:* + +See <> for deploying with Cloud Foundry. + [[win]] *win:* diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc new file mode 100644 index 00000000000..34c225ed831 --- /dev/null +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -0,0 +1,80 @@ +[[running-on-cloudfoundry]] +=== Running {beatname_uc} on Cloud Foundry + +You can use {beatname_uc} on Cloud Foundry to retrieve and ship logs. + +ifeval::["{release-state}"=="unreleased"] + +However, version {version} of {beatname_uc} has not yet been +released, no build is currently available for this version. + +endif::[] + + +[float] +==== Cloud Foundry credentials + +{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The uaac +command will create the required credentials for connecting to loggregator. + +["source", "sh"] +------------------------------------------------ +uaac client add {beatname_lc} --name {beatname_lc} --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +------------------------------------------------ + +[WARNING] +======================================= +*Use a unique secret:* The uaac command above is just an example and the secret should be changed and the +`{beatname_lc}.yml` should be updated with your choosen secret. +======================================= + + +[float] +==== Cloud Foundry deploy manifests + +You deploy {beatname_uc} as an application with no route. + +Cloud Foundry requires that 3 files exist inside of a directory to allow {beatname_uc} to be pushed. The commands +below provide the basic steps for getting it up and running. + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +cd {beatname_lc}-{version}-linux-x86_64 +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/{beatname_lc}.yml +# Update api_address, client_id, client_secret in downloaded {beatname_lc}. +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/manifest.yml +------------------------------------------------ + + +[float] +==== Deploy + +To deploy {beatname_uc} to Cloud Foundry, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +cf push +------------------------------------------------ + +To check the status, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ cf apps + +name requested state instances memory disk urls +filebeat started 1/1 256M 1G +------------------------------------------------ + +Log events should start flowing to Elasticsearch. The events are annotated with +metadata added by the <> processor. + + +[WARNING] +======================================= +*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that +{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` +will result in duplicate events being pushed to Elasticsearch. +======================================= diff --git a/filebeat/docs/setting-up-running.asciidoc b/filebeat/docs/setting-up-running.asciidoc index 61f952b94c3..5968b3f571d 100644 --- a/filebeat/docs/setting-up-running.asciidoc +++ b/filebeat/docs/setting-up-running.asciidoc @@ -26,6 +26,8 @@ This section includes additional information on how to set up and run * <> +* <> + * <> @@ -41,6 +43,8 @@ include::./running-on-docker.asciidoc[] include::./running-on-kubernetes.asciidoc[] +include::./running-on-cloudfoundry.asciidoc[] + include::{libbeat-dir}/shared-systemd.asciidoc[] include::{libbeat-dir}/shared-shutdown.asciidoc[] diff --git a/metricbeat/docs/gettingstarted.asciidoc b/metricbeat/docs/gettingstarted.asciidoc index d185dd69d64..b32cd1280be 100644 --- a/metricbeat/docs/gettingstarted.asciidoc +++ b/metricbeat/docs/gettingstarted.asciidoc @@ -119,6 +119,11 @@ See <> for deploying Docker containers. See <> for deploying with Kubernetes. +[[cloudfoundry]] +*cloudfoundry:* + +See <> for deploying with Cloud Foundry. + [[win]] *win:* diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc new file mode 100644 index 00000000000..e6c25d02587 --- /dev/null +++ b/metricbeat/docs/running-on-cloudfoundry.asciidoc @@ -0,0 +1,80 @@ +[[running-on-cloudfoundry]] +=== Running {beatname_uc} on Cloud Foundry + +You can use {beatname_uc} on Cloud Foundry to retrieve and ship metrics. + +ifeval::["{release-state}"=="unreleased"] + +However, version {version} of {beatname_uc} has not yet been +released, no build is currently available for this version. + +endif::[] + + +[float] +==== Cloud Foundry credentials + +{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The uaac +command will create the required credentials for connecting to loggregator. + +["source", "sh"] +------------------------------------------------ +uaac client add {beatname_lc} --name {beatname_lc} --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only +------------------------------------------------ + +[WARNING] +======================================= +*Use a unique secret:* The uaac command above is just an example and the secret should be changed and the +`{beatname_lc}.yml` should be updated with your choosen secret. +======================================= + + +[float] +==== Cloud Foundry deploy manifests + +You deploy {beatname_uc} as an application with no route. + +Cloud Foundry requires that 3 files exist inside of a directory to allow {beatname_uc} to be pushed. The commands +below provide the basic steps for getting it up and running. + +["source", "sh", subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +cd {beatname_lc}-{version}-linux-x86_64 +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/{beatname_lc}.yml +# Update api_address, client_id, client_secret in downloaded {beatname_lc}. +curl -L -O https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/cloudfoundry/{beatname_lc}/manifest.yml +------------------------------------------------ + + +[float] +==== Deploy + +To deploy {beatname_uc} to Cloud Foundry, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +cf push +------------------------------------------------ + +To check the status, run: + +["source", "sh", subs="attributes"] +------------------------------------------------ +$ cf apps + +name requested state instances memory disk urls +metricbeat started 1/1 256M 1G +------------------------------------------------ + +Metrics should start flowing to Elasticsearch. The events are annotated with +metadata added by the <> processor. + + +[WARNING] +======================================= +*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that +{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` +will result in duplicate events being pushed to Elasticsearch. +======================================= diff --git a/metricbeat/docs/setting-up-running.asciidoc b/metricbeat/docs/setting-up-running.asciidoc index 305d802fa11..002cd8bd4b1 100644 --- a/metricbeat/docs/setting-up-running.asciidoc +++ b/metricbeat/docs/setting-up-running.asciidoc @@ -26,6 +26,8 @@ This section includes additional information on how to set up and run * <> +* <> + * <> //MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too. @@ -40,6 +42,8 @@ include::./running-on-docker.asciidoc[] include::./running-on-kubernetes.asciidoc[] +include::./running-on-cloudfoundry.asciidoc[] + include::{libbeat-dir}/shared-systemd.asciidoc[] include::{libbeat-dir}/shared-shutdown.asciidoc[]