diff --git a/.github/workflows/bosh-release.yml b/.github/workflows/bosh-release.yml index 9cababf815..5b9db6969e 100644 --- a/.github/workflows/bosh-release.yml +++ b/.github/workflows/bosh-release.yml @@ -25,7 +25,8 @@ jobs: - name: Check out the codebase. uses: actions/checkout@v3 - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@0c611abec1fc679030907286a2c084cc92fb8f2e + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@251cc053a43110ce4dea3cdb6f038147c82d0257 - name: Install BOSH CLI shell: bash run: | diff --git a/.github/workflows/tanzu-tile.yml b/.github/workflows/tanzu-tile.yml new file mode 100644 index 0000000000..3c15658641 --- /dev/null +++ b/.github/workflows/tanzu-tile.yml @@ -0,0 +1,58 @@ +name: Tanzu Tile + +# The workflow triggered by any change in deployments/cloudfoundry/bosh/ +# or /deployments/cloudfoundry/tile + +on: + pull_request: + paths: + - 'deployments/cloudfoundry/bosh/**' + - 'deployments/cloudfoundry/tile/**' + +permissions: + contents: write + +defaults: + run: + working-directory: 'deployments/cloudfoundry/tile' + +jobs: + + test: + name: Test Tanzu Tile creation + # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved + runs-on: ubuntu-20.04 + steps: + - name: Check out the codebase. + uses: actions/checkout@v3 + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Install BOSH CLI + shell: bash + run: | + brew install cloudfoundry/tap/bosh-cli + bosh -v + - name: Install PCF CLI + shell: bash + run: | + pip install tile-generator + - name: Run release script + shell: bash + run: | + ./make-latest-tile + tanzu_tile_regex="product/splunk-otel-collector-*.pivotal" + size="$(stat -c '%s' $tanzu_tile_regex)" + if [[ $size -eq 0 ]]; then + echo "File is empty!" >&2 + exit 1 + fi + + tanzu_tile_path=$(pwd)/$tanzu_tile_regex + echo "tanzu_tile_path=$tanzu_tile_path" >> $GITHUB_ENV + + - name: Uploading artifacts + uses: actions/upload-artifact@v3 + with: + name: tanzu-tile-latest.pivotal + path: ${{ env.tanzu_tile_path }} diff --git a/deployments/cloudfoundry/README.md b/deployments/cloudfoundry/README.md index d1396695f8..1217090a9f 100644 --- a/deployments/cloudfoundry/README.md +++ b/deployments/cloudfoundry/README.md @@ -1,5 +1,9 @@ # Splunk OpenTelemetry Collector Pivotal Cloud Foundry (PCF) Integrations +Supported Tanzu Application Service (TAS) versions: v2 + +Unsupported TAS versions: v3 + ### Cloud Foundry Buildpack This integration can be used to install and run the Collector as a sidecar to your app. @@ -7,5 +11,15 @@ In this configuration, the Collector will run in the same container as the app. ### Bosh Release -This is a Bosh release of the Splunk Collector. This deploys the Collector to the PCF -environment as a standalone deployment. \ No newline at end of file +This is a Bosh release of the Collector. This deploys the Collector to the PCF +environment as a standalone deployment. + +### Tanzu Tile + +This is a Tanzu Tile of the Collector, which is a packaged release of the collector +that can be integrated into the Ops Manager. The Tanzu Tile enables users to download, install, +run, configure, and update the collector all from the Ops Manager. + +[Tanzu Tile UI](./tile/resources/tanzu_tile_in_ops_mgr.png) + +[Tanzu Tile Configuration UI](./tile/resources/tanzu_tile_config_options.png) \ No newline at end of file diff --git a/deployments/cloudfoundry/bosh/.gitignore b/deployments/cloudfoundry/bosh/.gitignore index f288752d5e..7def1d8df8 100644 --- a/deployments/cloudfoundry/bosh/.gitignore +++ b/deployments/cloudfoundry/bosh/.gitignore @@ -1,6 +1,6 @@ latest-release.tgz -signalfx-agent.tar.gz splunk_otel_collector .dev_builds blobs -dev_releases \ No newline at end of file +dev_releases +bosh-env diff --git a/deployments/cloudfoundry/bosh/DEVELOPMENT.md b/deployments/cloudfoundry/bosh/DEVELOPMENT.md index 0abe11615c..62497ae67f 100644 --- a/deployments/cloudfoundry/bosh/DEVELOPMENT.md +++ b/deployments/cloudfoundry/bosh/DEVELOPMENT.md @@ -2,35 +2,65 @@ ## Getting Started -[Bosh Release Documentation (Overview)](https://bosh.io/docs/create-release/) +### Start the BOSH Director -[Quick Start Guide](https://bosh.io/docs/bosh-lite/) +[Local machine prerequisites](https://bosh.io/docs/quick-start/#prerequisites) +- The latest VirtualBox environment v7 is incompatible with this + functionality. Downgrade VirtualBox to [v6](https://www.virtualbox.org/wiki/Download_Old_Builds_6_1) to + ensure it can work. Tested successfully on `v6.1.42` + + +Automated start process: +```shell +# Sets up BOSH Director locally +make +# Makes sure local shell gets proper credentials to access director +source bosh-env/virtualbox/.envrc +``` -### Supplementary Getting Started Checklist +Manual start process: +[Follow the quick start guide to run a BOSH Director.](https://bosh.io/docs/quick-start/) -- Make sure director is running - - Quick start guide to run a BOSH Director can be found [here.](https://bosh.io/docs/quick-start/) -- Need to upload right Ubuntu/OS blob - - Guide can be found [here.](https://bosh.io/docs/uploading-stemcells/) - - The following is an example command to upload the Warden (BOSH Lite) Ubuntu Bionic (18.04.6 LTS) stemcell: +Common Errors: +- "Waiting for the agent on VM" timeouts + + - Run ```make reinstall-director``` + +Delete director: +```shell +make delete-director +``` +### Upload Ubuntu/OS blob - ```shell - $ bosh upload-stemcell --sha1 d44dc2d1b3f8415b41160ad4f82bc9d30b8dfdce \ - https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent?v=1.71 - ``` +Note: If you ran ```make``` successfully, you can skip this step. + +- [Uploading stemcells guide](https://bosh.io/docs/uploading-stemcells/) +- [Official BOSH stemcells references (including SHAs)](https://bosh.io/stemcells). +- The following is an example command to upload the Warden (BOSH Lite) Ubuntu Bionic (18.04.6 LTS) stemcell: + +```shell +bosh upload-stemcell --sha1 d44dc2d1b3f8415b41160ad4f82bc9d30b8dfdce \ +https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent?v=1.71 +```` + +### Supplemental Documentation Links + +[BOSH Release Documentation (Overview)](https://bosh.io/docs/create-release/) + +[Quick Start Guide](https://bosh.io/docs/bosh-lite/) ## Create Local BOSH Release ```shell -$ # Check release script for more environment variables that can be set. -$ export IS_DEV_RELEASE=1 -$ ./release +# Check release script for more environment variables that can be set. +export IS_DEV_RELEASE=1 +./release ``` ## BOSH Release Usage ```shell -$ bosh -d splunk-otel-collector deploy deployment.yaml +bosh -d splunk-otel-collector deploy deployment.yaml ``` Further explanation of the `deployment.yaml` file is found [here.](#deployment-config) @@ -106,6 +136,9 @@ $ bosh deployments # View all bosh VMs. This will show if a deployment's VM is running or failed. $ bosh vms +# View debug logs for a task (e.g. upload or deploy) that failed +$ bosh task task_number --debug + # View logs for a given deployment. Downloads a TAR file from the deployment. $ bosh logs -d diff --git a/deployments/cloudfoundry/bosh/Makefile b/deployments/cloudfoundry/bosh/Makefile new file mode 100644 index 0000000000..39070283d5 --- /dev/null +++ b/deployments/cloudfoundry/bosh/Makefile @@ -0,0 +1,29 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Explicitly define the shell we will use for commands. +SHELL?=/bin/bash + +.DEFAULT_GOAL=install-director + +.PHONY: delete-director +delete-director: + ./delete-director.sh + +.PHONY: install-director +install-director: + ./install-director.sh + +.PHONY: reinstall-director +reinstall-director: delete-director install-director diff --git a/deployments/cloudfoundry/bosh/config/blobs.yml b/deployments/cloudfoundry/bosh/config/blobs.yml deleted file mode 100644 index f2e7226175..0000000000 --- a/deployments/cloudfoundry/bosh/config/blobs.yml +++ /dev/null @@ -1,6 +0,0 @@ -signalfx_agent/signalfx-agent.tar.gz: - size: 194032255 - sha: sha256:40ca24994f792d4b7c40f8640bea1fc420fbfe5eed918b4d63b4293e3b1e07dd -splunk_otel_collector/splunk_otel_collector: - size: 163463346 - sha: sha256:db0ddd022c2b8f4fa00d3d8f9764fcee4cc1de9fb2da32b1f7d536b677c89261 diff --git a/deployments/cloudfoundry/bosh/delete-director.sh b/deployments/cloudfoundry/bosh/delete-director.sh new file mode 100755 index 0000000000..0f7d0cd439 --- /dev/null +++ b/deployments/cloudfoundry/bosh/delete-director.sh @@ -0,0 +1,28 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -euo pipefail + +BOSH_DIRECTOR_DIR="./bosh-env/virtualbox" +BOSH_DIRECTOR_DEPLOYMENT_DIR="bosh-deployment" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ -d $BOSH_DIRECTOR_DIR ] && [ -d $BOSH_DIRECTOR_DIR/$BOSH_DIRECTOR_DEPLOYMENT_DIR ]; then + cd $BOSH_DIRECTOR_DIR + ./$BOSH_DIRECTOR_DEPLOYMENT_DIR/virtualbox/delete-env.sh + cd $SCRIPT_DIR + rm -rf $BOSH_DIRECTOR_DIR +fi \ No newline at end of file diff --git a/deployments/cloudfoundry/bosh/install-director.sh b/deployments/cloudfoundry/bosh/install-director.sh new file mode 100755 index 0000000000..151b29969d --- /dev/null +++ b/deployments/cloudfoundry/bosh/install-director.sh @@ -0,0 +1,43 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +set -euo pipefail + +BOSH_DIRECTOR_DIR="./bosh-env/virtualbox" +BOSH_DIRECTOR_DEPLOYMENT_DIR="bosh-deployment" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ -d $BOSH_DIRECTOR_DIR/$BOSH_DIRECTOR_DEPLOYMENT_DIR ]; then + echo "Bosh director already exists" + echo "If you want to build again, run make reinstall-director" + exit 0 +fi + +mkdir -p $BOSH_DIRECTOR_DIR +cd $BOSH_DIRECTOR_DIR + +git clone --depth 1 https://github.com/cloudfoundry/bosh-deployment.git + +./$BOSH_DIRECTOR_DEPLOYMENT_DIR/virtualbox/create-env.sh + +source .envrc +bosh -e vbox env + +bosh -e vbox update-cloud-config bosh-deployment/warden/cloud-config.yml +bosh upload-stemcell --sha1 d44dc2d1b3f8415b41160ad4f82bc9d30b8dfdce \ +https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-bionic-go_agent?v=1.71 + +cd $SCRIPT_DIR \ No newline at end of file diff --git a/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/spec b/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/spec index fb2a2a9167..975ec49c5f 100644 --- a/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/spec +++ b/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/spec @@ -8,7 +8,6 @@ templates: packages: - splunk_otel_collector -- signalfx_agent properties: diff --git a/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/templates/ctl.erb b/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/templates/ctl.erb index d003c6466f..ea6038a232 100644 --- a/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/templates/ctl.erb +++ b/deployments/cloudfoundry/bosh/jobs/splunk-otel-collector/templates/ctl.erb @@ -13,7 +13,9 @@ case $1 in echo $$ > $PIDFILE - exec /var/vcap/packages/splunk_otel_collector/splunk_otel_collector \ + # Collector needs to be run as user that has permissions to get metrics from RLP + # gateway + exec chpst -u vcap:vcap /var/vcap/packages/splunk_otel_collector/splunk_otel_collector \ --config /var/vcap/jobs/splunk-otel-collector/bin/config/otel-collector-config.yaml \ >> $LOG_DIR/splunk-otel-collector.stdout.log \ 2>> $LOG_DIR/splunk-otel-collector.stderr.log diff --git a/deployments/cloudfoundry/bosh/packages/signalfx_agent/packaging b/deployments/cloudfoundry/bosh/packages/signalfx_agent/packaging deleted file mode 100644 index 0fbdefb380..0000000000 --- a/deployments/cloudfoundry/bosh/packages/signalfx_agent/packaging +++ /dev/null @@ -1,16 +0,0 @@ -set -e -x - -tar xzf signalfx_agent/signalfx-agent.tar.gz - -cp -r signalfx-agent/* ${BOSH_INSTALL_TARGET}/ - -pushd $BOSH_INSTALL_TARGET - -./bin/patch-interpreter $(pwd) - -rm -f ./bin/signalfx-agent \ - ./bin/agent-status - -popd - -rm -f signalfx_agent/signalfx-agent.tar.gz diff --git a/deployments/cloudfoundry/bosh/packages/signalfx_agent/spec b/deployments/cloudfoundry/bosh/packages/signalfx_agent/spec deleted file mode 100644 index 2d544d5bdb..0000000000 --- a/deployments/cloudfoundry/bosh/packages/signalfx_agent/spec +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: signalfx_agent - -dependencies: [] - -files: - - signalfx_agent/signalfx-agent.tar.gz # From https://github.com/signalfx/signalfx-agent/releases diff --git a/deployments/cloudfoundry/bosh/release b/deployments/cloudfoundry/bosh/release index a78ddc5a0a..fb362eacf5 100755 --- a/deployments/cloudfoundry/bosh/release +++ b/deployments/cloudfoundry/bosh/release @@ -1,3 +1,17 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash set -euo pipefail @@ -11,9 +25,6 @@ set -x # - OTEL_VERSION: Version of the Splunk OpenTelemetry Collector to deploy as a part of this release. # Default: "latest". Valid version must be >=0.48.0. -# - SMART_AGENT_VERSION: Version of the Smart Agent to download. This is a dependency of the Splunk Collector -# signalfx receiver. Default: "latest". - # - BOSH_RELEASE_VERSION: Version of the Bosh release being created. Defaults to 0 if not specified, so any release # except the first should have a pre-set value. @@ -44,30 +55,12 @@ fi OTEL_BINARY_DOWNLOAD_URL=${OTEL_BASE_URL}/download/${OTEL_VERSION}/otelcol_${OS} -SMART_AGENT_VERSION="${SMART_AGENT_VERSION:-latest}" -SMART_AGENT_BASE_URL="https://github.com/signalfx/signalfx-agent/releases" -SMART_AGENT="signalfx_agent" -SMART_AGENT_TAR="signalfx-agent.tar.gz" - -if [ $SMART_AGENT_VERSION = "latest" ]; then - SMART_AGENT_VERSION=$( wget -qO- --header="Accept: application/json" "${SMART_AGENT_BASE_URL}/latest" | jq -r '.tag_name' ) - if [ -z "$SMART_AGENT_VERSION" ]; then - echo "Failed to get tag_name for latest release from $SMART_AGENT_BASE_URL/latest" >&2 - exit 1 - fi -fi - -SMART_AGENT_VERSIONED_TAR=signalfx-agent-${SMART_AGENT_VERSION#v}.tar.gz -SMART_AGENT_DOWNLOAD_URL=$SMART_AGENT_BASE_URL/download/${SMART_AGENT_VERSION}/$SMART_AGENT_VERSIONED_TAR - -# Download Splunk's distribution of the OpenTelemetry Collector and the SignalFx Agent locally +# Download Splunk's distribution of the OpenTelemetry Collector wget -nv -O "$SCRIPT_DIR/$OTEL_BINARY" $OTEL_BINARY_DOWNLOAD_URL -wget -nv -O "$SCRIPT_DIR/$SMART_AGENT_TAR" $SMART_AGENT_DOWNLOAD_URL bosh init-release # blobs are the way Bosh releases handle external dependencies. bosh add-blob ${SCRIPT_DIR}/${OTEL_BINARY} ${OTEL_BINARY}/${OTEL_BINARY} -bosh add-blob ${SCRIPT_DIR}/${SMART_AGENT_TAR} ${SMART_AGENT}/${SMART_AGENT_TAR} if (( IS_DEV_RELEASE )) then @@ -81,5 +74,6 @@ else # If we're doing an official release we want to delete dev releases, and create a tarball of the release # to be used by the tile. rm -rf $SCRIPT_DIR/.dev_builds $SCRIPT_DIR/dev_releases - bosh create-release --force --version ${OTEL_VERSION}-${BOSH_RELEASE_VERSION} --tarball ./latest-release.tgz --name splunk-otel-collector + bosh create-release --force --version ${OTEL_VERSION}-${BOSH_RELEASE_VERSION} --tarball ./latest-release.tgz --name splunk-otel-collector + rm -rf $SCRIPT_DIR/splunk_otel_collector fi diff --git a/deployments/cloudfoundry/tile/.gitignore b/deployments/cloudfoundry/tile/.gitignore new file mode 100644 index 0000000000..e62e4877ec --- /dev/null +++ b/deployments/cloudfoundry/tile/.gitignore @@ -0,0 +1,3 @@ +product/ +release/ +tile-history.yml diff --git a/deployments/cloudfoundry/tile/DEVELOPMENT.MD b/deployments/cloudfoundry/tile/DEVELOPMENT.MD new file mode 100644 index 0000000000..33e2c287ed --- /dev/null +++ b/deployments/cloudfoundry/tile/DEVELOPMENT.MD @@ -0,0 +1,88 @@ +# Development Guide + +## VMware Tanzu Tile Documentation + +[Tanzu Tile Introduction](https://docs.pivotal.io/tiledev/2-10/tile-basics.html) + +[How Tiles Work](https://docs.pivotal.io/tiledev/2-10/tile-structure.html) + +## Tile Software Dependencies + +[Tile Generator](https://docs.pivotal.io/tiledev/2-10/tile-generator.html) + +[PCF CLI](https://docs.pivotal.io/tiledev/2-10/pcf-command.html) + +## Development Workflow + +### Environment setup + +- Refer to [this guide](https://github.com/signalfx/signalfx-agent/tree/main/pkg/monitors/cloudfoundry) +on how to setup the Tanzu environment and local authentication information. +- A local file in your working directory is required for PCF commands to work. This file must be named `metadata`. +Example contents: +```yaml +--- +# Values are found in the hammer file from Self Service's "ops_manager" key +opsmgr: + url: https://pcf..cf-app.com + username: pivotalcf + password: plain_text_password +``` + +### CLI Commands +```shell +$ ./make-latest-tile +# Alternatively, if just making changes to the tile config without touching the BOSH release, you can just run "tile build" +# instead of the whole make-latest-tile script + +$ pcf import product/splunk-otel-collector-.pivotal +$ pcf install splunk-otel-collector + +# Optional: Use a configuration file to set tile variables instead of manually in the Ops Manager UI +$ pcf configure splunk-otel-collector tile_config.yaml +``` + +Sample `tile_config.yaml` file contents: +```yaml +--- +cloudfoundry_rlp_gateway_shard_id: "opentelemetry" +cloudfoundry_rlp_gateway_tls_insecure_skip_verify: true +# Note: UAA credentials are from the UAA user created in the Tanzu service setup referenced +cloudfoundry_uaa_password: { 'secret': } +cloudfoundry_uaa_username: +cloudfoundry_uaa_tls_insecure_skip_verify: true +splunk_access_token: +splunk_realm: us0 +otel_proxy_http: "" +otel_proxy_https: "" +otel_proxy_no: "" +splunk_api_url: "" +splunk_ingest_url: "" +``` + +Once tile is installed and configured you can go to the Ops Manager in your browser. Confirm your +`Splunk Opentelemetry Collector` tile is there, green, and the correct version. Select `Review Pending Changes` -> +Check box for staged changes on your tile -> `APPLY CHANGES` + +Once changes are successfully applied, you should see data populating the charts in the Splunk Cloud Observability Suite + +### Debugging Common Issues + +1. Tile fails to start. The most common issue is UAA authentication errors or an incorrect Splunk access token. + - Check logs found under the `CHANGE LOGS` header within the Tanzu Ops Manager. + - Check logs for the tile itself + - Click into tile + - Check `Logs` tab, download if possible. + - If no logs are found, go to `Status` tab, and click the download button in the `Logs` column. You can download + the logs from the `Logs` tab once the download is finished. + +2. Tile shows up as running, but charts aren't populating data. This is most likely a metric naming mismatch. TAS v3.0+ +is currently unsupported due to metric name format changes. + - Check logs to make sure no errors are showing up + - Check metrics manually coming from Tanzu to see if they match charts. + - Follow steps to + [Access Metrics Using the Firehose PLugin](https://docs.pivotal.io/application-service/2-13/loggregator/data-sources.html#cf-nozzle) + - You can use the `hammer` login command instead of `cf login` + - Example metric filter command: + ```cf nozzle -no-filter | grep -i mem | grep -i percent``` + - Compare output of CLI metric filter with chart variables in the Cloud foundry dashboard. \ No newline at end of file diff --git a/deployments/cloudfoundry/tile/README.md b/deployments/cloudfoundry/tile/README.md new file mode 100644 index 0000000000..f55fdf70a8 --- /dev/null +++ b/deployments/cloudfoundry/tile/README.md @@ -0,0 +1,26 @@ +# Splunk OpenTelemetry Collector Tanzu Tile + +This directory is used to generate a Tanzu tile of the [Splunk OpenTelemetry Collector](https://github.com/signalfx/splunk-otel-collector). +The Tanzu tile uses the BOSH release to deploy the collector as a [loggregator firehose nozzle](https://docs.pivotal.io/tiledev/2-2/nozzle.html). + +## Dependencies + +The `release` script requires: + +- `jq` +- `wget` +- [Bosh CLI](https://bosh.io/docs/cli-v2-install/) +- [Tile Generator](https://docs.pivotal.io/tiledev/2-10/tile-generator.html) + +## Releasing + +```shell +$ ./make-latest-tile +``` +This command will create the BOSH release, and package it as a dependency for the tile. It will then generate the +tile with the same version as the collector. If successful, the tile will be found here: +`./product/splunk-otel-collector-.pivotal` + +## Development and Configuration + +Refer to the [Development Guide](./DEVELOPMENT.md) for more information. diff --git a/deployments/cloudfoundry/tile/make-latest-tile b/deployments/cloudfoundry/tile/make-latest-tile new file mode 100755 index 0000000000..462812fa06 --- /dev/null +++ b/deployments/cloudfoundry/tile/make-latest-tile @@ -0,0 +1,51 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +# This script will create a release of the bosh release of the collector, +# and generate the tile with the bosh release as a dependency. + +set -euo pipefail +set -x + +OTEL_VERSION=${OTEL_VERSION:-latest} +OTEL_BINARY="splunk_otel_collector" +OTEL_BASE_URL="https://github.com/signalfx/splunk-otel-collector/releases" + +if [ $OTEL_VERSION = "latest" ]; then + OTEL_VERSION=$( wget -qO- --header="Accept: application/json" "${OTEL_BASE_URL}/latest" | jq -r '.tag_name' ) + if [ -z "$OTEL_VERSION" ]; then + echo "Failed to get tag_name for latest release from $OTEL_VERSION/latest" >&2 + exit 1 + fi +fi + +# Export so bosh release is guaranteed to use the same version of the collector +export OTEL_VERSION + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +mkdir -p $SCRIPT_DIR/resources + +# Create bosh release and move it to be accessible by tile +(cd $SCRIPT_DIR/../bosh; ./release) +mv $SCRIPT_DIR/../bosh/latest-release.tgz $SCRIPT_DIR/resources/ + +# Tile version needs to be of the form "X.Y.Z", OTEL_VERSION is currently "vX.Y.Z" +# This command removes the "v" from the beginning. +OTEL_VERSION="${OTEL_VERSION:1}" + +# Build new version of tile +rm -f $SCRIPT_DIR/tile-history.yml +tile build ${OTEL_VERSION} diff --git a/deployments/cloudfoundry/tile/resources/icon.png b/deployments/cloudfoundry/tile/resources/icon.png new file mode 100644 index 0000000000..e8f512585d Binary files /dev/null and b/deployments/cloudfoundry/tile/resources/icon.png differ diff --git a/deployments/cloudfoundry/tile/resources/tanzu_tile_config_options.png b/deployments/cloudfoundry/tile/resources/tanzu_tile_config_options.png new file mode 100644 index 0000000000..76b0e66cab Binary files /dev/null and b/deployments/cloudfoundry/tile/resources/tanzu_tile_config_options.png differ diff --git a/deployments/cloudfoundry/tile/resources/tanzu_tile_in_ops_mgr.png b/deployments/cloudfoundry/tile/resources/tanzu_tile_in_ops_mgr.png new file mode 100644 index 0000000000..e567a20cfc Binary files /dev/null and b/deployments/cloudfoundry/tile/resources/tanzu_tile_in_ops_mgr.png differ diff --git a/deployments/cloudfoundry/tile/tile.yml b/deployments/cloudfoundry/tile/tile.yml new file mode 100644 index 0000000000..ed798d6715 --- /dev/null +++ b/deployments/cloudfoundry/tile/tile.yml @@ -0,0 +1,132 @@ +--- +# The high-level description of the tile. +# +name: splunk-otel-collector # By convention lowercase with dashes +icon_file: resources/icon.png +label: Splunk OpenTelemetry Collector for Tanzu Network +description: | + This tile contains a BOSH Release of the Splunk distribution of OpenTelemetry Collector, + and a firehose nozzle job that sends metrics to the Splunk Observability Cloud. + + The firehose nozzle pulls events off of the Loggregator Firehose via the RLP + Gateway and sends them to the Splunk Observability Cloud. This uses the + `cloudfoundry` receiver from the Splunk Opentelemetry Collector under the + hood. + + This also includes a BOSH release of the Splunk OpenTelemetry Collector, called "splunk-otel-collector" + that can be used in deployments to put the collector on individual VMs. + +# Specify the packages to be included in the tile. +packages: + - name: splunk_collector_firehose_nozzle + type: bosh-release + path: resources/latest-release.tgz + jobs: + - name: splunk-otel-collector + instances: 1 + templates: + - name: splunk-otel-collector + release: splunk-otel-collector + properties: + cloudfoundry: + rlp_gateway: + endpoint: https://log-stream.(( ..cf.cloud_controller.system_domain.value )) + shard_id: (( .properties.cloudfoundry_rlp_gateway_shard_id.value )) + tls: + insecure_skip_verify: (( .properties.cloudfoundry_rlp_gateway_tls_insecure_skip_verify.value )) + uaa: + endpoint: https://uaa.(( ..cf.cloud_controller.system_domain.value )) + password: (( .properties.cloudfoundry_uaa_password.value )) + username: (( .properties.cloudfoundry_uaa_username.value )) + tls: + insecure_skip_verify: (( .properties.cloudfoundry_uaa_tls_insecure_skip_verify.value )) + otel: + proxy: + http: (( .properties.otel_proxy_http.value )) + https: (( .properties.otel_proxy_https.value )) + no: (( .properties.otel_proxy_no.value )) + splunk: + access_token: (( .properties.splunk_access_token.value )) + api_url: (( .properties.splunk_api_url.value )) + ingest_url: (( .properties.splunk_ingest_url.value )) + realm: (( .properties.splunk_realm.value )) + +# Forms allow users to input variable values for Splunk Collector Bosh Release and nozzle job +forms: +- name: nozzle_form + label: Nozzle Config + description: Splunk OpenTelemetry Collector nozzle configuration values + properties: + - name: cloudfoundry_rlp_gateway_shard_id + type: string + label: RLP gateway shard ID + description: Metrics are load balanced between receivers that use the same shard ID. Only use if multiple receivers must receive all metrics instead of balancing metrics between them. + + - name: cloudfoundry_rlp_gateway_tls_insecure_skip_verify + type: boolean + label: Skip TLS validation for RLP + description: Whether to skip TLS verification for the RLP gateway endpoint + default: false + + - name: cloudfoundry_uaa_username + type: string + label: UAA username + description: Name of the UAA user. This UAA user must have the client_credentials and refresh_token authorized grant types, and logs.admin authority. + optional: false + + - name: cloudfoundry_uaa_password + type: secret + label: UAA password + description: Password of the UAA user + optional: false + + - name: cloudfoundry_uaa_tls_insecure_skip_verify + type: boolean + label: Skip TLS validation for UAA + description: Whether to skip TLS verification for the UAA endpoint + default: false + + - name: otel_proxy_http + type: string + label: HTTP proxy URL + description: URL used as the proxy URL for HTTP requests unless overridden by otel_proxy_no + optional: true + + - name: otel_proxy_https + type: string + label: HTTPS proxy URL + description: URL used as the proxy for HTTPS requests unless overridden by otel_proxy_no + optional: true + + - name: otel_proxy_no + type: string + label: Proxy exclusions + description: Comma-separated hosts that should be excluded from proxying. Hosts should be in the form of an IP address prefix (regular or CIDR notation), a domain name, or a special DNS label (*). + optional: true + +- name: splunk_form + label: Splunk Observability Cloud + description: Splunk Observability Cloud configuration values + properties: + - name: splunk_access_token + type: string + label: Access token + optional: false + + - name: splunk_realm + type: string + label: Realm + description: Realm (e.g. 'us0', 'ap0'). This option will implicitly set the 'Ingest base URL' and 'API server base URL' config below. If 'Ingest base URL' and 'API server base URL' are explicitly set below, this option is ignored. + optional: true + + - name: splunk_api_url + type: string + label: API server base URL + description: The API server's base URL for Splunk Observability Cloud. This option takes precedence over the realm. + optional: true + + - name: splunk_ingest_url + type: string + label: Ingest base URL + description: The base URL for ingest to Splunk Observability Cloud. This option takes precedence over the realm. + optional: true