diff --git a/_attributes/common-attributes.adoc b/_attributes/common-attributes.adoc index 0c2a5c5d5fe1..8c2a6e0f109c 100644 --- a/_attributes/common-attributes.adoc +++ b/_attributes/common-attributes.adoc @@ -114,18 +114,19 @@ ifdef::openshift-origin[] :CNVSubscriptionSpecName: community-kubevirt-hyperconverged endif::[] //distributed tracing -:DTProductName: Red Hat OpenShift distributed tracing -:DTShortName: distributed tracing -:DTProductVersion: 2.8 -:JaegerName: Red Hat OpenShift distributed tracing platform -:JaegerShortName: distributed tracing platform -:JaegerVersion: 1.42.0 +:DTProductName: Red Hat OpenShift distributed tracing platform +:DTShortName: distributed tracing platform +:DTProductVersion: 2.9 +:JaegerName: Red Hat OpenShift distributed tracing platform (Jaeger) +:JaegerShortName: distributed tracing platform (Jaeger) +:JaegerVersion: 1.47.0 :OTELName: Red Hat OpenShift distributed tracing data collection :OTELShortName: distributed tracing data collection -:OTELVersion: 0.74.0 -:TempoName: Tempo Operator -:TempoShortName: Tempo -:TempoVersion: 0.1.0 +:OTELVersion: 0.81.0 +:TempoName: Red Hat OpenShift distributed tracing platform (Tempo) +:TempoShortName: distributed tracing platform (Tempo) +:TempoOperator: Tempo Operator +:TempoVersion: 2.1.1 //logging :logging-title: logging subsystem for Red Hat OpenShift :logging-title-uc: Logging subsystem for Red Hat OpenShift diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index e80d580574d4..620398f208f0 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -3538,19 +3538,40 @@ Topics: Topics: - Name: Distributed tracing architecture File: distr-tracing-architecture -- Name: Distributed tracing installation - Dir: distr_tracing_install - Topics: - - Name: Installing distributed tracing - File: distr-tracing-installing - - Name: Configuring the distributed tracing platform - File: distr-tracing-deploying-jaeger - - Name: Configuring distributed tracing data collection - File: distr-tracing-deploying-otel - - Name: Upgrading distributed tracing - File: distr-tracing-updating - - Name: Removing distributed tracing - File: distr-tracing-removing +- Name: Distributed tracing platform (Jaeger) + Dir: distr_tracing_jaeger + Topics: + - Name: Installing + File: distr-tracing-jaeger-installing + - Name: Configuring + File: distr-tracing-jaeger-configuring + - Name: Updating + File: distr-tracing-jaeger-updating + - Name: Removing + File: distr-tracing-jaeger-removing +- Name: Distributed tracing platform (Tempo) + Dir: distr_tracing_tempo + Topics: + - Name: Installing + File: distr-tracing-tempo-installing + - Name: Configuring + File: distr-tracing-tempo-configuring + - Name: Updating + File: distr-tracing-tempo-updating + - Name: Removing + File: distr-tracing-tempo-removing +- Name: Distributed tracing data collection (OpenTelemetry) + Dir: distr_tracing_otel + Topics: + - Name: Installing + File: distr-tracing-otel-installing + - Name: Configuring + File: distr-tracing-otel-configuring + - Name: Forwarding + File: distr-tracing-otel-forwarding + - Name: Migrating + File: distr-tracing-otel-migrating + --- Name: Virtualization Dir: virt diff --git a/distr_tracing/distr_tracing_install/distr-tracing-deploying-otel.adoc b/distr_tracing/distr_tracing_install/distr-tracing-deploying-otel.adoc deleted file mode 100644 index e1e19f61dc1e..000000000000 --- a/distr_tracing/distr_tracing_install/distr-tracing-deploying-otel.adoc +++ /dev/null @@ -1,11 +0,0 @@ -:_content-type: ASSEMBLY -[id="distr-tracing-deploying-otel"] -= Configuring and deploying distributed tracing data collection -include::_attributes/common-attributes.adoc[] -:context: deploying-distr-tracing-data-collection - -toc::[] - -The {OTELName} Operator uses a custom resource definition (CRD) file that defines the architecture and configuration settings to be used when creating and deploying the {OTELName} resources. You can either install the default configuration or modify the file to better suit your business requirements. - -include::modules/distr-tracing-config-otel-collector.adoc[leveloffset=+1] diff --git a/distr_tracing/distr_tracing_jaeger/_attributes b/distr_tracing/distr_tracing_jaeger/_attributes new file mode 120000 index 000000000000..20cc1dcb77bf --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc similarity index 96% rename from distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc rename to distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc index 4f5de2decc7c..9fcaefed0919 100644 --- a/distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc +++ b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc @@ -1,8 +1,8 @@ :_content-type: ASSEMBLY -[id="distr-tracing-deploying"] -= Configuring and deploying distributed tracing +[id="distr-tracing-jaeger-configuring"] += Configuring and deploying distributed tracing platform Jaeger include::_attributes/common-attributes.adoc[] -:context: deploying-distr-tracing-platform +:context: distr-tracing-jaeger-configuring toc::[] @@ -20,7 +20,6 @@ metadata: spec: strategy: production <1> ---- - <1> The {JaegerName} Operator currently supports the following deployment strategies: * *allInOne* (Default) - This strategy is intended for development, testing, and demo purposes; it is not intended for production use. The main backend components, Agent, Collector, and Query service, are all packaged into a single executable which is configured, by default. to use in-memory storage. @@ -47,7 +46,6 @@ The streaming deployment strategy is currently unsupported on {ibmzProductName}. [NOTE] ==== There are two ways to install and use {DTProductName}, as part of a service mesh or as a stand alone component. If you have installed {DTShortName} as part of {SMProductName}, you can perform basic configuration as part of the xref:../../service_mesh/v2x/installing-ossm.adoc#installing-ossm[ServiceMeshControlPlane] but for completely control you should configure a Jaeger CR and then xref:../../service_mesh/v2x/ossm-observability.adoc#ossm-config-external-jaeger_observability[reference your distributed tracing configuration file in the ServiceMeshControlPlane]. - ==== include::modules/distr-tracing-deploy-default.adoc[leveloffset=+1] @@ -74,8 +72,6 @@ include::modules/distr-tracing-config-default.adoc[leveloffset=+2] include::modules/distr-tracing-config-jaeger-collector.adoc[leveloffset=+2] -//include::modules/distr-tracing-config-otel-collector.adoc[leveloffset=+2] - include::modules/distr-tracing-config-sampling.adoc[leveloffset=+2] include::modules/distr-tracing-config-storage.adoc[leveloffset=+2] diff --git a/distr_tracing/distr_tracing_install/distr-tracing-installing.adoc b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc similarity index 87% rename from distr_tracing/distr_tracing_install/distr-tracing-installing.adoc rename to distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc index 9a64b8c418e9..707e993efe73 100644 --- a/distr_tracing/distr_tracing_install/distr-tracing-installing.adoc +++ b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc @@ -1,8 +1,8 @@ :_content-type: ASSEMBLY -[id="installing-distributed-tracing"] -= Installing distributed tracing +[id="dist-tracing-jaeger-installing"] += Installing distributed tracing platform Jaeger include::_attributes/common-attributes.adoc[] -:context: install-distributed-tracing +:context: dist-tracing-jaeger-installing toc::[] @@ -34,10 +34,3 @@ include::modules/distr-tracing-install-overview.adoc[leveloffset=+1] include::modules/distr-tracing-install-elasticsearch.adoc[leveloffset=+1] include::modules/distr-tracing-install-jaeger-operator.adoc[leveloffset=+1] - -include::modules/distr-tracing-install-otel-operator.adoc[leveloffset=+1] - -//// -== Next steps -* xref:../../distr_tracing/distr_tracing_install/distr-tracing-deploying.adoc#deploying-distributed-tracing[Deploy {DTProductName}]. -//// diff --git a/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-removing.adoc b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-removing.adoc new file mode 100644 index 000000000000..421e7851ceb4 --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-removing.adoc @@ -0,0 +1,31 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-jaeger-removing"] += Removing distributed tracing platform Jaeger +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-jaeger-removing + +toc::[] + +The steps for removing {DTProductName} from an {product-title} cluster are as follows: + +. Shut down any {DTProductName} pods. +. Remove any {DTProductName} instances. +. Remove the {JaegerName} Operator. +. Remove the {OTELName} Operator. + +include::modules/distr-tracing-removing-instance.adoc[leveloffset=+1] + +include::modules/distr-tracing-removing-instance-cli.adoc[leveloffset=+1] + + +== Removing the {DTProductName} Operators + +.Procedure + +. Follow the instructions for xref:../../operators/admin/olm-deleting-operators-from-cluster.adoc#olm-deleting-operators-from-a-cluster[Deleting Operators from a cluster]. + +* Remove the {JaegerName} Operator. + +//* Remove the {OTELName} Operator. + +* After the {JaegerName} Operator has been removed, if appropriate, remove the OpenShift Elasticsearch Operator. diff --git a/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-updating.adoc b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-updating.adoc new file mode 100644 index 000000000000..ab2f4938aef4 --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-updating.adoc @@ -0,0 +1,24 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-jaeger-updating"] += Updating distributed tracing platform Jaeger +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-jaeger-updating + +toc::[] + +Operator Lifecycle Manager (OLM) controls the installation, upgrade, and role-based access control (RBAC) of Operators in a cluster. The OLM runs by default in {product-title}. +OLM queries for available Operators as well as upgrades for installed Operators. +For more information about how {product-title} handles upgrades, see the xref:../../operators/understanding/olm/olm-understanding-olm.adoc#olm-understanding-olm[Operator Lifecycle Manager] documentation. + +During an update, the {DTProductName} Operators upgrade the managed {DTShortName} instances to the version associated with the Operator. Whenever a new version of the {JaegerName} Operator is installed, all the {JaegerShortName} application instances managed by the Operator are upgraded to the Operator's version. For example, after upgrading the Operator from 1.10 installed to 1.11, the Operator scans for running {JaegerShortName} instances and upgrades them to 1.11 as well. + +For specific instructions on how to update the OpenShift Elasticsearch Operator, see xref:../../logging/cluster-logging-upgrading.adoc#cluster-logging-upgrading_cluster-logging-upgrading[Updating OpenShift Logging]. + +include::modules/distr-tracing-change-operator-20.adoc[leveloffset=+1] + +[IMPORTANT] +==== +If you have not already updated your OpenShift Elasticsearch Operator as described in xref:../../logging/cluster-logging-upgrading.adoc[Updating OpenShift Logging] complete that update before updating your {JaegerName} Operator. +==== + +For instructions on how to update the Operator channel, see xref:../../operators/admin/olm-upgrading-operators.adoc[Updating installed Operators]. diff --git a/distr_tracing/distr_tracing_jaeger/images b/distr_tracing/distr_tracing_jaeger/images new file mode 120000 index 000000000000..e4c5bd02a10a --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/images @@ -0,0 +1 @@ +../images/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_jaeger/modules b/distr_tracing/distr_tracing_jaeger/modules new file mode 120000 index 000000000000..43aab75b53c9 --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/modules @@ -0,0 +1 @@ +../modules/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_jaeger/snippets b/distr_tracing/distr_tracing_jaeger/snippets new file mode 120000 index 000000000000..5a3f5add140e --- /dev/null +++ b/distr_tracing/distr_tracing_jaeger/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_otel/_attributes b/distr_tracing/distr_tracing_otel/_attributes new file mode 120000 index 000000000000..20cc1dcb77bf --- /dev/null +++ b/distr_tracing/distr_tracing_otel/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_otel/distr-tracing-otel-configuring.adoc b/distr_tracing/distr_tracing_otel/distr-tracing-otel-configuring.adoc new file mode 100644 index 000000000000..562dc205f5b5 --- /dev/null +++ b/distr_tracing/distr_tracing_otel/distr-tracing-otel-configuring.adoc @@ -0,0 +1,18 @@ +:_content-type: ASSEMBLY +[id="distr-tracing-otel-configuring"] += Configuring and deploying distributed tracing data collection OpenTelemetry +include::_attributes/common-attributes.adoc[] +:context: distr-tracing-otel-configuring + +toc::[] + +The {OTELName} Operator uses a custom resource definition (CRD) file that defines the architecture and configuration settings to be used when creating and deploying the {OTELName} resources. You can either install the default configuration or modify the file to better suit your business requirements. + +include::modules/distr-tracing-config-otel-collector.adoc[leveloffset=+1] + +include::modules/distr-tracing-config-otel-send-metrics-monitoring-stack.adoc[leveloffset=+1] + +[role="_additional-resources"] +[id="additional-resources_deploy-otel"] +== Additional resources +* xref:../../monitoring/enabling-monitoring-for-user-defined-projects.adoc[Enabling monitoring for user-defined projects] diff --git a/distr_tracing/distr_tracing_otel/distr-tracing-otel-forwarding.adoc b/distr_tracing/distr_tracing_otel/distr-tracing-otel-forwarding.adoc new file mode 100644 index 000000000000..4a8e496d0534 --- /dev/null +++ b/distr_tracing/distr_tracing_otel/distr-tracing-otel-forwarding.adoc @@ -0,0 +1,136 @@ +:_content-type: PROCEDURE +[id="distr-tracing-otel-forwarding"] += Forwarding traces to TempoStack by using OpenTelemetry Collector +include::_attributes/common-attributes.adoc[] +:context: distr-tracing-otel-forwarding + +To configure log forwarding to the TempoStack, you deploy and configure an OpenTelemetry collector. + +.Prerequisites + +* Tempo Operator is installed. +* TempoStack is deployed on the cluster. +* Red Hat OpenShift distributed tracing data collection operator is installed. + +.Procedure + +For the following steps, we will deploy an OpenTelemetry collector using the recomended processors/receivers/explorters, this example will deploy the collector in deployment mode, for other modes see the OpenTelemetry collector documentation. + +. Create a service account for the OpenTelemetry collector, here is an example: ++ +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector-deployment +---- + +. Create a cluster role, this cluster role will be associated with the service account created in the first step: ++ +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector +rules: +# Permissions for pods and namespaces resources are needed for the k8sattributesprocessor +# Permissions for infrastructures and infrastructures/status are needed for resourcedetectionprocessor +- apiGroups: ["", "config.openshift.io"] + resources: ["pods", "namespaces", "infrastructures", "infrastructures/status"] + verbs: ["get", "watch", "list"] +---- + +Some permissions are neccesary to make the `k8sattributesprocessor` and `resourcedetectionprocessor` works, and those are recomended. + +. Bind the cluster role to the service account: ++ +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector +subjects: +- kind: ServiceAccount + name: otel-collector-deployment + namespace: otel-collector-example +roleRef: + kind: ClusterRole + name: otel-collector + apiGroup: rbac.authorization.k8s.io +---- + +. Create the YAML file that will define the `OpenTelemetryCollector` custom resource (CR): ++ +---- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: otel +spec: + mode: deployment + serviceAccount: otel-collector-deployment + config: | + receivers: + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: + opencensus: + otlp: + protocols: + grpc: + http: + zipkin: + processors: + batch: + k8sattributes: + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + resourcedetection: + detectors: [openshift] + exporters: + otlp: + endpoint: "tempo-simplest-distributor:4317" # Endpoint for the tempo distributor. + tls: + insecure: true + service: + pipelines: + traces: + receivers: [jaeger, opencensus, otlp, zipkin] + processors: [memory_limiter, k8sattributes, resourcedetection, batch] + exporters: [otlp] +---- + +The collector is configured with a receiver for Jaeger traces, OpenCensus traces over the OpenCensus protocol, Zipkin traces over the Zipkin protocol and OTLP traces over the GRPC protocol. The collector exporter is configured to export OTLP, and is pointing to the tempo distributor endpoint, in this case "tempo-simplest-distributor:4317" which should be already created. + +You can find more examples of how to deploy OpenTelemetry with tempo in sidecar mode and multitenant here: https://github.com/os-observability/redhat-rhosdt-samples + +[TIP] +==== +You can deploy the `tracegen` as a test: +---- +apiVersion: batch/v1 +kind: Job +metadata: + name: tracegen +spec: + template: + spec: + containers: + - name: tracegen + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/tracegen:latest + command: + - "./tracegen" + args: + - -otlp-endpoint=otel-collector:4317 + - -otlp-insecure + - -duration=30s + - -workers=1 + restartPolicy: Never + backoffLimit: 4 +---- +==== diff --git a/distr_tracing/distr_tracing_otel/distr-tracing-otel-installing.adoc b/distr_tracing/distr_tracing_otel/distr-tracing-otel-installing.adoc new file mode 100644 index 000000000000..736c83b14f76 --- /dev/null +++ b/distr_tracing/distr_tracing_otel/distr-tracing-otel-installing.adoc @@ -0,0 +1,26 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-otel-installing"] += Installing distributed tracing data collection OpenTelemetry +include::_attributes/common-attributes.adoc[] +:context: install-distributed-tracing-otel + +toc::[] + +== Prerequisites + +Before you can install {DTProductName}, review the installation activities, and ensure that you meet the prerequisites: + +* Possess an active {product-title} subscription on your Red Hat account. If you do not have a subscription, contact your sales representative for more information. + +* Review the xref:../../architecture/architecture-installation.adoc#installation-overview_architecture-installation[{product-title} {product-version} overview]. +* Install {product-title} {product-version}. + +** xref:../../installing/installing_aws/installing-aws-account.adoc#installing-aws-account[Install {product-title} {product-version} on AWS] +** xref:../../installing/installing_aws/installing-aws-user-infra.adoc#installing-aws-user-infra[Install {product-title} {product-version} on user-provisioned AWS] +** xref:../../installing/installing_bare_metal/installing-bare-metal.adoc#installing-bare-metal[Install {product-title} {product-version} on bare metal] +** xref:../../installing/installing_vsphere/installing-vsphere.adoc#installing-vsphere[Install {product-title} {product-version} on vSphere] +* Install the version of the OpenShift CLI (`oc`) that matches your {product-title} version and add it to your path. + +* An account with the `cluster-admin` role. + +include::modules/distr-tracing-install-otel-operator.adoc[leveloffset=+1] diff --git a/distr_tracing/distr_tracing_otel/distr-tracing-otel-migrating.adoc b/distr_tracing/distr_tracing_otel/distr-tracing-otel-migrating.adoc new file mode 100644 index 000000000000..1e3ba972446b --- /dev/null +++ b/distr_tracing/distr_tracing_otel/distr-tracing-otel-migrating.adoc @@ -0,0 +1,14 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-otel-migrating"] += Migrating from {JaegerName} to {OTELName} +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-otel-migrating + +toc::[] + +This migration guide is intended for customers who have already instrumented their applications using Jaeger for distributed tracing and are now looking to transition to OpenTelemetry. OpenTelemetry is an open-source project that provides a set of APIs, libraries, agents, and instrumentation to facilitate observability in distributed systems. Fortunately, you don't need to change the SDKs in your applications, as the OpenTelemetry Collector can ingest the Jaeger protocol. This guide will walk you through the process of configuring the OpenTelemetry Collector and your applications to report traces seamlessly. + +include::modules/distr-tracing-otel-migrating-from-jaeger-with-sidecars.adoc[leveloffset=+1] + +include::modules/distr-tracing-otel-migrating-from-jaeger-without-sidecars.adoc[leveloffset=+1] + diff --git a/distr_tracing/distr_tracing_otel/images b/distr_tracing/distr_tracing_otel/images new file mode 120000 index 000000000000..e4c5bd02a10a --- /dev/null +++ b/distr_tracing/distr_tracing_otel/images @@ -0,0 +1 @@ +../images/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_otel/modules b/distr_tracing/distr_tracing_otel/modules new file mode 120000 index 000000000000..43aab75b53c9 --- /dev/null +++ b/distr_tracing/distr_tracing_otel/modules @@ -0,0 +1 @@ +../modules/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_otel/snippets b/distr_tracing/distr_tracing_otel/snippets new file mode 120000 index 000000000000..5a3f5add140e --- /dev/null +++ b/distr_tracing/distr_tracing_otel/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_tempo/_attributes b/distr_tracing/distr_tracing_tempo/_attributes new file mode 120000 index 000000000000..20cc1dcb77bf --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-configuring.adoc b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-configuring.adoc new file mode 100644 index 000000000000..2db76d753c47 --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-configuring.adoc @@ -0,0 +1,94 @@ +:_content-type: ASSEMBLY +[id="distr-tracing-tempo-configuring"] += Configuring and deploying distributed tracing platform Tempo +include::_attributes/common-attributes.adoc[] +:context: distr-tracing-tempo-configuring + +toc::[] + +The {TempoName} Operator uses a custom resource definition (CRD) file that defines the architecture and configuration settings to be used when creating and deploying the {TempoShortName} resources. You can either install the default configuration or modify the file to better suit your business requirements. + +{TempoName} has predefined deployment strategies. You specify a deployment strategy in the custom resource file. When you create a {TempoShortName} instance the Operator uses this configuration file to create the objects necessary for the deployment. + +.Tempo custom resource file showing deployment strategy +[source,yaml] +---- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoStack +metadata: + name: simple + namespace: tempo-example +spec: + storage: + secret: + # Name of the secret with the information to access the object storage + name: tempostack-odf + type: s3 + storageSize: 1Gi + template: + queryFrontend: + jaegerQuery: + # Enable the Jaeger UI + enabled: true +---- + +The {TempoName} Operator currently supports the following deployment strategies: + +* *allInOne* (Default) - This strategy is intended for development, testing, and demo purposes; it is not intended for production use. The main backend components, Agent, Collector, and Query service, are all packaged into a single executable which is configured, by default. to use in-memory storage. ++ +[NOTE] +==== +In-memory storage is not persistent, which means that if the {TempoShortName} instance shuts down, restarts, or is replaced, that your trace data will be lost. And in-memory storage cannot be scaled, since each pod has its own memory. For persistent storage, you must use the `production` or `streaming` strategies, which use Elasticsearch as the default storage. +==== + +* *production* - The production strategy is intended for production environments, where long term storage of trace data is important, as well as a more scalable and highly available architecture is required. Each of the backend components is therefore deployed separately. The Agent can be injected as a sidecar on the instrumented application. The Query and Collector services are configured with a supported storage type - currently Elasticsearch. Multiple instances of each of these components can be provisioned as required for performance and resilience purposes. + +* *streaming* - The streaming strategy is designed to augment the production strategy by providing a streaming capability that effectively sits between the Collector and the Elasticsearch backend storage. This provides the benefit of reducing the pressure on the backend storage, under high load situations, and enables other trace post-processing capabilities to tap into the real time span data directly from the streaming platform (https://access.redhat.com/documentation/en-us/red_hat_amq/7.6/html/using_amq_streams_on_openshift/index[AMQ Streams]/ https://kafka.apache.org/documentation/[Kafka]). ++ +[NOTE] +==== +The streaming strategy requires an additional Red Hat subscription for AMQ Streams. +==== + +[NOTE] +==== +The streaming deployment strategy is currently unsupported on {ibmzProductName}. +==== + +[NOTE] +==== +There are two ways to install and use {DTProductName}, as part of a service mesh or as a stand alone component. If you have installed {DTShortName} as part of {SMProductName}, you can perform basic configuration as part of the xref:../../service_mesh/v2x/installing-ossm.adoc#installing-ossm[ServiceMeshControlPlane] but for completely control you should configure a Tempo CR and then xref:../../service_mesh/v2x/ossm-observability.html#ossm-config-external-tempo_observability[reference your distributed tracing configuration file in the ServiceMeshControlPlane]. + +==== + +include::modules/distr-tracing-tempo-deploy-default.adoc[leveloffset=+1] + +include::modules/distr-tracing-tempo-deploy-production-es.adoc[leveloffset=+1] + +include::modules/distr-tracing-tempo-deploy-streaming.adoc[leveloffset=+1] + +[id="validating-your-tempo-deployment"] +== Validating your deployment + +include::modules/distr-tracing-tempo-accessing-console.adoc[leveloffset=+2] + +[id="customizing-your-tempo-deployment"] +== Customizing your deployment + +include::modules/distr-tracing-tempo-deployment-best-practices.adoc[leveloffset=+2] + +ifdef::openshift-enterprise,openshift-dedicated[] +For information about configuring persistent storage, see xref:../../storage/understanding-persistent-storage.adoc[Understanding persistent storage] and the appropriate configuration topic for your chosen storage option. +endif::[] + +include::modules/distr-tracing-tempo-config-default.adoc[leveloffset=+2] + +include::modules/distr-tracing-tempo-config-collector.adoc[leveloffset=+2] + +include::modules/distr-tracing-tempo-config-sampling.adoc[leveloffset=+2] + +include::modules/distr-tracing-tempo-config-storage.adoc[leveloffset=+2] + +include::modules/distr-tracing-tempo-config-query.adoc[leveloffset=+2] + +include::modules/distr-tracing-tempo-config-ingester.adoc[leveloffset=+2] diff --git a/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-installing.adoc b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-installing.adoc new file mode 100644 index 000000000000..ee49d6af2999 --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-installing.adoc @@ -0,0 +1,13 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-tempo-installing"] += Installing distributed tracing platform Tempo +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-tempo-installing + +toc::[] + +include::modules/distr-tracing-tempo-install-about.adoc[leveloffset=+1] + +include::modules/distr-tracing-tempo-install-web-console.adoc[leveloffset=+1] + +include::modules/distr-tracing-tempo-install-cli.adoc[leveloffset=+1] diff --git a/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-removing.adoc b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-removing.adoc new file mode 100644 index 000000000000..c903281836a0 --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-removing.adoc @@ -0,0 +1,31 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-tempo-removing"] += Removing distributed tracing platform Tempo +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-tempo-removing + +toc::[] + +The steps for removing {DTProductName} from an {product-title} cluster are as follows: + +. Shut down any {DTProductName} pods. +. Remove any {DTProductName} instances. +. Remove the {TempoName} Operator. +. Remove the {OTELName} Operator. + +include::modules/distr-tracing-removing-tempo-instance.adoc[leveloffset=+1] + +include::modules/distr-tracing-removing-tempo-instance-cli.adoc[leveloffset=+1] + + +== Removing the {DTProductName} Operators + +.Procedure + +. Follow the instructions for xref:../../operators/admin/olm-deleting-operators-from-cluster.adoc#olm-deleting-operators-from-a-cluster[Deleting Operators from a cluster]. + +* Remove the {TempoName} Operator. + +//* Remove the {OTELName} Operator. + +* After the {TempoName} Operator has been removed, if appropriate, remove the OpenShift Elasticsearch Operator. diff --git a/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-updating.adoc b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-updating.adoc new file mode 100644 index 000000000000..d38b16551cc1 --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/distr-tracing-tempo-updating.adoc @@ -0,0 +1,24 @@ +:_content-type: ASSEMBLY +[id="dist-tracing-tempo-updating"] += Updating distributed tracing platform Tempo +include::_attributes/common-attributes.adoc[] +:context: dist-tracing-tempo-updating + +toc::[] + +Operator Lifecycle Manager (OLM) controls the installation, upgrade, and role-based access control (RBAC) of Operators in a cluster. The OLM runs by default in {product-title}. +OLM queries for available Operators as well as upgrades for installed Operators. +For more information about how {product-title} handles upgrades, see the xref:../../operators/understanding/olm/olm-understanding-olm.adoc#olm-understanding-olm[Operator Lifecycle Manager] documentation. + +During an update, the {DTProductName} Operators upgrade the managed {DTShortName} instances to the version associated with the Operator. Whenever a new version of the {TempoName} Operator is installed, all the {TempoShortName} application instances managed by the Operator are upgraded to the Operator's version. For example, after upgrading the Operator from 1.10 installed to 1.11, the Operator scans for running {TempoShortName} instances and upgrades them to 1.11 as well. + +For specific instructions on how to update the OpenShift Elasticsearch Operator, see xref:../../logging/cluster-logging-upgrading.adoc#cluster-logging-upgrading_cluster-logging-upgrading[Updating OpenShift Logging]. + +include::modules/distr-tracing-tempo-change-operator-20.adoc[leveloffset=+1] + +[IMPORTANT] +==== +If you have not already updated your OpenShift Elasticsearch Operator as described in xref:../../logging/cluster-logging-upgrading.adoc[Updating OpenShift Logging] complete that update before updating your {TempoName} Operator. +==== + +For instructions on how to update the Operator channel, see xref:../../operators/admin/olm-upgrading-operators.adoc[Updating installed Operators]. diff --git a/distr_tracing/distr_tracing_tempo/images b/distr_tracing/distr_tracing_tempo/images new file mode 120000 index 000000000000..e4c5bd02a10a --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/images @@ -0,0 +1 @@ +../images/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_tempo/modules b/distr_tracing/distr_tracing_tempo/modules new file mode 120000 index 000000000000..43aab75b53c9 --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/modules @@ -0,0 +1 @@ +../modules/ \ No newline at end of file diff --git a/distr_tracing/distr_tracing_tempo/snippets b/distr_tracing/distr_tracing_tempo/snippets new file mode 120000 index 000000000000..5a3f5add140e --- /dev/null +++ b/distr_tracing/distr_tracing_tempo/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file diff --git a/modules/distr-tracing-accessing-jaeger-console.adoc b/modules/distr-tracing-accessing-jaeger-console.adoc index 69ada6de56b7..d098f475127a 100644 --- a/modules/distr-tracing-accessing-jaeger-console.adoc +++ b/modules/distr-tracing-accessing-jaeger-console.adoc @@ -1,7 +1,6 @@ //// Module included in the following assemblies: -* distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc -* distr_tracing/distr_tracing_install/distr-tracing-deploying-otel.adoc +* distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: PROCEDURE [id="distr-tracing-accessing-jaeger-console_{context}"] diff --git a/modules/distr-tracing-config-default.adoc b/modules/distr-tracing-config-default.adoc index 3c4488dd88a2..15d3778b8b89 100644 --- a/modules/distr-tracing-config-default.adoc +++ b/modules/distr-tracing-config-default.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-default_{context}"] diff --git a/modules/distr-tracing-config-ingester.adoc b/modules/distr-tracing-config-ingester.adoc index a6b4930dbbb8..d48b3c12d99a 100644 --- a/modules/distr-tracing-config-ingester.adoc +++ b/modules/distr-tracing-config-ingester.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-ingester_{context}"] diff --git a/modules/distr-tracing-config-jaeger-collector.adoc b/modules/distr-tracing-config-jaeger-collector.adoc index 6a32b00d4b15..b6d631d7b8c1 100644 --- a/modules/distr-tracing-config-jaeger-collector.adoc +++ b/modules/distr-tracing-config-jaeger-collector.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-jaeger-collector_{context}"] diff --git a/modules/distr-tracing-config-otel-collector.adoc b/modules/distr-tracing-config-otel-collector.adoc index 3156c704d9e0..4c55470ebf5b 100644 --- a/modules/distr-tracing-config-otel-collector.adoc +++ b/modules/distr-tracing-config-otel-collector.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: --distr_tracing_install/distributed-tracing-deploying-otel.adoc +-distr_tracing_otel/distr-tracing-otel-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-otel-collector_{context}"] @@ -29,6 +29,10 @@ metadata: namespace: tracing-system spec: mode: deployment + ports: + - name: promexporter + port: 8889 + protocol: TCP config: | receivers: otlp: @@ -41,12 +45,20 @@ spec: endpoint: jaeger-production-collector-headless.tracing-system.svc:14250 tls: ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true # by default resource attributes are dropped service: pipelines: traces: receivers: [otlp] processors: [] exporters: [jaeger] + metrics: + receivers: [otlp] + processors: [] + exporters: [prometheus] ---- [NOTE] @@ -77,7 +89,7 @@ If a component is configured, but not defined within the `service` section then |exporters: |An exporter sends data to one or more backends/destinations. By default, no exporters are configured. There must be at least one enabled exporter for a configuration to be considered valid. Exporters are enabled by being added to a pipeline. Exporters may come with default settings, but many require configuration to specify at least the destination and security settings. -|`logging`, `jaeger` +|`logging`, `jaeger`, `prometheus`, |None |exporters: @@ -125,4 +137,28 @@ If a component is configured, but not defined within the `service` section then |You enable exporters for tracing by adding them under `service.pipelines.traces`. | |None + +|service: + pipelines: + metrics: + receivers: +|You enable receivers for metrics by adding them under `service.pipelines.metrics`. +| +|None + +|service: + pipelines: + metrics: + processors: +|You enable processors for metircs by adding them under `service.pipelines.metrics`. +| +|None + +|service: + pipelines: + metrics: + exporters: +|You enable exporters for metrics by adding them under `service.pipelines.metrics`. +| +|None |=== diff --git a/modules/distr-tracing-config-otel-send-metrics-monitoring-stack.adoc b/modules/distr-tracing-config-otel-send-metrics-monitoring-stack.adoc new file mode 100644 index 000000000000..a1242628e295 --- /dev/null +++ b/modules/distr-tracing-config-otel-send-metrics-monitoring-stack.adoc @@ -0,0 +1,40 @@ +//// +This module is included in the following assemblies: +- distr_tracing_install/distributed-tracing-deploying-otel.adoc +//// +:_content-type: REFERENCE +[id="distr-tracing-send-metrics-monitoring-stack_{context}"] += Sending metrics to the monitoring stack + +You can configure the monitoring stack to scrape OpenTelemetry collector metrics endpoints and to remove +duplicated labels that the monitoring stack has added during scraping. + +.Sample `+PodMonitor+` that configures the monitoring stack to scrape collector metrics +[source,yaml] +---- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: otel-collector +spec: + selector: + matchLabels: + app.kubernetes.io/name: otel-collector + podMetricsEndpoints: + - port: metrics <1> + - port: promexporter <2> + relabelings: + - action: labeldrop + regex: pod + - action: labeldrop + regex: container + - action: labeldrop + regex: endpoint + metricRelabelings: + - action: labeldrop + regex: instance + - action: labeldrop + regex: job +---- +<1> The name of the internal metrics port for the OpenTelemetry collector. This port name is always `+metrics+`. +<2> The name of the Prometheus exporter port for the OpenTelemetry collector. This port name is defined in the `+.spec.ports+` section of the OpenTelemetry collector CR. diff --git a/modules/distr-tracing-config-query.adoc b/modules/distr-tracing-config-query.adoc index 1aa8ba06d64e..8a45af248b0a 100644 --- a/modules/distr-tracing-config-query.adoc +++ b/modules/distr-tracing-config-query.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-query_{context}"] diff --git a/modules/distr-tracing-config-sampling.adoc b/modules/distr-tracing-config-sampling.adoc index 9e743c4072d9..6d8fc643ed57 100644 --- a/modules/distr-tracing-config-sampling.adoc +++ b/modules/distr-tracing-config-sampling.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-sampling_{context}"] diff --git a/modules/distr-tracing-config-storage.adoc b/modules/distr-tracing-config-storage.adoc index 8947448d3432..51efcb69e47b 100644 --- a/modules/distr-tracing-config-storage.adoc +++ b/modules/distr-tracing-config-storage.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-config-storage_{context}"] diff --git a/modules/distr-tracing-deploy-default.adoc b/modules/distr-tracing-deploy-default.adoc index 627adb8bc6f7..091c9dfd689a 100644 --- a/modules/distr-tracing-deploy-default.adoc +++ b/modules/distr-tracing-deploy-default.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-deploy-production-es.adoc b/modules/distr-tracing-deploy-production-es.adoc index 7420097adead..a211689abcfa 100644 --- a/modules/distr-tracing-deploy-production-es.adoc +++ b/modules/distr-tracing-deploy-production-es.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-deploy-streaming.adoc b/modules/distr-tracing-deploy-streaming.adoc index 473e522f7d12..e343a784b88d 100644 --- a/modules/distr-tracing-deploy-streaming.adoc +++ b/modules/distr-tracing-deploy-streaming.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-deployment-best-practices.adoc b/modules/distr-tracing-deployment-best-practices.adoc index 8e6098bc6140..753d04cf7bba 100644 --- a/modules/distr-tracing-deployment-best-practices.adoc +++ b/modules/distr-tracing-deployment-best-practices.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: CONCEPT [id="distr-tracing-deployment-best-practices_{context}"] diff --git a/modules/distr-tracing-install-elasticsearch.adoc b/modules/distr-tracing-install-elasticsearch.adoc index 822df38406e2..fa3b87fda264 100644 --- a/modules/distr-tracing-install-elasticsearch.adoc +++ b/modules/distr-tracing-install-elasticsearch.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-installing.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-install-jaeger-operator.adoc b/modules/distr-tracing-install-jaeger-operator.adoc index 37bcc6ba7cd7..67480dae5e69 100644 --- a/modules/distr-tracing-install-jaeger-operator.adoc +++ b/modules/distr-tracing-install-jaeger-operator.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-installing.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-install-otel-operator.adoc b/modules/distr-tracing-install-otel-operator.adoc index d85255daa30c..b84ad238ee41 100644 --- a/modules/distr-tracing-install-otel-operator.adoc +++ b/modules/distr-tracing-install-otel-operator.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-installing.adoc +- distr_tracing_otel/distr-tracing-otel-installing.adoc //// :_content-type: PROCEDURE diff --git a/modules/distr-tracing-install-overview.adoc b/modules/distr-tracing-install-overview.adoc index d082c4e98944..7492aefd2139 100644 --- a/modules/distr-tracing-install-overview.adoc +++ b/modules/distr-tracing-install-overview.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-installing.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-installing.adoc //// :_content-type: CONCEPT diff --git a/modules/distr-tracing-otel-migrating-from-jaeger-with-sidecars.adoc b/modules/distr-tracing-otel-migrating-from-jaeger-with-sidecars.adoc new file mode 100644 index 000000000000..6141842e3f02 --- /dev/null +++ b/modules/distr-tracing-otel-migrating-from-jaeger-with-sidecars.adoc @@ -0,0 +1,146 @@ +// Module included in the following assemblies: +// +// * distr-tracing-otel-migrating.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-otel-migrating-from-jaeger-with-sidecars_{context}"] += (Temporary title) Migrating from {JaegerName} to {OTELName} with sidecars + +.Prerequisites + +* {JaegerName} is used on the cluster. +* {OTELName} is installed. + +.Procedure + +The {OTELName} Operator can inject sidecars into Deployment workloads. + +Configure OpenTelemetry Collector as sidecar + +---- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: otel + namespace: otel-collector-example +spec: + mode: sidecar + config: | + receivers: + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: + processors: + batch: + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + resourcedetection: + detectors: [openshift] + timeout: 2s + exporters: + otlp: + endpoint: "tempo--gateway:8090" <1> + tls: + insecure: true + service: + pipelines: + traces: + receivers: [jaeger] + processors: [memory_limiter, resourcedetection, batch] + exporters: [otlp] +---- +<1> This points to the Gateway of a Tempo instance deployed by using the `` Tempo Operator. + +Create a Service Account for running your application: + +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector-sidecar +---- + +Create a ClusterRole for the permissions needed by some processors + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector-sidecar +rules: +# Permissions for infrastructures and infrastructures/status are needed for resourcedetectionprocessor +- apiGroups: ["config.openshift.io"] + resources: ["infrastructures", "infrastructures/status"] + verbs: ["get", "watch", "list"] +---- + +Create a ClusterRoleBinding to give those permissions to the Service Account + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector-sidecar +subjects: +- kind: ServiceAccount + name: otel-collector-sidecar +roleRef: + kind: ClusterRole + name: otel-collector-sidecar + apiGroup: rbac.authorization.k8s.io +---- + +Deploy the OpenTelemetry Collector as sidecar + +---- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: otel + namespace: otel-collector-example +spec: + mode: sidecar + config: | + receivers: + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: + processors: + batch: + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + resourcedetection: + detectors: [openshift] + timeout: 2s + exporters: + otlp: + endpoint: "tempo-example-gateway:8090" + tls: + insecure: true + service: + pipelines: + traces: + receivers: [jaeger] + processors: [memory_limiter, resourcedetection, batch] + exporters: [otlp] +---- + +Remove the injected Jaeger Agent from your application + +To remove the automatic sidecar injection from the Jaeger Operator, you need to remove the "sidecar.jaegertracing.io/inject": "true" annotation from your Deployment. +Enable automatic injection of the OpenTelemetry sidecar + +To enable automatic injection of the OpenTelemetry sidecar, add the sidecar.opentelemetry.io/inject: "true" annotation to the .spec.template.metadata.annotations field of your Deployment. +Use the created Service Account for your deployment + +Use the Service Account we created before (otel-collector-sidecar) for the deployment of your application. This will allow the processors to get the correct information and add it to your traces. diff --git a/modules/distr-tracing-otel-migrating-from-jaeger-without-sidecars.adoc b/modules/distr-tracing-otel-migrating-from-jaeger-without-sidecars.adoc new file mode 100644 index 000000000000..9bbe53fb9e06 --- /dev/null +++ b/modules/distr-tracing-otel-migrating-from-jaeger-without-sidecars.adoc @@ -0,0 +1,122 @@ +// Module included in the following assemblies: +// +// * distr-tracing-otel-migrating.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-otel-migrating-from-jaeger-without-sidecars_{context}"] += (Temporary title) Migrating from {JaegerName} to {OTELName} without sidecars + +.Prerequisites + +* Existing instrumentation of applications uses {JaegerName} for tracing. +* {OTELName} is installed. + +.Procedure + +Configure OpenTelemetry Collector deployment + +Create the OpenShift project where the OpenTelemetry Collector will be deployed + +---- +apiVersion: project.openshift.io/v1 +kind: Project +metadata: + name: observability +---- + +Create a Service Account for running the OpenTelemetry Collector instance: + +---- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector-deployment + namespace: observability +---- + +Create a ClusterRole for some permissions needed by some processors + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector +rules: +# Permissions for pods and namespaces resources are needed for the k8sattributesprocessor +# Permissions for infrastructures and infrastructures/status are needed for resourcedetectionprocessor +- apiGroups: ["", "config.openshift.io"] + resources: ["pods", "namespaces", "infrastructures", "infrastructures/status"] + verbs: ["get", "watch", "list"] +---- + +Create a ClusterRoleBinding to give those permissions to the Service Account + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector +subjects: +- kind: ServiceAccount + name: otel-collector-deployment + namespace: observability +roleRef: + kind: ClusterRole + name: otel-collector + apiGroup: rbac.authorization.k8s.io +---- + +Create the OpenTelemetry Collector instance + +Note this collector will be exporting traces to a Tempo instance. Ensure you create your Tempo instance using the Red Hat Tempo Operator and place here the correct endpoint. + +---- +apiVersion: opentelemetry.io/v1alpha1 +kind: OpenTelemetryCollector +metadata: + name: otel + namespace: observability +spec: + mode: deployment + serviceAccount: otel-collector-deployment + config: | + receivers: + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: + processors: + batch: + k8sattributes: + memory_limiter: + check_interval: 1s + limit_percentage: 50 + spike_limit_percentage: 30 + resourcedetection: + detectors: [openshift] + exporters: + otlp: + endpoint: "tempo-example-gateway:8090" + tls: + insecure: true + service: + pipelines: + traces: + receivers: [jaeger] + processors: [memory_limiter, k8sattributes, resourcedetection, batch] + exporters: [otlp] +---- + +Point your tracing endpoint to the OpenTelemetry Operator + +If you are exporting your traces directly from your application to Jaeger, you will need to change the API endpoint from the Jaeger endpoint to the OpenTelemetry Collector endpoint. + +For instance, if you are using Golang and exporting your traces using the jaegerexporter, you can be using something like this: + +---- +exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) +---- + +url needs to point now to the OpenTelemetry Collector API endpoint. diff --git a/modules/distr-tracing-removing-tempo-instance-cli.adoc b/modules/distr-tracing-removing-tempo-instance-cli.adoc new file mode 100644 index 000000000000..61336a5b69b7 --- /dev/null +++ b/modules/distr-tracing-removing-tempo-instance-cli.adoc @@ -0,0 +1,89 @@ +//Module included in the following assemblies: +// +//* distr_tracing_install/dist-tracing-tempo-removing.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-removing-tempo-instance-cli_{context}"] += Removing a {TempoName} instance from the CLI + +. Log in to the {product-title} CLI. ++ +[source,terminal] +---- +$ oc login --username= +---- ++ +. To display the {TempoShortName} instances run the command: ++ +[source,terminal] +---- +$ oc get deployments -n +---- ++ +For example: ++ +[source,terminal] +---- +$ oc get deployments -n openshift-operators +---- ++ +The names of Operators have the suffix `-operator`. The following example shows two {TempoName} Operators and four {TempoShortName} instances: ++ +[source,terminal] +---- +$ oc get deployments -n openshift-operators +---- ++ +You should see output similar to the following: ++ +[source,terminal] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +elasticsearch-operator 1/1 1 1 93m +tempo-operator 1/1 1 1 49m +tempo-test 1/1 1 1 7m23s +tempo-test2 1/1 1 1 6m48s +tracing1 1/1 1 1 7m8s +tracing2 1/1 1 1 35m +---- ++ +. To remove an instance of {TempoShortName}, run the following command: ++ +[source,terminal] +---- +$ oc delete tempo -n +---- ++ +For example: ++ +[source,terminal] +---- +$ oc delete tempo tracing2 -n openshift-operators +---- ++ + +. To verify the deletion, run the `oc get deployments` command again: ++ +[source,terminal] +---- +$ oc get deployments -n +---- + ++ +For example: ++ +[source,terminal] +---- +$ oc get deployments -n openshift-operators +---- ++ +You should see generated output that is similar to the following example: ++ +[source,terminal] +---- +NAME READY UP-TO-DATE AVAILABLE AGE +tempo-operator 1/1 1 1 50m +tempo-test 1/1 1 1 8m14s +tempo-test2 1/1 1 1 7m39s +tracing1 1/1 1 1 7m59s +---- diff --git a/modules/distr-tracing-removing-tempo-instance.adoc b/modules/distr-tracing-removing-tempo-instance.adoc new file mode 100644 index 000000000000..5c1ff099cab9 --- /dev/null +++ b/modules/distr-tracing-removing-tempo-instance.adoc @@ -0,0 +1,28 @@ +//Module included in the following assemblies: +// +//* distr_tracing_install/dist-tracing-tempo-removing.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-removing-tempo-instance_{context}"] += Removing a {TempoName} instance using the web console + +[NOTE] +==== +When deleting an instance that uses the in-memory storage, all data is permanently lost. Data stored in a persistent storage such as Elasticsearch is not be deleted when a {TempoName} instance is removed. +==== + +.Procedure + +. Log in to the {product-title} web console. + +. Navigate to *Operators* -> *Installed Operators*. + +. Select the name of the project where the Operators are installed from the *Project* menu, for example, `openshift-operators`. + +. Click the {TempoName} Operator. + +. Click the *Tempo* tab. + +. Click the Options menu {kebab} next to the instance you want to delete and select *Delete Tempo*. + +. In the confirmation message, click *Delete*. diff --git a/modules/distr-tracing-rn-fixed-issues.adoc b/modules/distr-tracing-rn-fixed-issues.adoc index 6fffaa723b2f..6586b4d15029 100644 --- a/modules/distr-tracing-rn-fixed-issues.adoc +++ b/modules/distr-tracing-rn-fixed-issues.adoc @@ -13,6 +13,26 @@ Cause - Why did this happen? Fix - What did we change to fix the problem? Result - How has the behavior changed as a result? Try to avoid “It is fixed” or “The issue is resolved” or “The error no longer presents”. //// + + + + + + + + + + + + + + + + + + + + * link:https://issues.redhat.com/browse/OSSM-1910[OSSM-1910] Because of an issue introduced in version 2.6, TLS connections could not be established with {product-title} {SMProductShortName}. diff --git a/modules/distr-tracing-rn-known-issues.adoc b/modules/distr-tracing-rn-known-issues.adoc index 737bc1fbe259..69364014e60a 100644 --- a/modules/distr-tracing-rn-known-issues.adoc +++ b/modules/distr-tracing-rn-known-issues.adoc @@ -16,6 +16,16 @@ Result - If the workaround does not completely address the problem. These limitations exist in {DTProductName}: + + + + + + + + + + * Apache Spark is not supported. ifndef::openshift-rosa[] @@ -24,8 +34,16 @@ endif::openshift-rosa[] These are the known issues for {DTProductName}: -* link:https://issues.redhat.com/browse/OBSDA-220[OBSDA-220] In some cases, if you try to pull an image using {OTELShortName}, the image pull fails and a `Failed to pull image` error message appears. -There is no workaround for this issue. + + + + + + + + + + * link:https://issues.redhat.com/browse/TRACING-2057[TRACING-2057] The Kafka API has been updated to `v1beta2` to support the Strimzi Kafka Operator 0.23.0. However, this API version is not supported by AMQ Streams 1.6.3. If you have the following environment, your Jaeger services will not be upgraded, and you cannot create new Jaeger services or modify existing Jaeger services: diff --git a/modules/distr-tracing-rn-new-features.adoc b/modules/distr-tracing-rn-new-features.adoc index cc2674c57079..96b176a759bc 100644 --- a/modules/distr-tracing-rn-new-features.adoc +++ b/modules/distr-tracing-rn-new-features.adoc @@ -13,6 +13,148 @@ Result – If changed, describe the current user experience. This release adds improvements related to the following components and concepts. +== New features and enhancements {DTProductName} 2.9 + +This release of {DTProductName} addresses Common Vulnerabilities and Exposures (CVEs) and bug fixes. + +=== {JaegerName} + +==== Bug fixes + +* Expose Jaeger Query gRPC port (16685) on the Jaeger Query service. + +// link:https://issues.redhat.com/browse/TRACING-3322[TRACING-3322] + +* When deploying Service Mesh on SNO in a disconnected environment , the Jaeger Pod frequently goes into Pending state. + +// link:https://issues.redhat.com/browse/TRACING-3312[TRACING-3312] + +* Wrong port is exposed for jaeger-production-query resulting in connection refused. +Fixed by exposing Jaeger Query gRPC port (16685) on the Jaeger Query deployment. + +// link:https://issues.redhat.com/browse/TRACING-2968[TRACING-29638] + +* Jaeger operator pod restarting with OOMKilled with the default memory value. +Fixed by removing resource limits. + +// link:https://issues.redhat.com/browse/TRACING-3173[TRACING-3173] + +=== {TempoName} + +The {TempoName} is Technology Preview feature for {DTProductName}. + +==== Enhancements + +* Support the operator level 4 (Deep Insights). +This feature enables upgrade, monitoring and alerting of `+TempoStack+` instances and operator. +* Add Ingress and OpenShift Route configuration for the Gateway. +* Support managed and unmanaged state in the `+TempoStack+` custom resource. +* Expose additional ingestion protocols (Jaeger Thrift binary, Jaeger Thrift compact, Jaeger gRPC and Zipkin) in the Distributor service. +When the Gateway is enabled only OTLP gRPC is enabled. +* Expose Jaeger Query gRPC endpoint on the Query Frontend service. +* Support multitenancy without the Gateway (without authentication and authorization). + +==== Bug fixes + +* Support disconnected environments. + +// link:https://issues.redhat.com/browse/TRACING-3145[TRACING-3145] + +* Enable mTLS communication between Tempo components. + +// link:https://issues.redhat.com/browse/TRACING-3091[TRACING-3091] + +* Remove resource limits for Tempo operator. + +// link:https://issues.redhat.com/browse/TRACING-3204[TRACING-3204] + +==== Known issues + +* Custom TLS CA option is not implemented for connecting to an object storage. + +// link:https://issues.redhat.com/browse/TRACING-3462[TRACING-3462] + +* When you use the Jaeger user interface (UI) with Tempo Operator, the Jaeger UI lists only services that have sent traces within the last 15 minutes. +For services that have not sent traces within the last 15 minutes, those traces are still stored even though they are not visible in the Jaeger UI. + +// link:https://issues.redhat.com/browse/TRACING-3139[TRACING-3139] + +* Tempo query frontend service should not use internal mTLS when gateway is not deployed. This issue does not affect Jaeger Query API. The workaround is to disable mTLS. + +.Disable mTLS +. Open the Tempo operator config map for editing. ++ +[source,console] +---- +$ oc edit configmap tempo-operator-manager-config -n openshift-operators <1> +---- +<1> The namespace where the Tempo operator is installed. + +. Disable the mTLS in the operator config. ++ +[source,yaml] +---- +data: + controller_manager_config.yaml: | + featureGates: + httpEncryption: false + grpcEncryption: false + builtInCertManagement: + enabled: false +---- +. Restart Tempo operator pod ++ +[source,console] +---- +$ oc rollout restart deployment.apps/tempo-operator-controller -n openshift-operators +---- + +// link:https://issues.redhat.com/browse/TRACING-3510[TRACING-3510] + +=== {OTELName} + +The {OTELName} is Technology Preview feature for {DTProductName}. + +==== Enhancements + +* Support OTLP metrics ingestion. +The metrics can be forwarded and stored in the `+user-workload-monitoring+` via the Prometheus exporter. +* Support the operator level 4 (Deep Insights). +This feature enables upgrade and monitoring of `+OpenTelemetryCollector+` instances and operator. +* Report traces and metrics from remote clusters using OTLP/HTTP(s). +* Collect OpenShift specific resource attributes via `+resourcedetection+` processor. +* Support managed and unmanaged state in the `+OpenTelemetryCollector+` custom resouce. + +==== Bug fixes + +* OpenTelemetry operator crashlooping after receiving opentelemetry-operator.v0.74.0-5. Fixed by properly configuring autoscaler. + +// link:https://issues.redhat.com/browse/TRACING-3190[TRACING-3190] + +==== Known issues + +* Operator level needs to be set to 4 (Deep Insights). + +// link:https://issues.redhat.com/browse/TRACING-3431[TRACING-3431] + +=== Component versions supported in {DTProductName} version 2.9 + +[options="header"] +|=== +|Operator |Component |Version +|{JaegerName} +|Jaeger +|1.47.0 + +|{OTELName} +|OpenTelemetry +|0.81.0 + +|{TempoName} +|{TempoShortName} +|2.1.1 +|=== + == New features and enhancements {DTProductName} 2.8 This release of {DTProductName} addresses Common Vulnerabilities and Exposures (CVEs) and bug fixes. @@ -75,7 +217,8 @@ This release of {DTProductName} addresses Common Vulnerabilities and Exposures ( This release of {DTProductName} addresses Common Vulnerabilities and Exposures (CVEs) and bug fixes. -This release introduces support for ingesting OpenTelemetry protocol (OTLP) to the {JaegerName} Operator. The Operator now automatically enables the OTLP ports: +This release introduces support for ingesting OpenTelemetry protocol (OTLP) to the {JaegerName} Operator. +The Operator now automatically enables the OTLP ports: * Port 4317 is used for OTLP gRPC protocol. * Port 4318 is used for OTLP HTTP protocol. @@ -96,19 +239,20 @@ This release also adds support for collecting Kubernetes resource attributes to |0.56 |=== - == New features and enhancements {DTProductName} 2.4 This release of {DTProductName} addresses Common Vulnerabilities and Exposures (CVEs) and bug fixes. This release also adds support for auto-provisioning certificates using the Red Hat Elasticsearch Operator. -* Self-provisioning, which means using the {JaegerName} Operator to call the Red Hat Elasticsearch Operator during installation. Self provisioning is fully supported with this release. +* Self-provisioning, which means using the {JaegerName} Operator to call the Red Hat Elasticsearch Operator during installation. +Self provisioning is fully supported with this release. * Creating the Elasticsearch instance and certificates first and then configuring the {JaegerShortName} to use the certificate is a Technology Preview for this release. [NOTE] ==== -When upgrading to {DTProductName} 2.4, the Operator recreates the Elasticsearch instance, which might take five to ten minutes. Distributed tracing will be down and unavailable for that period. +When upgrading to {DTProductName} 2.4, the Operator recreates the Elasticsearch instance, which might take five to ten minutes. +Distributed tracing will be down and unavailable for that period. ==== === Component versions supported in {DTProductName} version 2.4 @@ -147,7 +291,8 @@ This release of {DTProductName} addresses Common Vulnerabilities and Exposures ( This release of {DTProductName} addresses Common Vulnerabilities and Exposures (CVEs) and bug fixes. -With this release, the {JaegerName} Operator is now installed to the `openshift-distributed-tracing` namespace by default. Before this update, the default installation had been in the `openshift-operators` namespace. +With this release, the {JaegerName} Operator is now installed to the `openshift-distributed-tracing` namespace by default. +Before this update, the default installation had been in the `openshift-operators` namespace. === Component versions supported in {DTProductName} version 2.3.0 @@ -201,7 +346,8 @@ This release of {DTProductName} addresses Common Vulnerabilities and Exposures ( == New features and enhancements {DTProductName} 2.0.0 -This release marks the rebranding of Red Hat OpenShift Jaeger to {DTProductName}. This release consists of the following changes, additions, and improvements: +This release marks the rebranding of Red Hat OpenShift Jaeger to {DTProductName}. +This release consists of the following changes, additions, and improvements: * {DTProductName} now consists of the following two main components: @@ -209,7 +355,8 @@ This release marks the rebranding of Red Hat OpenShift Jaeger to {DTProductName} ** *{OTELName}* - This component is based on the open source link:https://opentelemetry.io/[OpenTelemetry project]. -* Updates {JaegerName} Operator to Jaeger 1.28. Going forward, {DTProductName} will only support the `stable` Operator channel. Channels for individual releases are no longer supported. +* Updates {JaegerName} Operator to Jaeger 1.28. Going forward, {DTProductName} will only support the `stable` Operator channel. +Channels for individual releases are no longer supported. * Introduces a new {OTELName} Operator based on OpenTelemetry 0.33. Note that this Operator is a Technology Preview feature. diff --git a/modules/distr-tracing-rn-technology-preview.adoc b/modules/distr-tracing-rn-technology-preview.adoc index bb06420e5a5d..26d6a4ca858e 100644 --- a/modules/distr-tracing-rn-technology-preview.adoc +++ b/modules/distr-tracing-rn-technology-preview.adoc @@ -17,6 +17,28 @@ Technology Preview features are not supported with Red Hat production service le For more information about the support scope of Red Hat Technology Preview features, see link:https://access.redhat.com/support/offerings/techpreview/[Technology Preview Features Support Scope]. ==== + +== {DTProductName} 2.9.0 Technology Preview + + + + + + + + + + + + + + + + + + + + == {DTProductName} 2.8.0 Technology Preview diff --git a/modules/distr-tracing-sidecar-automatic.adoc b/modules/distr-tracing-sidecar-automatic.adoc index 9c425dc12c7a..67713554a81e 100644 --- a/modules/distr-tracing-sidecar-automatic.adoc +++ b/modules/distr-tracing-sidecar-automatic.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="dist-tracing-sidecar-automatic_{context}"] diff --git a/modules/distr-tracing-sidecar-manual.adoc b/modules/distr-tracing-sidecar-manual.adoc index 1f914ed319cb..be79e87505dc 100644 --- a/modules/distr-tracing-sidecar-manual.adoc +++ b/modules/distr-tracing-sidecar-manual.adoc @@ -1,6 +1,6 @@ //// This module included in the following assemblies: -- distr_tracing_install/distr-tracing-deploying-jaeger.adoc +- distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc //// :_content-type: REFERENCE [id="distr-tracing-sidecar-manual_{context}"] diff --git a/modules/distr-tracing-tempo-accessing-console.adoc b/modules/distr-tracing-tempo-accessing-console.adoc new file mode 100644 index 000000000000..8cbafbb737a2 --- /dev/null +++ b/modules/distr-tracing-tempo-accessing-console.adoc @@ -0,0 +1,60 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-accessing-console_{context}"] += Accessing the Tempo console + +To access the Tempo console you must have either {SMProductName} or {DTProductName} installed, and {TempoName} installed, configured, and deployed. + +The installation process creates a route to access the Tempo console. + +If you know the URL for the Tempo console, you can access it directly. If you do not know the URL, use the following directions. + +.Procedure from OpenShift console +. Log in to the {product-title} web console as a user with cluster-admin rights. If you use {product-dedicated}, you must have an account with the `dedicated-admin` role. + +. Navigate to *Networking* -> *Routes*. + +. On the *Routes* page, select the control plane project, for example `tracing-system`, from the *Namespace* menu. ++ +The *Location* column displays the linked address for each route. ++ +. If necessary, use the filter to find the `tempo` route. Click the route *Location* to launch the console. + +. Click *Log In With OpenShift*. + +//// +.Procedure from Kiali console + +. Launch the Kiali console. + +. Click *Distributed Tracing* in the left navigation pane. + +. Click *Log In With OpenShift*. +//// + +.Procedure from the CLI + +. Log in to the {product-title} CLI as a user with the `cluster-admin` role. If you use {product-dedicated}, you must have an account with the `dedicated-admin` role. ++ +[source,terminal] +---- +$ oc login --username= https://:6443 +---- ++ +. To query for details of the route using the command line, enter the following command. In this example, `tracing-system` is the control plane namespace. ++ +[source,terminal] +---- +$ export TEMPO_URL=$(oc get route -n tracing-system tempo -o jsonpath='{.spec.host}') +---- ++ +. Launch a browser and navigate to ``\https://``, where `` is the route that you discovered in the previous step. + +. Log in using the same user name and password that you use to access the {Product-title} console. + +. If you have added services to the service mesh and have generated traces, you can use the filters and *Find Traces* button to search your trace data. ++ +If you are validating the console installation, there is no trace data to display. diff --git a/modules/distr-tracing-tempo-change-operator-20.adoc b/modules/distr-tracing-tempo-change-operator-20.adoc new file mode 100644 index 000000000000..3c5757e5f00f --- /dev/null +++ b/modules/distr-tracing-tempo-change-operator-20.adoc @@ -0,0 +1,22 @@ +// Module included in the following assemblies: +// +// * dist_tracing/dist_tracing_install/dist-tracing-tempo-updating.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-changing-operator-channel_{context}"] += Changing the Operator channel for 2.0 + +{DTProductName} 2.0.0 made the following changes: + +* Renamed the Red Hat OpenShift Jaeger Operator to the {JaegerName} Operator. + +* Stopped support for individual release channels. Going forward, the {JaegerName} Operator will only support the *stable* Operator channel. Maintenance channels, for example *1.24-stable*, will no longer be supported by future Operators. + +As part of the update to version 2.0, you must update your OpenShift Elasticsearch and {JaegerName} Operator subscriptions. + +.Prerequisites + +* The {product-title} version is 4.6 or later. +* You have updated the OpenShift Elasticsearch Operator. +* You have backed up the Jaeger custom resource file. +* An account with the `cluster-admin` role. If you use {product-dedicated}, you must have an account with the `dedicated-admin` role. diff --git a/modules/distr-tracing-tempo-config-collector.adoc b/modules/distr-tracing-tempo-config-collector.adoc new file mode 100644 index 000000000000..c9b08c5e2a15 --- /dev/null +++ b/modules/distr-tracing-tempo-config-collector.adoc @@ -0,0 +1,66 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-collector_{context}"] += Tempo Collector configuration options + +The Tempo Collector is the component responsible for receiving the spans that were captured by the tracer and writing them to persistent Elasticsearch storage when using the `production` strategy, or to AMQ Streams when using the `streaming` strategy. + +The Collectors are stateless and thus many instances of Tempo Collector can be run in parallel. Collectors require almost no configuration, except for the location of the Elasticsearch cluster. + +.Parameters used by the Operator to define the Tempo Collector +[options="header"] +[cols="l, a, a"] +|=== +|Parameter |Description |Values +|collector: + replicas: +|Specifies the number of Collector replicas to create. +|Integer, for example, `5` +|=== + + +.Configuration parameters passed to the Collector +[options="header"] +[cols="l, a, a"] +|=== +|Parameter |Description |Values +|spec: + collector: + options: {} +|Configuration options that define the Tempo Collector. +| + +|options: + collector: + num-workers: +|The number of workers pulling from the queue. +|Integer, for example, `50` + +|options: + collector: + queue-size: +|The size of the Collector queue. +|Integer, for example, `2000` + +|options: + kafka: + producer: + topic: tempo-spans +|The `topic` parameter identifies the Kafka configuration used by the Collector to produce the messages, and the Ingester to consume the messages. +|Label for the producer. + +|options: + kafka: + producer: + brokers: my-cluster-kafka-brokers.kafka:9092 +|Identifies the Kafka configuration used by the Collector to produce the messages. If brokers are not specified, and you have AMQ Streams 1.4.0+ installed, the {TempoName} Operator will self-provision Kafka. +| + +|options: + log-level: +|Logging level for the Collector. +|Possible values: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. +|=== diff --git a/modules/distr-tracing-tempo-config-default.adoc b/modules/distr-tracing-tempo-config-default.adoc new file mode 100644 index 000000000000..65f03066bf34 --- /dev/null +++ b/modules/distr-tracing-tempo-config-default.adoc @@ -0,0 +1,123 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-default_{context}"] += Distributed tracing default configuration options + +The Tempo custom resource (CR) defines the architecture and settings to be used when creating the {TempoShortName} resources. You can modify these parameters to customize your {TempoShortName} implementation to your business needs. + +.Tempo generic YAML example +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: name +spec: + strategy: + allInOne: + options: {} + resources: {} + agent: + options: {} + resources: {} + collector: + options: {} + resources: {} + sampling: + options: {} + storage: + type: + options: {} + query: + options: {} + resources: {} + ingester: + options: {} + resources: {} + options: {} +---- + +.Tempo parameters +[options="header"] +|=== +|Parameter |Description |Values |Default value + +|`apiVersion:` +||API version to use when creating the object. +|`tempotracing.io/v1` +|`tempotracing.io/v1` + +|`kind:` +|Defines the kind of Kubernetes object to create. +|`tempo` +| + +|`metadata:` +|Data that helps uniquely identify the object, including a `name` string, `UID`, and optional `namespace`. +| +|{product-title} automatically generates the `UID` and completes the `namespace` with the name of the project where the object is created. + +|`name:` +|Name for the object. +|The name of your {TempoShortName} instance. +|`tempo-all-in-one-inmemory` + +|`spec:` +|Specification for the object to be created. +|Contains all of the configuration parameters for your {TempoShortName} instance. When a common definition for all Tempo components is required, it is defined under the `spec` node. When the definition relates to an individual component, it is placed under the `spec/` node. +|N/A + +|`strategy:` +|Tempo deployment strategy +|`allInOne`, `production`, or `streaming` +|`allInOne` + +|`allInOne:` +|Because the `allInOne` image deploys the Agent, Collector, Query, Ingester, and Tempo UI in a single pod, configuration for this deployment must nest component configuration under the `allInOne` parameter. +| +| + +|`agent:` +|Configuration options that define the Agent. +| +| + +|`collector:` +|Configuration options that define the Tempo Collector. +| +| + +|`sampling:` +|Configuration options that define the sampling strategies for tracing. +| +| + +|`storage:` +|Configuration options that define the storage. All storage-related options must be placed under `storage`, rather than under the `allInOne` or other component options. +| +| + +|`query:` +|Configuration options that define the Query service. +| +| + +|`ingester:` +|Configuration options that define the Ingester service. +| +| +|=== + +The following example YAML is the minimum required to create a {TempoName} deployment using the default settings. + +.Example minimum required dist-tracing-all-in-one.yaml +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: tempo-all-in-one-inmemory +---- diff --git a/modules/distr-tracing-tempo-config-ingester.adoc b/modules/distr-tracing-tempo-config-ingester.adoc new file mode 100644 index 000000000000..4d56ba6e3de8 --- /dev/null +++ b/modules/distr-tracing-tempo-config-ingester.adoc @@ -0,0 +1,76 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-ingester_{context}"] += Ingester configuration options + +Ingester is a service that reads from a Kafka topic and writes to the Elasticsearch storage backend. If you are using the `allInOne` or `production` deployment strategies, you do not need to configure the Ingester service. + +.Tempo parameters passed to the Ingester +[options="header"] +[cols="l, a, a"] +|=== +|Parameter |Description |Values +|spec: + ingester: + options: {} +|Configuration options that define the Ingester service. +| + +|options: + deadlockInterval: +|Specifies the interval, in seconds or minutes, that the Ingester must wait for a message before terminating. +The deadlock interval is disabled by default (set to `0`), to avoid terminating the Ingester when no messages arrive during system initialization. +|Minutes and seconds, for example, `1m0s`. Default value is `0`. + +|options: + kafka: + consumer: + topic: +|The `topic` parameter identifies the Kafka configuration used by the collector to produce the messages, and the Ingester to consume the messages. +|Label for the consumer. For example, `tempo-spans`. + +|options: + kafka: + consumer: + brokers: +|Identifies the Kafka configuration used by the Ingester to consume the messages. +|Label for the broker, for example, `my-cluster-kafka-brokers.kafka:9092`. + +|options: + log-level: +|Logging level for the Ingester. +|Possible values: `debug`, `info`, `warn`, `error`, `fatal`, `dpanic`, `panic`. +|=== + +.Streaming Collector and Ingester example +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: simple-streaming +spec: + strategy: streaming + collector: + options: + kafka: + producer: + topic: tempo-spans + brokers: my-cluster-kafka-brokers.kafka:9092 + ingester: + options: + kafka: + consumer: + topic: tempo-spans + brokers: my-cluster-kafka-brokers.kafka:9092 + ingester: + deadlockInterval: 5 + storage: + type: elasticsearch + options: + es: + server-urls: http://elasticsearch:9200 +---- diff --git a/modules/distr-tracing-tempo-config-query.adoc b/modules/distr-tracing-tempo-config-query.adoc new file mode 100644 index 000000000000..2daf364b041f --- /dev/null +++ b/modules/distr-tracing-tempo-config-query.adoc @@ -0,0 +1,67 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-query_{context}"] += Query configuration options + +Query is a service that retrieves traces from storage and hosts the user interface to display them. + +.Parameters used by the {TempoName} Operator to define Query +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value + +|spec: + query: + replicas: +|Specifies the number of Query replicas to create. +|Integer, for example, `2` +| +|=== + + +.Configuration parameters passed to Query +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value + +|spec: + query: + options: {} +|Configuration options that define the Query service. +| +| + +|options: + log-level: +|Logging level for Query. +|Possible values: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. +| + +|options: + query: + base-path: +|The base path for all tempo-query HTTP routes can be set to a non-root value, for example, `/tempo` would cause all UI URLs to start with `/tempo`. This can be useful when running tempo-query behind a reverse proxy. +|/ +| +|=== + +.Sample Query configuration +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: "Tempo" +metadata: + name: "my-tempo" +spec: + strategy: allInOne + allInOne: + options: + log-level: debug + query: + base-path: /tempo +---- diff --git a/modules/distr-tracing-tempo-config-sampling.adoc b/modules/distr-tracing-tempo-config-sampling.adoc new file mode 100644 index 000000000000..df6e598c3ad8 --- /dev/null +++ b/modules/distr-tracing-tempo-config-sampling.adoc @@ -0,0 +1,99 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-sampling_{context}"] += Distributed tracing sampling configuration options + +The {TempoName} Operator can be used to define sampling strategies that will be supplied to tracers that have been configured to use a remote sampler. + +While all traces are generated, only a few are sampled. Sampling a trace marks the trace for further processing and storage. + +[NOTE] +==== +This is not relevant if a trace was started by the Envoy proxy, as the sampling decision is made there. The Tempo sampling decision is only relevant when the trace is started by an application using the client. +==== + +When a service receives a request that contains no trace context, the client starts a new trace, assigns it a random trace ID, and makes a sampling decision based on the currently installed sampling strategy. The sampling decision propagates to all subsequent requests in the trace so that other services are not making the sampling decision again. + +{TempoShortName} libraries support the following samplers: + +* *Probabilistic* - The sampler makes a random sampling decision with the probability of sampling equal to the value of the `sampling.param` property. For example, using `sampling.param=0.1` samples approximately 1 in 10 traces. + +* *Rate Limiting* - The sampler uses a leaky bucket rate limiter to ensure that traces are sampled with a certain constant rate. For example, using `sampling.param=2.0` samples requests with the rate of 2 traces per second. + +.Tempo sampling options +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|spec: + sampling: + options: {} + default_strategy: + service_strategy: +|Configuration options that define the sampling strategies for tracing. +| +|If you do not provide configuration, the Collectors will return the default probabilistic sampling policy with 0.001 (0.1%) probability for all services. + +|default_strategy: + type: +service_strategy: + type: +|Sampling strategy to use. See descriptions above. +|Valid values are `probabilistic`, and `ratelimiting`. +|`probabilistic` + +|default_strategy: + param: +service_strategy: + param: +|Parameters for the selected sampling strategy. +|Decimal and integer values (0, .1, 1, 10) +|1 +|=== + +This example defines a default sampling strategy that is probabilistic, with a 50% chance of the trace instances being sampled. + +.Probabilistic sampling example +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: with-sampling +spec: + sampling: + options: + default_strategy: + type: probabilistic + param: 0.5 + service_strategies: + - service: alpha + type: probabilistic + param: 0.8 + operation_strategies: + - operation: op1 + type: probabilistic + param: 0.2 + - operation: op2 + type: probabilistic + param: 0.4 + - service: beta + type: ratelimiting + param: 5 +---- + +If there are no user-supplied configurations, the {TempoShortName} uses the following settings: + +.Default sampling +[source,yaml] +---- +spec: + sampling: + options: + default_strategy: + type: probabilistic + param: 1 +---- diff --git a/modules/distr-tracing-tempo-config-storage.adoc b/modules/distr-tracing-tempo-config-storage.adoc new file mode 100644 index 000000000000..366d982e297d --- /dev/null +++ b/modules/distr-tracing-tempo-config-storage.adoc @@ -0,0 +1,728 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: REFERENCE +[id="distr-tracing-tempo-config-storage_{context}"] += Distributed tracing storage configuration options + +You configure storage for the Collector, Ingester, Distributor, and Query services under `spec.storage`. Multiple instances of each of these components can be provisioned as required for performance and resilience purposes. + +.General storage parameters used by the {TempoName} Operator to define distributed tracing storage + +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|spec: + storage: + type: +|Type of storage to use for the deployment. +|`memory` or `elasticsearch`. +Memory storage is only appropriate for development, testing, demonstrations, and proof of concept environments as the data does not persist if the pod is shut down. For production environments {TempoShortName} supports Elasticsearch for persistent storage. +|`memory` + +|storage: + secretname: +|Name of the secret, for example `tracing-secret`. +| +|N/A + +|storage: + options: {} +|Configuration options that define the storage. +| +| +|=== + +.Elasticsearch index cleaner parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|storage: + esIndexCleaner: + enabled: +|When using Elasticsearch storage, by default a job is created to clean old traces from the index. This parameter enables or disables the index cleaner job. +|`true`/ `false` +|`true` + +|storage: + esIndexCleaner: + numberOfDays: +|Number of days to wait before deleting an index. +|Integer value +|`7` + +|storage: + esIndexCleaner: + schedule: +|Defines the schedule for how often to clean the Elasticsearch index. +|Cron expression +|"55 23 * * *" +|=== + +[id="distributed-tracing-config-auto-provisioning-es_{context}"] +== Auto-provisioning an Elasticsearch instance + +When you deploy a Tempo custom resource, the {TempoName} Operator uses the OpenShift Elasticsearch Operator to create an Elasticsearch cluster based on the configuration provided in the `storage` section of the custom resource file. The {TempoName} Operator will provision Elasticsearch if the following configurations are set: + +* `spec.storage:type` is set to `elasticsearch` +* `spec.storage.elasticsearch.doNotProvision` set to `false` +* `spec.storage.options.es.server-urls` is not defined, that is, there is no connection to an Elasticsearch instance that was not provisioned by the Red Hat Elasticsearch Operator. + +When provisioning Elasticsearch, the {TempoName} Operator sets the Elasticsearch custom resource `name` to the value of `spec.storage.elasticsearch.name` from the Tempo custom resource. If you do not specify a value for `spec.storage.elasticsearch.name`, the Operator uses `elasticsearch`. + +.Restrictions + +* You can have only one {TempoShortName} with self-provisioned Elasticsearch instance per namespace. The Elasticsearch cluster is meant to be dedicated for a single {TempoShortName} instance. +* There can be only one Elasticsearch per namespace. + +[NOTE] +==== +If you already have installed Elasticsearch as part of OpenShift Logging, the {TempoName} Operator can use the installed OpenShift Elasticsearch Operator to provision storage. +==== + +The following configuration parameters are for a _self-provisioned_ Elasticsearch instance, that is an instance created by the {TempoName} Operator using the OpenShift Elasticsearch Operator. You specify configuration options for self-provisioned Elasticsearch under `spec:storage:elasticsearch` in your configuration file. + +.Elasticsearch resource configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|elasticsearch: + properties: + doNotProvision: +|Use to specify whether or not an Elasticsearch instance should be provisioned by the {TempoName} Operator. +|`true`/`false` +|`true` + +|elasticsearch: + properties: + name: +|Name of the Elasticsearch instance. The {TempoName} Operator uses the Elasticsearch instance specified in this parameter to connect to Elasticsearch. +|string +|`elasticsearch` + +|elasticsearch: + nodeCount: +|Number of Elasticsearch nodes. For high availability use at least 3 nodes. Do not use 2 nodes as “split brain” problem can happen. +|Integer value. For example, Proof of concept = 1, +Minimum deployment =3 +|3 + +|elasticsearch: + resources: + requests: + cpu: +|Number of central processing units for requests, based on your environment's configuration. +|Specified in cores or millicores, for example, 200m, 0.5, 1. For example, Proof of concept = 500m, +Minimum deployment =1 +|1 + +|elasticsearch: + resources: + requests: + memory: +|Available memory for requests, based on your environment's configuration. +|Specified in bytes, for example, 200Ki, 50Mi, 5Gi. For example, Proof of concept = 1Gi, +Minimum deployment = 16Gi* +|16Gi + +|elasticsearch: + resources: + limits: + cpu: +|Limit on number of central processing units, based on your environment's configuration. +|Specified in cores or millicores, for example, 200m, 0.5, 1. For example, Proof of concept = 500m, +Minimum deployment =1 +| + +|elasticsearch: + resources: + limits: + memory: +|Available memory limit based on your environment's configuration. +|Specified in bytes, for example, 200Ki, 50Mi, 5Gi. For example, Proof of concept = 1Gi, +Minimum deployment = 16Gi* +| + +|elasticsearch: + redundancyPolicy: +|Data replication policy defines how Elasticsearch shards are replicated across data nodes in the cluster. If not specified, the {TempoName} Operator automatically determines the most appropriate replication based on number of nodes. +|`ZeroRedundancy`(no replica shards), `SingleRedundancy`(one replica shard), `MultipleRedundancy`(each index is spread over half of the Data nodes), `FullRedundancy` (each index is fully replicated on every Data node in the cluster). +| + +|elasticsearch: + useCertManagement: +|Use to specify whether or not {TempoShortName} should use the certificate management feature of the Red Hat Elasticsearch Operator. This feature was added to {logging-title} 5.2 in {product-title} 4.7 and is the preferred setting for new Tempo deployments. +|`true`/`false` +|`true` + +| +3+|*Each Elasticsearch node can operate with a lower memory setting though this is NOT recommended for production deployments. For production use, you should have no less than 16Gi allocated to each pod by default, but preferably allocate as much as you can, up to 64Gi per pod. +|=== + +.Production storage example +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: simple-prod +spec: + strategy: production + storage: + type: elasticsearch + elasticsearch: + nodeCount: 3 + resources: + requests: + cpu: 1 + memory: 16Gi + limits: + memory: 16Gi +---- + +.Storage example with persistent storage: +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: simple-prod +spec: + strategy: production + storage: + type: elasticsearch + elasticsearch: + nodeCount: 1 + storage: # <1> + storageClassName: gp2 + size: 5Gi + resources: + requests: + cpu: 200m + memory: 4Gi + limits: + memory: 4Gi + redundancyPolicy: ZeroRedundancy +---- + +<1> Persistent storage configuration. In this case AWS `gp2` with `5Gi` size. When no value is specified, {TempoShortName} uses `emptyDir`. The OpenShift Elasticsearch Operator provisions `PersistentVolumeClaim` and `PersistentVolume` which are not removed with {TempoShortName} instance. You can mount the same volumes if you create a {TempoShortName} instance with the same name and namespace. + + +[id="distributed-tracing-config-external-es_{context}"] +== Connecting to an existing Elasticsearch instance + +You can use an existing Elasticsearch cluster for storage with {DTShortName}. An existing Elasticsearch cluster, also known as an _external_ Elasticsearch instance, is an instance that was not installed by the {TempoName} Operator or by the Red Hat Elasticsearch Operator. + +When you deploy a Tempo custom resource, the {TempoName} Operator will not provision Elasticsearch if the following configurations are set: + +* `spec.storage.elasticsearch.doNotProvision` set to `true` +* `spec.storage.options.es.server-urls` has a value +* `spec.storage.elasticsearch.name` has a value, or if the Elasticsearch instance name is `elasticsearch`. + +The {TempoName} Operator uses the Elasticsearch instance specified in `spec.storage.elasticsearch.name` to connect to Elasticsearch. + +.Restrictions + +* You cannot share or reuse a {product-title} logging Elasticsearch instance with {TempoShortName}. The Elasticsearch cluster is meant to be dedicated for a single {TempoShortName} instance. + +[NOTE] +==== +Red Hat does not provide support for your external Elasticsearch instance. You can review the tested integrations matrix on the link:https://access.redhat.com/articles/5381021[Customer Portal]. +==== + +The following configuration parameters are for an already existing Elasticsearch instance, also known as an _external_ Elasticsearch instance. In this case, you specify configuration options for Elasticsearch under `spec:storage:options:es` in your custom resource file. + +.General ES configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es: + server-urls: +|URL of the Elasticsearch instance. +|The fully-qualified domain name of the Elasticsearch server. +|`http://elasticsearch..svc:9200` + +|es: + max-doc-count: +|The maximum document count to return from an Elasticsearch query. This will also apply to aggregations. If you set both `es.max-doc-count` and `es.max-num-spans`, Elasticsearch will use the smaller value of the two. +| +|10000 + +|es: + max-num-spans: +|[*Deprecated* - Will be removed in a future release, use `es.max-doc-count` instead.] The maximum number of spans to fetch at a time, per query, in Elasticsearch. If you set both `es.max-num-spans` and `es.max-doc-count`, Elasticsearch will use the smaller value of the two. +| +|10000 + +|es: + max-span-age: +|The maximum lookback for spans in Elasticsearch. +| +|72h0m0s + +|es: + sniffer: +|The sniffer configuration for Elasticsearch. The client uses the sniffing process to find all nodes automatically. Disabled by default. +|`true`/ `false` +|`false` + +|es: + sniffer-tls-enabled: +|Option to enable TLS when sniffing an Elasticsearch Cluster. The client uses the sniffing process to find all nodes automatically. Disabled by default +|`true`/ `false` +|`false` + +|es: + timeout: +|Timeout used for queries. When set to zero there is no timeout. +| +|0s + +|es: + username: +|The username required by Elasticsearch. The basic authentication also loads CA if it is specified. See also `es.password`. +| +| + +|es: + password: +|The password required by Elasticsearch. See also, `es.username`. +| +| + +|es: + version: +|The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch. +| +|0 +|=== + +.ES data replication parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es: + num-replicas: +|The number of replicas per index in Elasticsearch. +| +|1 + +|es: + num-shards: +|The number of shards per index in Elasticsearch. +| +|5 +|=== + +.ES index configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es: + create-index-templates: +|Automatically create index templates at application startup when set to `true`. When templates are installed manually, set to `false`. +|`true`/ `false` +|`true` + +|es: + index-prefix: +|Optional prefix for {TempoShortName} indices. For example, setting this to "production" creates indices named "production-tracing-*". +| +| +|=== + +.ES bulk processor configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es: + bulk: + actions: +|The number of requests that can be added to the queue before the bulk processor decides to commit updates to disk. +| +|1000 + +//What is the default here? The original text said "Set to zero to disable. By default, this is disabled." +|es: + bulk: + flush-interval: +|A `time.Duration` after which bulk requests are committed, regardless of other thresholds. To disable the bulk processor flush interval, set this to zero. +| +|200ms + +|es: + bulk: + size: +|The number of bytes that the bulk requests can take up before the bulk processor decides to commit updates to disk. +| +|5000000 + +|es: + bulk: + workers: +|The number of workers that are able to receive and commit bulk requests to Elasticsearch. +| +|1 +|=== + +.ES TLS configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es: + tls: + ca: +|Path to a TLS Certification Authority (CA) file used to verify the remote servers. +| +|Will use the system truststore by default. + +|es: + tls: + cert: +|Path to a TLS Certificate file, used to identify this process to the remote servers. +| +| + +|es: + tls: + enabled: +|Enable transport layer security (TLS) when talking to the remote servers. Disabled by default. +|`true`/ `false` +|`false` + +|es: + tls: + key: +|Path to a TLS Private Key file, used to identify this process to the remote servers. +| +| + +|es: + tls: + server-name: +|Override the expected TLS server name in the certificate of the remote servers. +| +| +//Clarification of "if specified" for `token-file` and `username`, does that mean if this is set? Or that it only loads the CA if one is specified (that is, if es.tls.ca has a value?) +|es: + token-file: +|Path to a file containing the bearer token. This flag also loads the Certification Authority (CA) file if it is specified. +| +| +|=== + +.ES archive configuration parameters +[options="header"] +[cols="l, a, a, a"] +|=== +|Parameter |Description |Values |Default value +|es-archive: + bulk: + actions: +|The number of requests that can be added to the queue before the bulk processor decides to commit updates to disk. +| +|0 + +//What is the default here? The original text said "Set to zero to disable. By default, this is disabled." +|es-archive: + bulk: + flush-interval: +|A `time.Duration` after which bulk requests are committed, regardless of other thresholds. To disable the bulk processor flush interval, set this to zero. +| +|0s + +|es-archive: + bulk: + size: +|The number of bytes that the bulk requests can take up before the bulk processor decides to commit updates to disk. +| +|0 + +|es-archive: + bulk: + workers: +|The number of workers that are able to receive and commit bulk requests to Elasticsearch. +| +|0 + +|es-archive: + create-index-templates: +|Automatically create index templates at application startup when set to `true`. When templates are installed manually, set to `false`. +|`true`/ `false` +|`false` + +|es-archive: + enabled: +|Enable extra storage. +|`true`/ `false` +|`false` + +|es-archive: + index-prefix: +|Optional prefix for {TempoShortName} indices. For example, setting this to "production" creates indices named "production-tracing-*". +| +| + +|es-archive: + max-doc-count: +|The maximum document count to return from an Elasticsearch query. This will also apply to aggregations. +| +|0 + +|es-archive: + max-num-spans: +|[*Deprecated* - Will be removed in a future release, use `es-archive.max-doc-count` instead.] The maximum number of spans to fetch at a time, per query, in Elasticsearch. +| +|0 + +|es-archive: + max-span-age: +|The maximum lookback for spans in Elasticsearch. +| +|0s + +|es-archive: + num-replicas: +|The number of replicas per index in Elasticsearch. +| +|0 + +|es-archive: + num-shards: +|The number of shards per index in Elasticsearch. +| +|0 + +|es-archive: + password: +|The password required by Elasticsearch. See also, `es.username`. +| +| + +|es-archive: + server-urls: +|The comma-separated list of Elasticsearch servers. Must be specified as fully qualified URLs, for example, `\http://localhost:9200`. +| +| + +|es-archive: + sniffer: +|The sniffer configuration for Elasticsearch. The client uses the sniffing process to find all nodes automatically. Disabled by default. +|`true`/ `false` +|`false` + +|es-archive: + sniffer-tls-enabled: +|Option to enable TLS when sniffing an Elasticsearch Cluster. The client uses the sniffing process to find all nodes automatically. Disabled by default. +|`true`/ `false` +|`false` + +|es-archive: + timeout: +|Timeout used for queries. When set to zero there is no timeout. +| +|0s + +|es-archive: + tls: + ca: +|Path to a TLS Certification Authority (CA) file used to verify the remote servers. +| +|Will use the system truststore by default. + +|es-archive: + tls: + cert: +|Path to a TLS Certificate file, used to identify this process to the remote servers. +| +| + +|es-archive: + tls: + enabled: +|Enable transport layer security (TLS) when talking to the remote servers. Disabled by default. +|`true`/ `false` +|`false` + +|es-archive: + tls: + key: +|Path to a TLS Private Key file, used to identify this process to the remote servers. +| +| + +|es-archive: + tls: + server-name: +|Override the expected TLS server name in the certificate of the remote servers. +| +| + +//Clarification of "if specified" for next two rows, does that mean if this is set? Or that it only loads the CA if one is specified (that is, if es-archive.tls.ca has a value?) +|es-archive: + token-file: +|Path to a file containing the bearer token. This flag also loads the Certification Authority (CA) file if it is specified. +| +| + +|es-archive: + username: +|The username required by Elasticsearch. The basic authentication also loads CA if it is specified. See also `es-archive.password`. +| +| + +|es-archive: + version: +|The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch. +| +|0 +|=== + + +.Storage example with volume mounts +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: simple-prod +spec: + strategy: production + storage: + type: elasticsearch + options: + es: + server-urls: https://quickstart-es-http.default.svc:9200 + index-prefix: my-prefix + tls: + ca: /es/certificates/ca.crt + secretName: tracing-secret + volumeMounts: + - name: certificates + mountPath: /es/certificates/ + readOnly: true + volumes: + - name: certificates + secret: + secretName: quickstart-es-http-certs-public +---- + +The following example shows a Tempo CR using an external Elasticsearch cluster with TLS CA certificate mounted from a volume and user/password stored in a secret. + +.External Elasticsearch example: +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: simple-prod +spec: + strategy: production + storage: + type: elasticsearch + options: + es: + server-urls: https://quickstart-es-http.default.svc:9200 # <1> + index-prefix: my-prefix + tls: # <2> + ca: /es/certificates/ca.crt + secretName: tracing-secret # <3> + volumeMounts: # <4> + - name: certificates + mountPath: /es/certificates/ + readOnly: true + volumes: + - name: certificates + secret: + secretName: quickstart-es-http-certs-public +---- +<1> URL to Elasticsearch service running in default namespace. +<2> TLS configuration. In this case only CA certificate, but it can also contain es.tls.key and es.tls.cert when using mutual TLS. +<3> Secret which defines environment variables ES_PASSWORD and ES_USERNAME. Created by kubectl create secret generic tracing-secret --from-literal=ES_PASSWORD=changeme --from-literal=ES_USERNAME=elastic +<4> Volume mounts and volumes which are mounted into all storage components. + +[id="distr-tracing-manage-es-certificates_{context}"] += Managing certificates with Elasticsearch + +You can create and manage certificates using the Red Hat Elasticsearch Operator. Managing certificates using the Red Hat Elasticsearch Operator also lets you use a single Elasticsearch cluster with multiple Tempo Collectors. + +:FeatureName: Managing certificates with Elasticsearch +include::snippets/technology-preview.adoc[leveloffset=+1] + +Starting with version 2.4, the {TempoName} Operator delegates certificate creation to the Red Hat Elasticsearch Operator by using the following annotations in the Elasticsearch custom resource: + +* `logging.openshift.io/elasticsearch-cert-management: "true"` +* `logging.openshift.io/elasticsearch-cert.tempo-: "user.tempo"` +* `logging.openshift.io/elasticsearch-cert.curator-: "system.logging.curator"` + +Where the `` is the name of the Elasticsearch node. For example, if you create an Elasticsearch node named `custom-es`, your custom resource might look like the following example. + +.Example Elasticsearch CR showing annotations +[source,yaml] +---- +apiVersion: logging.openshift.io/v1 +kind: Elasticsearch +metadata: + annotations: + logging.openshift.io/elasticsearch-cert-management: "true" + logging.openshift.io/elasticsearch-cert.tempo-custom-es: "user.tempo" + logging.openshift.io/elasticsearch-cert.curator-custom-es: "system.logging.curator" + name: custom-es +spec: + managementState: Managed + nodeSpec: + resources: + limits: + memory: 16Gi + requests: + cpu: 1 + memory: 16Gi + nodes: + - nodeCount: 3 + proxyResources: {} + resources: {} + roles: + - master + - client + - data + storage: {} + redundancyPolicy: ZeroRedundancy +---- + +.Prerequisites + +* {product-title} 4.7 +* {logging-title} 5.2 +* The Elasticsearch node and the Tempo instances must be deployed in the same namespace. For example, `tracing-system`. + +You enable certificate management by setting `spec.storage.elasticsearch.useCertManagement` to `true` in the Tempo custom resource. + +.Example showing `useCertManagement` +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: tempo-prod +spec: + strategy: production + storage: + type: elasticsearch + elasticsearch: + name: custom-es + doNotProvision: true + useCertManagement: true +---- + +The {TempoName} Operator sets the Elasticsearch custom resource `name` to the value of `spec.storage.elasticsearch.name` from the Tempo custom resource when provisioning Elasticsearch. + +The certificates are provisioned by the Red Hat Elasticsearch Operator and the {TempoName} Operator injects the certificates. diff --git a/modules/distr-tracing-tempo-deploy-default.adoc b/modules/distr-tracing-tempo-deploy-default.adoc new file mode 100644 index 000000000000..a050672766c5 --- /dev/null +++ b/modules/distr-tracing-tempo-deploy-default.adoc @@ -0,0 +1,132 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-deploy-default_{context}"] += Deploying the {DTShortName} default strategy from the web console + +The custom resource definition (CRD) defines the configuration used when you deploy an instance of {DTProductName}. The default CR is configured with minimal resources to ensure that you can successfully install it on a default {product-title} installation. You can use this default configuration to create a basic {TempoName} instance, or you can define your own custom resource file. + +[NOTE] +==== +In-memory storage is not persistent. If the Tempo pod shuts down, restarts, or is replaced, your trace data will be lost. For persistent storage, you must use the `production` or `streaming` strategies, which use Elasticsearch as the default storage. +==== + +.Prerequisites + +* The {TempoName} Operator has been installed. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the cluster as a user with the `cluster-admin` role. +* Supported Object Store (AWS S3, Google Cloud Storage, Azure, Minio, OpenShift Data Foundation) + +.Procedure + +. Log in to the {product-title} web console as a user with the `cluster-admin` role. + +. Create a new project, for example `tracing-system`. ++ +[NOTE] +==== +If you are installing as part of Service Mesh, the {DTShortName} resources must be installed in the same namespace as the `ServiceMeshControlPlane` resource, for example `istio-system`. +==== ++ +.. Navigate to *Home* -> *Projects*. + +.. Click *Create Project*. + +.. Enter `tracing-system` in the *Name* field. + +.. Click *Create*. + +. Navigate to *Operators* -> *Installed Operators*. + +. If necessary, select `tracing-system` from the *Project* menu. You may have to wait a few moments for the Operators to be copied to the new project. + +. Click the {TempoName} Operator. On the *Details* tab, under *Provided APIs*, the Operator provides a single link. + +. Under *Tempo*, click *Create Instance*. + +. On the *Create Tempo* page, to install using the defaults, click *Create* to create the {TempoShortName} instance. + +. On the *Tempos* page, click the name of the {TempoShortName} instance, for example, `example`. + +. On the *Tempo Details* page, click the *Resources* tab. Wait until the pod has a status of "Running" before continuing. + + +[id="distr-tracing-deploy-default-cli_{context}"] +== Deploying the {DTShortName} default strategy from the CLI + +Follow this procedure to create an instance of {TempoShortName} from the command line. + +.Prerequisites + +* The {TempoName} Operator has been installed and verified. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the OpenShift CLI (`oc`) that matches your {product-title} version. +* You have access to the cluster as a user with the `cluster-admin` role. + +.Procedure + +. Log in to the {product-title} CLI as a user with the `cluster-admin` role. ++ +[source,terminal] +---- +$ oc login --username= https://:8443 +---- + +. Create a new project named `tracing-system`. ++ +[source,terminal] +---- +$ oc new-project tracing-system +---- + +. Create a Secret YAML file to specify the secret for your object storage bucket. For instance, if you are using AWS S3, create a file that uses the `access_key_id` and `access_key_secret` fields to specify your AWS credentials and bucketnames, endpoint and region to define the object storage location. For example: + +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +stringData: + endpoint: http://minio.minio.svc:9000 + bucket: tempo + access_key_id: tempo + access_key_secret: supersecret +type: Opaque +---- + +. Create a custom resource file named `tempo.yaml` that contains the following text: ++ +.Example tempo-all-in-one.yaml +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: example +---- + +. Run the following command to deploy {TempoShortName}: ++ +[source,terminal] +---- +$ oc create -n tracing-system -f tempo.yaml +---- + +. Run the following command to watch the progress of the pods during the installation process: ++ +[source,terminal] +---- +$ oc get pods -n tracing-system -w +---- ++ +After the installation process has completed, you should see output similar to the following example: ++ +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +tempo-all-in-one-inmemory-cdff7897b-qhfdx 2/2 Running 0 24s +---- diff --git a/modules/distr-tracing-tempo-deploy-production-es.adoc b/modules/distr-tracing-tempo-deploy-production-es.adoc new file mode 100644 index 000000000000..2aff731995b5 --- /dev/null +++ b/modules/distr-tracing-tempo-deploy-production-es.adoc @@ -0,0 +1,136 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-deploy-production_{context}"] += Deploying the {DTShortName} production strategy from the web console + +The `production` deployment strategy is intended for production environments that require a more scalable and highly available architecture, and where long-term storage of trace data is important. + +.Prerequisites + +* The OpenShift Elasticsearch Operator has been installed. +* The {TempoName} Operator has been installed. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the cluster as a user with the `cluster-admin` role. + +.Procedure + +. Log in to the {product-title} web console as a user with the `cluster-admin` role. + +. Create a new project, for example `tracing-system`. ++ +[NOTE] +==== +If you are installing as part of Service Mesh, the {DTShortName} resources must be installed in the same namespace as the `ServiceMeshControlPlane` resource, for example `istio-system`. +==== ++ +.. Navigate to *Home* -> *Projects*. + +.. Click *Create Project*. + +.. Enter `tracing-system` in the *Name* field. + +.. Click *Create*. + +. Navigate to *Operators* -> *Installed Operators*. + +. If necessary, select `tracing-system` from the *Project* menu. You may have to wait a few moments for the Operators to be copied to the new project. + +. Click the {TempoName} Operator. On the *Overview* tab, under *Provided APIs*, the Operator provides a single link. + +. Under *Tempo*, click *Create Instance*. + +. On the *Create Tempo* page, replace the default `all-in-one` YAML text with your production YAML configuration, for example: + ++ +.Example tempo-production.yaml file with Elasticsearch +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: tempo-production + namespace: +spec: + strategy: production + ingress: + security: oauth-proxy + storage: + type: elasticsearch + elasticsearch: + nodeCount: 3 + redundancyPolicy: SingleRedundancy + esIndexCleaner: + enabled: true + numberOfDays: 7 + schedule: 55 23 * * * + esRollover: + schedule: '*/30 * * * *' +---- ++ + +. Click *Create* to create the {TempoShortName} instance. + +. On the *Tempos* page, click the name of the {TempoShortName} instance, for example, `tempo-prod-elasticsearch`. + +. On the *Tempo Details* page, click the *Resources* tab. Wait until all the pods have a status of "Running" before continuing. + + +[id="distr-tracing-deploy-production-cli_{context}"] +== Deploying the {DTShortName} production strategy from the CLI + +Follow this procedure to create an instance of {TempoShortName} from the command line. + +.Prerequisites + +* The OpenShift Elasticsearch Operator has been installed. +* The {TempoName} Operator has been installed. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the OpenShift CLI (`oc`) that matches your {product-title} version. +* You have access to the cluster as a user with the `cluster-admin` role. + +.Procedure + +. Log in to the {product-title} CLI as a user with the `cluster-admin` role. ++ +[source,terminal] +---- +$ oc login --username= https://:8443 +---- + +. Create a new project named `tracing-system`. ++ +[source,terminal] +---- +$ oc new-project tracing-system +---- + +. Create a custom resource file named `tempo-production.yaml` that contains the text of the example file in the previous procedure. + +. Run the following command to deploy {TempoShortName}: ++ +[source,terminal] +---- +$ oc create -n tracing-system -f tempo-production.yaml +---- ++ +. Run the following command to watch the progress of the pods during the installation process: ++ +[source,terminal] +---- +$ oc get pods -n tracing-system -w +---- ++ +After the installation process has completed, you should see output similar to the following example: ++ +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +elasticsearch-cdm-temposystemtempoproduction-1-6676cf568gwhlw 2/2 Running 0 10m +elasticsearch-cdm-temposystemtempoproduction-2-bcd4c8bf5l6g6w 2/2 Running 0 10m +elasticsearch-cdm-temposystemtempoproduction-3-844d6d9694hhst 2/2 Running 0 10m +tempo-production-collector-94cd847d-jwjlj 1/1 Running 3 8m32s +tempo-production-query-5cbfbd499d-tv8zf 3/3 Running 3 8m32s +---- diff --git a/modules/distr-tracing-tempo-deploy-streaming.adoc b/modules/distr-tracing-tempo-deploy-streaming.adoc new file mode 100644 index 000000000000..522c7068ac62 --- /dev/null +++ b/modules/distr-tracing-tempo-deploy-streaming.adoc @@ -0,0 +1,151 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-deploy-streaming_{context}"] += Deploying the {DTShortName} streaming strategy from the web console + +The `streaming` deployment strategy is intended for production environments that require a more scalable and highly available architecture, and where long-term storage of trace data is important. + +The `streaming` strategy provides a streaming capability that sits between the Collector and the Elasticsearch storage. This reduces the pressure on the storage under high load situations, and enables other trace post-processing capabilities to tap into the real-time span data directly from the Kafka streaming platform. + +[NOTE] +==== +The streaming strategy requires an additional Red Hat subscription for AMQ Streams. If you do not have an AMQ Streams subscription, contact your sales representative for more information. +==== + +[NOTE] +==== +The streaming deployment strategy is currently unsupported on {ibmzProductName}. +==== + +.Prerequisites + +* The AMQ Streams Operator has been installed. If using version 1.4.0 or higher you can use self-provisioning. Otherwise you must create the Kafka instance. +* The {TempoName} Operator has been installed. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the cluster as a user with the `cluster-admin` role. + +.Procedure + +. Log in to the {product-title} web console as a user with the `cluster-admin` role. + +. Create a new project, for example `tracing-system`. + ++ ++ + +.. Navigate to *Home* -> *Projects*. + +.. Click *Create Project*. + +.. Enter `tracing-system` in the *Name* field. + +.. Click *Create*. + +. Navigate to *Operators* -> *Installed Operators*. + +. If necessary, select `tracing-system` from the *Project* menu. You may have to wait a few moments for the Operators to be copied to the new project. + +. Click the {TempoName} Operator. On the *Overview* tab, under *Provided APIs*, the Operator provides a single link. + +. Under *Tempo*, click *Create Instance*. + +. On the *Create Tempo* page, replace the default `all-in-one` YAML text with your streaming YAML configuration, for example: + +.Example tempo-streaming.yaml file +[source,yaml] +---- +apiVersion: tempotracing.io/v1 +kind: Tempo +metadata: + name: tempo-streaming +spec: + strategy: streaming + collector: + options: + kafka: + producer: + topic: tempo-spans + #Note: If brokers are not defined,AMQStreams 1.4.0+ will self-provision Kafka. + brokers: my-cluster-kafka-brokers.kafka:9092 + storage: + type: elasticsearch + ingester: + options: + kafka: + consumer: + topic: tempo-spans + brokers: my-cluster-kafka-brokers.kafka:9092 + +---- +//TODO - find out if this storage configuration is correct for OpenShift + +. Click *Create* to create the {TempoShortName} instance. + +. On the *Tempos* page, click the name of the {TempoShortName} instance, for example, `tempo-streaming`. + +. On the *Tempo Details* page, click the *Resources* tab. Wait until all the pods have a status of "Running" before continuing. + + +[id="distr-tracing-deploy-streaming-cli_{context}"] +== Deploying the {DTShortName} streaming strategy from the CLI + +Follow this procedure to create an instance of {TempoShortName} from the command line. + +.Prerequisites + +* The AMQ Streams Operator has been installed. If using version 1.4.0 or higher you can use self-provisioning. Otherwise you must create the Kafka instance. +* The {TempoName} Operator has been installed. +* You have reviewed the instructions for how to customize the deployment. +* You have access to the OpenShift CLI (`oc`) that matches your {product-title} version. +* You have access to the cluster as a user with the `cluster-admin` role. + +Procedure + +. Log in to the {product-title} CLI as a user with the `cluster-admin` role. ++ +[source,terminal] +---- +$ oc login --username= https://:8443 +---- + +. Create a new project named `tracing-system`. ++ +[source,terminal] +---- +$ oc new-project tracing-system +---- + +. Create a custom resource file named `tempo-streaming.yaml` that contains the text of the example file in the previous procedure. + +. Run the following command to deploy Tempo: ++ +[source,terminal] +---- +$ oc create -n tracing-system -f tempo-streaming.yaml +---- ++ +. Run the following command to watch the progress of the pods during the installation process: ++ +[source,terminal] +---- +$ oc get pods -n tracing-system -w +---- ++ +After the installation process has completed, you should see output similar to the following example: ++ +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +elasticsearch-cdm-temposystemtempostreaming-1-697b66d6fcztcnn 2/2 Running 0 5m40s +elasticsearch-cdm-temposystemtempostreaming-2-5f4b95c78b9gckz 2/2 Running 0 5m37s +elasticsearch-cdm-temposystemtempostreaming-3-7b6d964576nnz97 2/2 Running 0 5m5s +tempo-streaming-collector-6f6db7f99f-rtcfm 1/1 Running 0 80s +tempo-streaming-entity-operator-6b6d67cc99-4lm9q 3/3 Running 2 2m18s +tempo-streaming-ingester-7d479847f8-5h8kc 1/1 Running 0 80s +tempo-streaming-kafka-0 2/2 Running 0 3m1s +tempo-streaming-query-65bf5bb854-ncnc7 3/3 Running 0 80s +tempo-streaming-zookeeper-0 2/2 Running 0 3m39s +---- diff --git a/modules/distr-tracing-tempo-deployment-best-practices.adoc b/modules/distr-tracing-tempo-deployment-best-practices.adoc new file mode 100644 index 000000000000..09ac786720a2 --- /dev/null +++ b/modules/distr-tracing-tempo-deployment-best-practices.adoc @@ -0,0 +1,15 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-configuring.adoc + +:_content-type: CONCEPT +[id="distr-tracing-tempo-deployment-best-practices_{context}"] += Deployment best practices + +* {DTProductName} instance names must be unique. + +* If you have a multitenant implementation and tenants are separated by namespaces, deploy a {TempoName} instance to each tenant namespace. + +** Agent as a daemonset is not supported for multitenant installations or {product-dedicated}. Agent as a sidecar is the only supported configuration for these use cases. + +* If you are installing {DTShortName} as part of {SMProductName}, the {DTShortName} resources must be installed in the same namespace as the `ServiceMeshControlPlane` resource. diff --git a/modules/distr-tracing-tempo-install-about.adoc b/modules/distr-tracing-tempo-install-about.adoc new file mode 100644 index 000000000000..99af58872457 --- /dev/null +++ b/modules/distr-tracing-tempo-install-about.adoc @@ -0,0 +1,15 @@ +// Module included in the following assemblies: +// +// * distr_tracing_tempo/distr-tracing-tempo-installing.adoc + +:_content-type: CONCEPT +[id="distr-tracing-tempo-install-about_{context}"] += About installing the {TempoShortName} + +Installing the {TempoShortName} involves the following steps: + +* Setting up supported object storage. +* Installing the {TempoOperator}. +* Creating a Secret for the object storage credentials. +* Creating a namespace for a TempoStack instance. +* Creating a TempoStack custom resource to deploy at least one TempoStack instance. diff --git a/modules/distr-tracing-tempo-install-cli.adoc b/modules/distr-tracing-tempo-install-cli.adoc new file mode 100644 index 000000000000..1b8ffce0341a --- /dev/null +++ b/modules/distr-tracing-tempo-install-cli.adoc @@ -0,0 +1,176 @@ +// Module included in the following assemblies: +// +//* distr_tracing_tempo/distr-tracing-tempo-installing.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-install-cli_{context}"] += Installing the {TempoShortName} by using the CLI + +You can install the {TempoShortName} on the command line. + +.Prerequisites + +* An active OpenShift CLI (`oc`) session by a xref:https://docs.openshift.com/container-platform/latest/admin_solutions/user_role_mgmt.html#creating-a-cluster-administrator[cluster administrator with the `cluster-admin` role]. See link:https://docs.openshift.com/container-platform/{ocp4-ver}/cli_reference/openshift_cli/getting-started-cli.html[Getting started with the OpenShift CLI]. ++ +TIP: Ensure that your OpenShift CLI (`oc`) version is up to date and matches your {product-title} version. + +* You are using a supported provider of object storage: link:https://www.redhat.com/en/technologies/cloud-computing/openshift-data-foundation[Red Hat OpenShift Data Foundation], link:https://min.io/[MinIO], link:https://aws.amazon.com/s3/[Amazon S3], link:https://azure.microsoft.com/en-us/products/storage/blobs/[Azure Blob Storage], link:https://cloud.google.com/storage/[Google Cloud Storage]. + +. Install the {TempoOperator}: + +.. Create a subscription. ++ +[source,yaml] +---- +$ oc apply -f - << EOF +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: tempo-product + namespace: openshift-operators +spec: + channel: stable + installPlanApproval: Automatic + name: tempo-product + source: redhat-operators + sourceNamespace: openshift-marketplace +EOF +---- + +.. Check the Operator status. ++ +[source,terminal] +---- +$ oc get csv -n openshift-operators +---- + +. Run `oc apply -f` to create a Secret for your object storage bucket. ++ +[TIP] +==== +[source,terminal] +---- +$ oc apply -f .yaml +---- + +[source,terminal] +---- +$ oc apply -f - << EOF + +EOF +---- +==== ++ +-- +include::snippets/distr-tracing-tempo-required-secret-parameters.adoc[] +-- ++ +-- +include::snippets/distr-tracing-tempo-secret-example.adoc[] +-- + +. Use `oc create namespace` or `oc apply` to create a namespace for the *TempoStack* instance that you will create in the next step. ++ +[TIP] +==== +[source,terminal] +---- +$ oc create namespace +---- + +[source,terminal] +---- +$ oc apply -f - << EOF +apiVersion: project.openshift.io/v1 +kind: Project +metadata: + labels: + kubernetes.io/metadata.name: + openshift.io/cluster-monitoring: "true" + name: +EOF +---- +==== + +. Create a *TempoStack* instance in the namespace that you created for the *TempoStack* instance in the previous step. ++ +NOTE: You can create multiple *TempoStack* instances in separate namespaces on the same cluster. ++ +.. Customize the `TempoStack` custom resource (CR): ++ +[source,yaml] +---- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoStack +metadata: + name: sample + namespace: +spec: + storageSize: 1Gi + storage: + secret: + name: <1> + type: <2> + template: + queryFrontend: + jaegerQuery: + enabled: true + ingress: + route: + termination: edge + type: route +---- +<1> The value of `name` in the `metadata` of the Secret. +<2> The accepted values are `azure` for Azure Blob Storage, `gcs` for Google Cloud Storage, and `s3` for Amazon S3 or MinIO or Red Hat OpenShift Data Foundation. ++ +.TempoStack CR for AWS S3 and MinIO storage +==== +[source,yaml] +---- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoStack +metadata: + name: simplest + namespace: tempostacknamespace +spec: + storageSize: 1Gi + storage: + secret: + name: minio-test + type: s3 + resources: + total: + limits: + memory: 2Gi + cpu: 2000m + template: + queryFrontend: + jaegerQuery: + enabled: true + ingress: + route: + termination: edge + type: route +---- +The stack deployed in this example is configured to receive Jaeger Thrift over HTTP and OpenTelemetry Protocol (OTLP), which permits visualizing the data with the Jaeger UI. +==== + +.. Apply the customized CR. ++ +[source,terminal] +---- +$ oc apply -f - << EOF + +EOF +---- + +link:https://github.com/openshift/openshift-docs/pull/62410/files#r1284564654[Waiting for SME input for the following:] + +* Command(s) to verify successful deployment of the TempoStack instance ++ +For example: +** A command that prints output to verify that pods are created on the namespace. +** A command that enables the CLI user to watch for the stack to stabilize. + +* If applicable, a command to open any UI console +* Any other commands? diff --git a/modules/distr-tracing-tempo-install-web-console.adoc b/modules/distr-tracing-tempo-install-web-console.adoc new file mode 100644 index 000000000000..0b29b1327e5f --- /dev/null +++ b/modules/distr-tracing-tempo-install-web-console.adoc @@ -0,0 +1,146 @@ +// Module included in the following assemblies: +// +//* distr_tracing_tempo/distr-tracing-tempo-installing.adoc + +:_content-type: PROCEDURE +[id="distr-tracing-tempo-install-web-console_{context}"] += Installing the {TempoShortName} by using the OpenShift web console + +You can install the {TempoShortName} in the OpenShift web console. + +.Prerequisites + +* You are logged in to the OpenShift xref:https://docs.openshift.com/container-platform/latest/web_console/web-console.html[web console] as a xref:https://docs.openshift.com/container-platform/latest/admin_solutions/user_role_mgmt.html#creating-a-cluster-administrator[cluster administrator with the `cluster-admin` role]. +// https://docs.openshift.com/container-platform/latest/web_console/web-console.html +// https://docs.openshift.com/container-platform/latest/admin_solutions/user_role_mgmt.html#creating-a-cluster-administrator + +* You are using a supported provider of object storage: link:https://www.redhat.com/en/technologies/cloud-computing/openshift-data-foundation[Red Hat OpenShift Data Foundation], link:https://min.io/[MinIO], link:https://aws.amazon.com/s3/[Amazon S3], link:https://azure.microsoft.com/en-us/products/storage/blobs/[Azure Blob Storage], link:https://cloud.google.com/storage/[Google Cloud Storage]. + +.Procedure + +. Install the {TempoOperator}: + +.. In the *Administrator* view of the OpenShift web console, go to *Operators* -> *OperatorHub* and search for `{TempoOperator}`. + +.. Select *{TempoOperator}* that is *OpenShift Operator for Tempo* -> *Install* -> *Install* -> *View Operator*. ++ +[IMPORTANT] +==== +This installs the Operator with the default presets: + +* *Update channel* -> *stable* +* *Installation mode* -> *All namespaces on the cluster* +* *Installed Namespace* -> *openshift-operators* +* *Update approval* -> *Automatic* +==== + +.. In the *Details* tab of the installed Operator page, under *ClusterServiceVersion details*, verify that the installation *Status* is *Succeeded*. + +. Create a Secret for your object storage bucket. ++ +You can do this in the *Administrator* view of the OpenShift web console: go to *Workloads* -> *Secrets* -> *Create* -> *From YAML*. ++ +-- +include::snippets/distr-tracing-tempo-required-secret-parameters.adoc[] +-- ++ +-- +include::snippets/distr-tracing-tempo-secret-example.adoc[] +-- + +. Create a namespace for the *TempoStack* instance that you will create in the next step. ++ +You can do this in the *Administrator* view of the OpenShift web console: go to *Home* -> *Projects* -> *Create Project*. + + +. Create a *TempoStack* instance. ++ +NOTE: You can create multiple *TempoStack* instances in separate namespaces on the same cluster. ++ +You can create a *TempoStack* instance as follows: + +.. Go to *Operators* -> *Installed Operators*. ++ +[TIP] +==== +You can use the *Project:* dropdown list to select the namespace that you created for the *TempoStack* instance in the previous step. +==== + +.. Select *TempoStack* -> *Create TempoStack* -> *YAML view*. + +.. In the *YAML view*, customize the `TempoStack` custom resource (CR): ++ +[source,yaml] +---- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoStack +metadata: + name: sample + namespace: +spec: + storageSize: 1Gi + storage: + secret: + name: <1> + type: <2> + template: + queryFrontend: + jaegerQuery: + enabled: true + ingress: + route: + termination: edge + type: route +---- +<1> The value of `name` in the `metadata` of the Secret. +<2> The accepted values are `azure` for Azure Blob Storage, `gcs` for Google Cloud Storage, and `s3` for Amazon S3 or MinIO or Red Hat OpenShift Data Foundation. ++ +.TempoStack CR for AWS S3 and MinIO storage +==== +[source,yaml] +---- +apiVersion: tempo.grafana.com/v1alpha1 +kind: TempoStack +metadata: + name: simplest + namespace: tempostacknamespace +spec: + storageSize: 1Gi + storage: + secret: + name: minio-test + type: s3 + resources: + total: + limits: + memory: 2Gi + cpu: 2000m + template: + queryFrontend: + jaegerQuery: + enabled: true + ingress: + route: + termination: edge + type: route +---- +The stack deployed in this example is configured to receive Jaeger Thrift over HTTP and OpenTelemetry Protocol (OTLP), which permits visualizing the data with the Jaeger UI. +==== + +.. Select *Create*. + +link:https://github.com/openshift/openshift-docs/pull/62410/files#r1284564005[Waiting for SME input for the following:] + +* Step to verify successful deployment of the TempoStack instance ++ +For example: +** Verify that pods are created on the namespace? +** Verify/watch/check somewhere (where?) in the OpenShift web console UI that the stack has stabilized. +* Is there any Tempo UI console? If so, how does the user open it? +* Any other verification steps? + +.Additional resources + +* link:https://operatorhub.io/[OperatorHub.io] +* link:https://docs.openshift.com/container-platform/latest/operators/admin/olm-adding-operators-to-cluster.html#olm-installing-from-operatorhub-using-web-console_olm-adding-operators-to-a-cluster[Installing from OperatorHub using the web console] +* link:https://docs.openshift.com/container-platform/latest/operators/user/olm-creating-apps-from-installed-operators.html[Creating applications from installed Operators] diff --git a/modules/ossm-config-sampling.adoc b/modules/ossm-config-sampling.adoc index 1b4dbb7bced8..2eae3b01796a 100644 --- a/modules/ossm-config-sampling.adoc +++ b/modules/ossm-config-sampling.adoc @@ -23,7 +23,7 @@ The Envoy proxy sampling rate applies for applications that are available to a S The Jaeger remote sampling rate applies to applications that are external to the Service Mesh, and do not use the Envoy proxy, such as a database. This sampling rate determines how much data the distributed tracing system collects and stores. ifdef::openshift-enterprise[] -For more information, see xref:../../distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc#distr-tracing-config-sampling_deploying-distributed-tracing-platform[Distributed tracing configuration options]. +For more information, see xref:../../distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc#distr-tracing-config-sampling_deploying-distributed-tracing-platform[Distributed tracing configuration options]. endif::[] ==== diff --git a/service_mesh/v2x/ossm-observability.adoc b/service_mesh/v2x/ossm-observability.adoc index 785e4dcd9f1f..438b425d9580 100644 --- a/service_mesh/v2x/ossm-observability.adoc +++ b/service_mesh/v2x/ossm-observability.adoc @@ -27,7 +27,7 @@ include::modules/ossm-config-sampling.adoc[leveloffset=+2] include::modules/ossm-jaeger-accessing-console.adoc[leveloffset=+1] ifdef::openshift-enterprise[] -For more information about configuring Jaeger, see the xref:../../distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc#distr-tracing-deploy-default_deploying-distributed-tracing-platform[distributed tracing documentation]. +For more information about configuring Jaeger, see the xref:../../distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc#distr-tracing-deploy-default_deploying-distributed-tracing-platform[distributed tracing documentation]. endif::[] include::modules/ossm-access-grafana.adoc[leveloffset=+1] diff --git a/service_mesh/v2x/ossm-reference-jaeger.adoc b/service_mesh/v2x/ossm-reference-jaeger.adoc index d49aefa046d7..829579e0132c 100644 --- a/service_mesh/v2x/ossm-reference-jaeger.adoc +++ b/service_mesh/v2x/ossm-reference-jaeger.adoc @@ -37,9 +37,9 @@ include::modules/distr-tracing-config-sampling.adoc[leveloffset=+2] include::modules/distr-tracing-config-storage.adoc[leveloffset=+2] ifdef::openshift-enterprise,openshift-dedicated[] -For more information about configuring Elasticsearch with {product-title}, see xref:../../logging/config/cluster-logging-log-store.adoc[Configuring the log store] or xref:../../distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc[Configuring and deploying distributed tracing]. +For more information about configuring Elasticsearch with {product-title}, see xref:../../logging/config/cluster-logging-log-store.adoc[Configuring the log store] or xref:../../distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc[Configuring and deploying distributed tracing]. -//TO DO For information about connecting to an external Elasticsearch instance, see xref:../../distr_tracing/distr_tracing_install/distr-tracing-deploying-jaeger.adoc#jaeger-config-external-es_jaeger-deploying[Connecting to an existing Elasticsearch instance]. +//TO DO For information about connecting to an external Elasticsearch instance, see xref:../../distr_tracing/distr_tracing_jaeger/distr-tracing-jaeger-configuring.adoc#jaeger-config-external-es_jaeger-deploying[Connecting to an existing Elasticsearch instance]. endif::[] include::modules/distr-tracing-config-query.adoc[leveloffset=+2] diff --git a/snippets/distr-tracing-tempo-required-secret-parameters.adoc b/snippets/distr-tracing-tempo-required-secret-parameters.adoc new file mode 100644 index 000000000000..d84df7c824fb --- /dev/null +++ b/snippets/distr-tracing-tempo-required-secret-parameters.adoc @@ -0,0 +1,72 @@ +// Text snippet included in the following modules: +// +// * distr-tracing-tempo-install-web-console.adoc +// * distr-tracing-tempo-install-cli.adoc + +:_content-type: SNIPPET + +.Required Secret parameters +[cols="25h,~"] +|=== +| Storage provider | Secret parameters + +//source: https://github.com/grafana/tempo-operator/blob/main/docs/tempostack/object_storage.md + +|link:https://access.redhat.com/documentation/en-us/red_hat_openshift_data_foundation/[Red Hat OpenShift Data Foundation] +| +`name: tempostack-dev-odf # example` + +`bucket: # requires an ObjectBucketClaim` + +`+endpoint: https://s3.openshift-storage.svc+` + +`access_key_id: ` + +`access_key_secret: ` + + +|MinIO +| +See link:https://operator.min.io/[MinIO Operator]. + +`name: tempostack-dev-minio # example` + +`bucket: # link:https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html#command-mc.mb[MinIO documentation]` + +`endpoint: ` + +`access_key_id: ` + +`access_key_secret: ` + +|Amazon S3 +| +`name: tempostack-dev-s3 # example` + +`bucket: # link:https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html[Amazon S3 documentation]` + +`endpoint: ` + +`access_key_id: ` + +`access_key_secret: ` + +|Microsoft Azure Blob Storage +| +`name: tempostack-dev-azure # example` + +`container: # link:https://learn.microsoft.com/en-us/rest/api/storageservices/create-container?tabs=azure-ad[Microsoft Azure documentation]` + +`account_name: ` + +`account_key: ` + +|Google Cloud Storage on Google Cloud Platform (GCP) +| +`name: tempostack-dev-gcs # example` + +`bucketname: # requires a link:https://cloud.google.com/storage/docs/creating-buckets[bucket] created in a link:https://cloud.google.com/resource-manager/docs/creating-managing-projects[GCP project]` + +`key.json: # requires a link:https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account[service account] in the bucket's GCP project for GCP authentication` + +|=== diff --git a/snippets/distr-tracing-tempo-secret-example.adoc b/snippets/distr-tracing-tempo-secret-example.adoc new file mode 100644 index 000000000000..0bf7c8829d6e --- /dev/null +++ b/snippets/distr-tracing-tempo-secret-example.adoc @@ -0,0 +1,23 @@ +// Text snippet included in the following modules: +// +// * distr-tracing-tempo-install-web-console.adoc +// * distr-tracing-tempo-install-cli.adoc + +:_content-type: SNIPPET + +.Secret for Amazon S3 and MinIO storage +==== +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: minio-test +stringData: + endpoint: http://minio.minio.svc:9000 + bucket: tempo + access_key_id: tempo + access_key_secret: supersecret +type: Opaque +---- +====