From c48532ef68ddedab527a7eb14ec9006478ed79c9 Mon Sep 17 00:00:00 2001 From: Ishwar Kanse Date: Tue, 16 Sep 2025 08:26:58 +0530 Subject: [PATCH] Add test for span links and other fixes. Assisted by Claude and Cursor IDE --- tests/Dockerfile-cypress | 4 +- tests/PATTERNFLY_COMMANDS_EXAMPLES.md | 6 +- tests/README.md | 2 +- tests/SELECTOR_BEST_PRACTICES.md | 10 +- tests/e2e/dt-plugin-tests.cy.ts | 133 ++++++++++++++++-- tests/fixtures/.chainsaw.yaml | 14 +- .../02-assert.yaml | 4 + .../02-install-otelcol.yaml | 30 +++- .../assert-user-workload-monitoring.yaml | 28 ++++ .../chainsaw-test.yaml | 11 ++ .../check_metrics.sh | 28 ++++ .../install-user-workload-monitoring.yaml | 8 ++ .../kubeadmin-traces-verify.yaml | 4 +- .../tempo-rbac-sa-1-traces-gen.yaml | 6 +- .../tempo-rbac-sa-1-traces-verify.yaml | 8 +- .../tempo-rbac-sa-2-traces-gen.yaml | 6 +- .../multitenancy-rbac/02-install-otelcol.yaml | 30 +++- .../assert-user-workload-monitoring.yaml | 28 ++++ .../multitenancy-rbac/chainsaw-test.yaml | 13 +- .../multitenancy-rbac/check_metrics.sh | 28 ++++ .../install-user-workload-monitoring.yaml | 8 ++ .../kubeadmin-traces-verify.yaml | 4 +- .../tempo-rbac-sa-1-traces-gen.yaml | 6 +- .../tempo-rbac-sa-1-traces-verify.yaml | 8 +- .../tempo-rbac-sa-2-traces-gen.yaml | 6 +- tests/fixtures/monitoring-with-perses.yaml | 9 ++ tests/views/operator-hub-page.ts | 11 +- 27 files changed, 401 insertions(+), 52 deletions(-) create mode 100644 tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/assert-user-workload-monitoring.yaml create mode 100755 tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/check_metrics.sh create mode 100644 tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/install-user-workload-monitoring.yaml create mode 100644 tests/fixtures/chainsaw-tests/multitenancy-rbac/assert-user-workload-monitoring.yaml create mode 100755 tests/fixtures/chainsaw-tests/multitenancy-rbac/check_metrics.sh create mode 100644 tests/fixtures/chainsaw-tests/multitenancy-rbac/install-user-workload-monitoring.yaml create mode 100644 tests/fixtures/monitoring-with-perses.yaml diff --git a/tests/Dockerfile-cypress b/tests/Dockerfile-cypress index b181c20..374bf54 100755 --- a/tests/Dockerfile-cypress +++ b/tests/Dockerfile-cypress @@ -15,6 +15,6 @@ RUN wget -O chainsaw.tar.gz https://github.com/kyverno/chainsaw/releases/downloa && mv chainsaw /usr/local/bin/ \ && rm -rf chainsaw.tar.gz -# Install apache2-utils for htpasswd required by OCP CI -RUN apt update && apt install -y apache2-utils \ +# Install apache2-utils for htpasswd required by OCP CI, jq and curl for metrics validation +RUN apt update && apt install -y apache2-utils jq curl \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/PATTERNFLY_COMMANDS_EXAMPLES.md b/tests/PATTERNFLY_COMMANDS_EXAMPLES.md index 7dca022..415c71e 100644 --- a/tests/PATTERNFLY_COMMANDS_EXAMPLES.md +++ b/tests/PATTERNFLY_COMMANDS_EXAMPLES.md @@ -111,7 +111,7 @@ cy.findByTestId('span-duration-bar').first().click(); // By test ID (multiple sp ```typescript // Single attribute validation -cy.muiTraceAttribute('net.peer.ip', '1.2.3.4'); +cy.muiTraceAttribute('network.peer.address', '1.2.3.4'); cy.muiTraceAttribute('peer.service', 'telemetrygen-client'); cy.muiTraceAttribute('k8s.container.name', 'telemetrygen', true); // Optional attribute @@ -122,7 +122,7 @@ cy.muiTraceAttribute('service.name', (text) => { // Bulk attribute validation (recommended for multiple attributes) cy.muiTraceAttributes({ - 'net.peer.ip': { value: '1.2.3.4' }, + 'network.peer.address': { value: '1.2.3.4' }, 'peer.service': { value: 'telemetrygen-client' }, 'k8s.container.name': { value: 'telemetrygen', @@ -168,7 +168,7 @@ describe('Trace inspection', () => { // Validate trace attributes efficiently cy.muiTraceAttributes({ - 'net.peer.ip': { value: '1.2.3.4' }, + 'network.peer.address': { value: '1.2.3.4' }, 'peer.service': { value: 'telemetrygen-client' }, 'service.name': { value: (text) => text.includes('rbac') diff --git a/tests/README.md b/tests/README.md index 6c959d1..2b81e6b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -138,7 +138,7 @@ cy.muiSpanBar('grpc-service').click() // Attribute validation (bulk) cy.muiTraceAttributes({ 'service.name': { value: 'my-service' }, - 'net.peer.ip': { value: '1.2.3.4' }, + 'network.peer.address': { value: '1.2.3.4' }, 'k8s.namespace': { value: 'test-ns', optional: true } }) ``` diff --git a/tests/SELECTOR_BEST_PRACTICES.md b/tests/SELECTOR_BEST_PRACTICES.md index 895e7bd..3139860 100644 --- a/tests/SELECTOR_BEST_PRACTICES.md +++ b/tests/SELECTOR_BEST_PRACTICES.md @@ -63,9 +63,9 @@ cy.muiSpanBar('http-rbac-2') // Click specific span bar cy.muiFirstSpanBar() // Click first span bar // Trace Attribute Validation -cy.muiTraceAttribute('net.peer.ip', '1.2.3.4') // Single attribute check +cy.muiTraceAttribute('network.peer.address', '1.2.3.4') // Single attribute check cy.muiTraceAttributes({ // Bulk attribute validation - 'net.peer.ip': { value: '1.2.3.4' }, + 'network.peer.address': { value: '1.2.3.4' }, 'service.name': { value: (text) => text.includes('rbac') } }) ``` @@ -219,7 +219,7 @@ cy.findByTestId('span-duration-bar').first().click() // By test ID ### Trace Attribute Validation ```typescript // Old approach (48+ lines of repetitive code): -cy.contains('.MuiTypography-h5', 'net.peer.ip').next('.MuiTypography-body1').should('have.text', '1.2.3.4') +cy.contains('.MuiTypography-h5', 'network.peer.address').next('.MuiTypography-body1').should('have.text', '1.2.3.4') cy.contains('.MuiTypography-h5', 'peer.service').next('.MuiTypography-body1').should('have.text', 'telemetrygen-client') cy.get('body').then(($body) => { if ($body.find('.MuiTypography-h5:contains("k8s.container.name")').length > 0) { @@ -230,7 +230,7 @@ cy.get('body').then(($body) => { // New approach (12 lines, 75% reduction): cy.muiTraceAttributes({ - 'net.peer.ip': { value: '1.2.3.4' }, + 'network.peer.address': { value: '1.2.3.4' }, 'peer.service': { value: 'telemetrygen-client' }, 'k8s.container.name': { value: 'telemetrygen', optional: true }, 'k8s.namespace.name': { @@ -243,7 +243,7 @@ cy.muiTraceAttributes({ }, 'TempoStack') // Or individual attribute checks: -cy.muiTraceAttribute('net.peer.ip', '1.2.3.4') +cy.muiTraceAttribute('network.peer.address', '1.2.3.4') cy.muiTraceAttribute('service.name', (text) => text.includes('rbac'), false, 'TempoStack') ``` diff --git a/tests/e2e/dt-plugin-tests.cy.ts b/tests/e2e/dt-plugin-tests.cy.ts index dd33558..9c1eed6 100644 --- a/tests/e2e/dt-plugin-tests.cy.ts +++ b/tests/e2e/dt-plugin-tests.cy.ts @@ -350,7 +350,7 @@ describe('OpenShift Distributed Tracing UI Plugin tests', () => { .and('have.text', 'Create a TempoMonolithic instance'); }); - it('Test Distributed Tracing UI plugin with Tempo instances and verify traces using user having cluster-admin role', function () { + it('Test Distributed Tracing UI plugin with Tempo instances and verify traces, span links using user having cluster-admin role', function () { cy.log('Create TempoStack and TempoMonolithic instances'); cy.exec( 'chainsaw test --config ./fixtures/.chainsaw.yaml --skip-delete ./fixtures/chainsaw-tests', @@ -358,16 +358,17 @@ describe('OpenShift Distributed Tracing UI Plugin tests', () => { env: { KUBECONFIG: Cypress.env('KUBECONFIG_PATH'), }, - timeout: 1800000, + timeout: 1200000, failOnNonZeroExit: true } ) .then((result) => { expect(result.code).to.eq(0); cy.log(`Chainsaw test ran successfully: ${result.stdout}`); }); - cy.log('Navigate to the observe/traces page'); - cy.visit('/observe/traces'); + cy.log('Navigate to the /observe/traces page'); + cy.visit('/observe/traces'); + cy.url().should('include', '/observe/traces'); cy.log('Assert traces in TempoStack instance.'); cy.pfTypeahead('Select a Tempo instance').click(); cy.pfSelectMenuItem('chainsaw-rbac / simplst').click(); @@ -384,8 +385,93 @@ describe('OpenShift Distributed Tracing UI Plugin tests', () => { cy.muiFirstTraceLink().click(); cy.findByTestId('span-duration-bar').eq(1).click(); cy.muiTraceAttributes({ - 'net.peer.ip': { value: '1.2.3.4' }, - 'peer.service': { value: 'telemetrygen-client' }, + 'network.peer.address': { value: '1.2.3.4' }, + 'peer.service': { value: (text) => ['telemetrygen-server', 'telemetrygen-client'].includes(text) }, + 'k8s.container.name': { value: 'telemetrygen', optional: true }, + 'k8s.namespace.name': { + value: (text) => ['chainsaw-test-rbac-1', 'chainsaw-test-rbac-2', 'chainsaw-mono-rbac-1', 'chainsaw-mono-rbac-2'].includes(text), + optional: true + }, + 'service.name': { + value: (text) => ['http-rbac-1', 'http-rbac-2', 'grpc-rbac-1', 'grpc-rbac-2'].includes(text) + } + }, 'TempoStack'); + cy.log('Click on the Links tab'); + cy.get('button.MuiTab-root').contains('Links').click(); + cy.log('Verify link details are present'); + // Verify first link (index 0) + cy.muiTraceAttribute('link.index', '0', false, 'Links'); + cy.muiTraceAttribute('link.type', 'random', false, 'Links'); + // Verify trace ID and span ID have valid format for first link (they will be different each time) + cy.contains('.MuiTypography-h5', 'trace ID').first().next('.MuiTypography-body1').invoke('text').then((traceId) => { + cy.log(`First link trace ID: ${traceId.trim()}`); + expect(traceId.trim()).to.match(/^[A-F0-9]{32}$/); + }); + cy.contains('.MuiTypography-h5', 'span ID').first().next('.MuiTypography-body1').invoke('text').then((spanId) => { + cy.log(`First link span ID: ${spanId.trim()}`); + expect(spanId.trim()).to.match(/^[A-F0-9]{16}$/); + }); + cy.log('Click on the first trace ID link to navigate to that trace'); + cy.contains('.MuiTypography-h5', 'trace ID').first().next('.MuiTypography-body1').invoke('text').then((traceId) => { + const cleanTraceId = traceId.trim(); + cy.get('a.MuiLink-root[href*="/observe/traces/"]').first().click(); + + cy.log('Verify URL contains the correct trace ID'); + cy.url().should('include', `/observe/traces/${cleanTraceId}`); + cy.log(`✓ Successfully navigated to trace: ${cleanTraceId}`); + }); + cy.log('Verify navigation by checking trace attributes'); + cy.findByTestId('span-duration-bar').eq(1).click(); + cy.muiTraceAttributes({ + 'network.peer.address': { value: '1.2.3.4' }, + 'peer.service': { value: (text) => ['telemetrygen-server', 'telemetrygen-client'].includes(text) }, + 'k8s.container.name': { value: 'telemetrygen', optional: true }, + 'k8s.namespace.name': { + value: (text) => ['chainsaw-test-rbac-1', 'chainsaw-test-rbac-2', 'chainsaw-mono-rbac-1', 'chainsaw-mono-rbac-2'].includes(text), + optional: true + }, + 'service.name': { + value: (text) => ['http-rbac-1', 'http-rbac-2', 'grpc-rbac-1', 'grpc-rbac-2'].includes(text) + } + }, 'TempoStack'); + + cy.log('Rerun the steps and select span ID from links'); + cy.pfBreadcrumb('Traces').click(); + cy.pfCloseButtonIfExists('Close chip group'); + cy.pfTypeahead('Select a Tempo instance').click(); + cy.pfSelectMenuItem('chainsaw-rbac / simplst').click(); + cy.pfTypeahead('Select a tenant').click(); + cy.pfSelectMenuItem('dev').click(); + cy.muiSelect('Select time range').click(); + cy.muiSelectOption('Last 1 hour').click(); + cy.pfMenuToggle('Service Name').click(); + cy.pfMenuToggleByLabel('Multi typeahead checkbox').click(); + cy.pfCheckMenuItem('http-rbac-1'); + cy.pfCheckMenuItem('http-rbac-2'); + cy.pfCheckMenuItem('grpc-rbac-1'); + cy.pfCheckMenuItem('grpc-rbac-2'); + cy.muiFirstTraceLink().click(); + cy.findByTestId('span-duration-bar').eq(1).click(); + cy.log('Click on the Links tab again'); + cy.get('button.MuiTab-root').contains('Links').click(); + cy.log('Click on the first span ID link to navigate to that span'); + cy.contains('.MuiTypography-h5', 'trace ID').first().next('.MuiTypography-body1').invoke('text').then((traceId) => { + const cleanTraceId = traceId.trim(); + cy.contains('.MuiTypography-h5', 'span ID').first().next('.MuiTypography-body1').invoke('text').then((spanId) => { + const cleanSpanId = spanId.trim(); + cy.contains('.MuiTypography-h5', 'span ID').first().next('.MuiTypography-body1').find('a').first().click(); + + cy.log('Verify URL contains the correct trace ID and span ID'); + cy.url().should('include', `/observe/traces/${cleanTraceId}`); + cy.url().should('include', `selectSpan=${cleanSpanId}`); + cy.log(`✓ Successfully navigated to trace: ${cleanTraceId} with selected span: ${cleanSpanId}`); + }); + }); + cy.log('Verify navigation by checking trace attributes'); + cy.findByTestId('span-duration-bar').eq(1).click(); + cy.muiTraceAttributes({ + 'network.peer.address': { value: '1.2.3.4' }, + 'peer.service': { value: (text) => ['telemetrygen-server', 'telemetrygen-client'].includes(text) }, 'k8s.container.name': { value: 'telemetrygen', optional: true }, 'k8s.namespace.name': { value: (text) => ['chainsaw-test-rbac-1', 'chainsaw-test-rbac-2', 'chainsaw-mono-rbac-1', 'chainsaw-mono-rbac-2'].includes(text), @@ -396,6 +482,37 @@ describe('OpenShift Distributed Tracing UI Plugin tests', () => { } }, 'TempoStack'); + cy.log('Rerun the steps and select span links from the Traces page'); + cy.pfBreadcrumb('Traces').click(); + cy.pfCloseButtonIfExists('Close chip group'); + cy.pfTypeahead('Select a Tempo instance').click(); + cy.pfSelectMenuItem('chainsaw-rbac / simplst').click(); + cy.pfTypeahead('Select a tenant').click(); + cy.pfSelectMenuItem('dev').click(); + cy.muiSelect('Select time range').click(); + cy.muiSelectOption('Last 1 hour').click(); + cy.pfMenuToggle('Service Name').click(); + cy.pfMenuToggleByLabel('Multi typeahead checkbox').click(); + cy.pfCheckMenuItem('http-rbac-1'); + cy.pfCheckMenuItem('http-rbac-2'); + cy.pfCheckMenuItem('grpc-rbac-1'); + cy.pfCheckMenuItem('grpc-rbac-2'); + cy.muiFirstTraceLink().click(); + cy.get('[data-testid="LaunchIcon"]').first().click(); + cy.get('a[role="menuitem"]').contains('Open linked span').first().click(); + cy.muiTraceAttributes({ + 'network.peer.address': { value: '1.2.3.4' }, + 'peer.service': { value: (text) => ['telemetrygen-server', 'telemetrygen-client'].includes(text) }, + 'k8s.container.name': { value: 'telemetrygen', optional: true }, + 'k8s.namespace.name': { + value: (text) => ['chainsaw-test-rbac-1', 'chainsaw-test-rbac-2', 'chainsaw-mono-rbac-1', 'chainsaw-mono-rbac-2'].includes(text), + optional: true + }, + 'service.name': { + value: (text) => ['http-rbac-1', 'http-rbac-2', 'grpc-rbac-1', 'grpc-rbac-2'].includes(text) + } + }, 'TempoMonolithic'); + cy.log('Assert traces in TempoMonolithic instance.'); cy.pfBreadcrumb('Traces').click(); cy.pfCloseButtonIfExists('Close chip group'); @@ -414,8 +531,8 @@ describe('OpenShift Distributed Tracing UI Plugin tests', () => { cy.muiFirstTraceLink().click(); cy.findByTestId('span-duration-bar').eq(1).click(); cy.muiTraceAttributes({ - 'net.peer.ip': { value: '1.2.3.4' }, - 'peer.service': { value: 'telemetrygen-client' }, + 'network.peer.address': { value: '1.2.3.4' }, + 'peer.service': { value: (text) => ['telemetrygen-server', 'telemetrygen-client'].includes(text) }, 'k8s.container.name': { value: 'telemetrygen', optional: true }, 'k8s.namespace.name': { value: (text) => ['chainsaw-test-rbac-1', 'chainsaw-test-rbac-2', 'chainsaw-mono-rbac-1', 'chainsaw-mono-rbac-2'].includes(text), diff --git a/tests/fixtures/.chainsaw.yaml b/tests/fixtures/.chainsaw.yaml index c2507c4..e8a8b4c 100644 --- a/tests/fixtures/.chainsaw.yaml +++ b/tests/fixtures/.chainsaw.yaml @@ -4,11 +4,11 @@ kind: Configuration metadata: name: configuration spec: - parallel: 4 + parallel: 2 timeouts: - assert: 6m0s - cleanup: 5m0s - delete: 5m0s - error: 5m0s - apply: 10s - exec: 15s + assert: 15m0s + cleanup: 10m0s + delete: 10m0s + error: 10m0s + apply: 2m0s + exec: 5m0s diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-assert.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-assert.yaml index 4062bd9..252e077 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-assert.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-assert.yaml @@ -24,6 +24,10 @@ spec: port: 4318 protocol: TCP targetPort: 4318 + - name: prometheus + port: 8889 + protocol: TCP + targetPort: 8889 selector: app.kubernetes.io/component: opentelemetry-collector app.kubernetes.io/instance: chainsaw-mmo-rbac.dev diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-install-otelcol.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-install-otelcol.yaml index 0ce5aed..faf644b 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-install-otelcol.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/02-install-otelcol.yaml @@ -39,8 +39,13 @@ apiVersion: opentelemetry.io/v1alpha1 kind: OpenTelemetryCollector metadata: name: dev + namespace: chainsaw-mmo-rbac spec: serviceAccount: otel-collector-deployment + mode: deployment + observability: + metrics: + enableMetrics: true config: | extensions: bearertokenauth: @@ -54,6 +59,19 @@ spec: protocols: http: + connectors: + spanmetrics: + histogram: + explicit: + buckets: [2ms, 4ms, 6ms, 8ms, 10ms, 50ms, 100ms, 200ms, 400ms, 800ms, 1s, 1400ms, 2s, 5s, 10s, 15s] + dimensions: + - name: http.method + default: GET + - name: http.status_code + dimensions_cache_size: 1000 + aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE" + metrics_flush_interval: 5s + processors: k8sattributes: extract: @@ -89,6 +107,11 @@ spec: authenticator: bearertokenauth headers: X-Scope-OrgID: dev # tenantName + prometheus: + add_metric_suffixes: false + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true service: telemetry: @@ -100,11 +123,14 @@ spec: traces/grpc: receivers: [otlp/grpc] processors: [k8sattributes] - exporters: [otlp,debug] + exporters: [otlp,debug,spanmetrics] traces/http: receivers: [otlp/http] processors: [k8sattributes] - exporters: [otlphttp,debug] + exporters: [otlphttp,debug,spanmetrics] + metrics: + receivers: [spanmetrics] + exporters: [prometheus,debug] --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/assert-user-workload-monitoring.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/assert-user-workload-monitoring.yaml new file mode 100644 index 0000000..a94fda1 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/assert-user-workload-monitoring.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-operator + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: thanos-ruler-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/chainsaw-test.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/chainsaw-test.yaml index 6606509..5d2d429 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/chainsaw-test.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/chainsaw-test.yaml @@ -5,6 +5,12 @@ metadata: spec: namespace: chainsaw-mmo-rbac steps: + - name: Enable user workload monitoring + try: + - apply: + file: install-user-workload-monitoring.yaml + - assert: + file: assert-user-workload-monitoring.yaml - name: step-01 try: - apply: @@ -59,3 +65,8 @@ spec: file: kubeadmin-traces-verify.yaml - assert: file: assert-kubeadmin-traces-verify.yaml + - name: Verify metrics + try: + - script: + timeout: 5m + content: ./check_metrics.sh diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/check_metrics.sh b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/check_metrics.sh new file mode 100755 index 0000000..e9140f8 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/check_metrics.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +oc create serviceaccount e2e-test-metrics-reader -n $NAMESPACE +oc adm policy add-cluster-role-to-user cluster-monitoring-view system:serviceaccount:$NAMESPACE:e2e-test-metrics-reader + +TOKEN=$(oc create token e2e-test-metrics-reader -n $NAMESPACE) +THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host') + +#Check metrics used in the prometheus rules created for TempoStack. Refer issue https://issues.redhat.com/browse/TRACING-3399 for skipped metrics. +metrics="traces_span_metrics_duration_bucket traces_span_metrics_duration_count traces_span_metrics_duration_sum traces_span_metrics_calls" + +for metric in $metrics; do +query="$metric" +count=0 + +# Keep fetching and checking the metrics until metrics with value is present. +while [[ $count -eq 0 ]]; do + response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=$query") + count=$(echo "$response" | jq -r '.data.result | length') + + if [[ $count -eq 0 ]]; then + echo "No metric '$metric' with value present. Retrying..." + sleep 5 # Wait for 5 seconds before retrying + else + echo "Metric '$metric' with value is present." + fi + done +done diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/install-user-workload-monitoring.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/install-user-workload-monitoring.yaml new file mode 100644 index 0000000..0d76888 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/install-user-workload-monitoring.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/kubeadmin-traces-verify.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/kubeadmin-traces-verify.yaml index a5d31da..9eed183 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/kubeadmin-traces-verify.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/kubeadmin-traces-verify.yaml @@ -46,7 +46,7 @@ spec: echo "Check for the strings in the trace output - cluster-admin should see complete traces" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -147,7 +147,7 @@ spec: echo "Check for the strings in the trace output - cluster-admin should see complete traces" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml index f8bb17f..780d654 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml @@ -8,13 +8,14 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-mmo-rbac.svc:4317 - --service=grpc-rbac-1 - --otlp-insecure - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-mono-rbac-1" restartPolicy: Never @@ -29,7 +30,7 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-mmo-rbac.svc:4318 @@ -37,6 +38,7 @@ spec: - --otlp-insecure - --service=http-rbac-1 - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-mono-rbac-1" restartPolicy: Never diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml index f8fb3dd..eb714a0 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml @@ -43,7 +43,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -85,7 +85,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -147,7 +147,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -189,7 +189,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" diff --git a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml index 85f5a4c..5c01a19 100644 --- a/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml +++ b/tests/fixtures/chainsaw-tests/monolithic-multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml @@ -8,13 +8,14 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-mmo-rbac.svc:4317 - --service=grpc-rbac-2 - --otlp-insecure - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-mono-rbac-2" restartPolicy: Never @@ -29,7 +30,7 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-mmo-rbac.svc:4318 @@ -37,6 +38,7 @@ spec: - --otlp-insecure - --service=http-rbac-2 - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-mono-rbac-2" restartPolicy: Never diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/02-install-otelcol.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/02-install-otelcol.yaml index 496526d..a1b2555 100644 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/02-install-otelcol.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/02-install-otelcol.yaml @@ -42,6 +42,10 @@ metadata: namespace: chainsaw-rbac spec: serviceAccount: otel-collector-deployment + mode: deployment + observability: + metrics: + enableMetrics: true config: | extensions: bearertokenauth: @@ -55,6 +59,19 @@ spec: protocols: http: + connectors: + spanmetrics: + histogram: + explicit: + buckets: [2ms, 4ms, 6ms, 8ms, 10ms, 50ms, 100ms, 200ms, 400ms, 800ms, 1s, 1400ms, 2s, 5s, 10s, 15s] + dimensions: + - name: http.method + default: GET + - name: http.status_code + dimensions_cache_size: 1000 + aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE" + metrics_flush_interval: 5s + processors: k8sattributes: extract: @@ -92,6 +109,11 @@ spec: authenticator: bearertokenauth headers: X-Scope-OrgID: "dev" + prometheus: + add_metric_suffixes: false + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true service: telemetry: @@ -103,11 +125,15 @@ spec: traces/grpc: receivers: [otlp/grpc] processors: [k8sattributes] - exporters: [otlp,debug] + exporters: [otlp,debug,spanmetrics] traces/http: receivers: [otlp/http] processors: [k8sattributes] - exporters: [otlphttp,debug] + exporters: [otlphttp,debug,spanmetrics] + metrics: + receivers: [spanmetrics] + exporters: [prometheus,debug] + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/assert-user-workload-monitoring.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/assert-user-workload-monitoring.yaml new file mode 100644 index 0000000..a94fda1 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/assert-user-workload-monitoring.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-operator + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: thanos-ruler-user-workload + namespace: openshift-user-workload-monitoring +(status.replicas == spec.replicas): true +spec: + (replicas >= `1`): true diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/chainsaw-test.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/chainsaw-test.yaml index e6ed21d..2c5c955 100755 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/chainsaw-test.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/chainsaw-test.yaml @@ -5,6 +5,12 @@ metadata: spec: namespace: chainsaw-rbac steps: + - name: Enable user workload monitoring + try: + - apply: + file: install-user-workload-monitoring.yaml + - assert: + file: assert-user-workload-monitoring.yaml - name: step-00 try: - apply: @@ -64,4 +70,9 @@ spec: - apply: file: kubeadmin-traces-verify.yaml - assert: - file: assert-kubeadmin-traces-verify.yaml \ No newline at end of file + file: assert-kubeadmin-traces-verify.yaml + - name: Verify metrics + try: + - script: + timeout: 5m + content: ./check_metrics.sh diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/check_metrics.sh b/tests/fixtures/chainsaw-tests/multitenancy-rbac/check_metrics.sh new file mode 100755 index 0000000..e9140f8 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/check_metrics.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +oc create serviceaccount e2e-test-metrics-reader -n $NAMESPACE +oc adm policy add-cluster-role-to-user cluster-monitoring-view system:serviceaccount:$NAMESPACE:e2e-test-metrics-reader + +TOKEN=$(oc create token e2e-test-metrics-reader -n $NAMESPACE) +THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host') + +#Check metrics used in the prometheus rules created for TempoStack. Refer issue https://issues.redhat.com/browse/TRACING-3399 for skipped metrics. +metrics="traces_span_metrics_duration_bucket traces_span_metrics_duration_count traces_span_metrics_duration_sum traces_span_metrics_calls" + +for metric in $metrics; do +query="$metric" +count=0 + +# Keep fetching and checking the metrics until metrics with value is present. +while [[ $count -eq 0 ]]; do + response=$(curl -k -H "Authorization: Bearer $TOKEN" -H "Content-type: application/json" "https://$THANOS_QUERIER_HOST/api/v1/query?query=$query") + count=$(echo "$response" | jq -r '.data.result | length') + + if [[ $count -eq 0 ]]; then + echo "No metric '$metric' with value present. Retrying..." + sleep 5 # Wait for 5 seconds before retrying + else + echo "Metric '$metric' with value is present." + fi + done +done diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/install-user-workload-monitoring.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/install-user-workload-monitoring.yaml new file mode 100644 index 0000000..0d76888 --- /dev/null +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/install-user-workload-monitoring.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/kubeadmin-traces-verify.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/kubeadmin-traces-verify.yaml index 57a62f6..69ba061 100644 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/kubeadmin-traces-verify.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/kubeadmin-traces-verify.yaml @@ -46,7 +46,7 @@ spec: echo "Check for the strings in the trace output - cluster-admin should see complete traces" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -147,7 +147,7 @@ spec: echo "Check for the strings in the trace output - cluster-admin should see complete traces" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml index d64e939..d7eeed9 100644 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-gen.yaml @@ -8,13 +8,14 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-rbac.svc:4317 - --service=grpc-rbac-1 - --otlp-insecure - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-test-rbac-1" restartPolicy: Never @@ -29,7 +30,7 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-rbac.svc:4318 @@ -37,6 +38,7 @@ spec: - --otlp-insecure - --service=http-rbac-1 - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-test-rbac-1" restartPolicy: Never diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml index bc0ed2f..3eaff6d 100644 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-1-traces-verify.yaml @@ -43,7 +43,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -86,7 +86,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -148,7 +148,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" @@ -190,7 +190,7 @@ spec: echo "Check for the strings in the trace output" stringsToSearch=( - "\"key\":\"net.peer.ip\"" + "\"key\":\"network.peer.address\"" "\"stringValue\":\"1.2.3.4\"" "\"key\":\"peer.service\"" "\"stringValue\":\"telemetrygen-client\"" diff --git a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml index 3633d50..5dab8e9 100644 --- a/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml +++ b/tests/fixtures/chainsaw-tests/multitenancy-rbac/tempo-rbac-sa-2-traces-gen.yaml @@ -8,13 +8,14 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-rbac.svc:4317 - --service=grpc-rbac-2 - --otlp-insecure - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-test-rbac-2" restartPolicy: Never @@ -29,7 +30,7 @@ spec: spec: containers: - name: telemetrygen - image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.92.0 + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:v0.137.0 args: - traces - --otlp-endpoint=dev-collector.chainsaw-rbac.svc:4318 @@ -37,6 +38,7 @@ spec: - --otlp-insecure - --service=http-rbac-2 - --traces=2 + - --span-links=2 - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.namespace.name="chainsaw-test-rbac-2" restartPolicy: Never diff --git a/tests/fixtures/monitoring-with-perses.yaml b/tests/fixtures/monitoring-with-perses.yaml new file mode 100644 index 0000000..6f3bb21 --- /dev/null +++ b/tests/fixtures/monitoring-with-perses.yaml @@ -0,0 +1,9 @@ +apiVersion: observability.openshift.io/v1alpha1 +kind: UIPlugin +metadata: + name: monitoring +spec: + type: Monitoring + monitoring: + perses: + enabled: true diff --git a/tests/views/operator-hub-page.ts b/tests/views/operator-hub-page.ts index a8509f5..4df858c 100644 --- a/tests/views/operator-hub-page.ts +++ b/tests/views/operator-hub-page.ts @@ -41,7 +41,16 @@ export const operatorHubPage = { cy.log('Installing operator using recommended namespace'); // Click operator recommended namespace radio button - cy.get('[data-test="Operator recommended Namespace:-radio-input"]').click(); + // Support both old and new OpenShift versions + cy.get('body').then(($body) => { + if ($body.find('#operator-namespace-recommended').length > 0) { + // New method: use ID selector + cy.get('#operator-namespace-recommended').click(); + } else { + // Old method: use data-test attribute + cy.get('[data-test="Operator recommended Namespace:-radio-input"]').click(); + } + }); // Enable monitoring checkbox if it exists (only for recommended namespace) helperfuncs.clickIfExist('[data-test="enable-monitoring"]');