From 846e80f37a6e544e9ad696591b3178f4f0f535a3 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Thu, 6 Aug 2026 16:13:55 -0700 Subject: [PATCH 1/4] Support runtime rule hot-update and DSL debugging for meter-analyzer-config Bring the meter-analyzer-config catalog to parity with otel-rules. Native meter (MeterReportService) rules now load through the same Rules/Rule pipeline otel-rules uses, so they participate in RuleSetMerger, are recorded in StaticRuleRegistry, support the optional layerDefinitions block, and generate source-named expression classes instead of falling back to MalExpr_. MeterProcessService implements MalConverterRegistry and publishes MalStaticBindingHook at boot, so a meter rule can be added, overridden or inactivated at runtime, and attached to a DSL debug session, without restarting the OAP. The internal MeterConfig / MeterConfigs model is removed in favour of the shared one. AnalyzerModuleProvider now declares StorageModule, matching every other Rules.loadRules caller: meter rule loading moved to start() because the merge consults the runtime-rule DB override resolver, which needs a live storage module. Behaviour change: an entry in meterAnalyzerActiveFiles (SW_METER_ANALYZER_ACTIVE_FILES) with no matching rule file now fails OAP startup instead of being silently ignored, matching otel-rules. Also removes the checkstyle.skip exemption from library-pprof-parser - the only module in the repo carrying one - and fixes the two violations it was hiding. --- .github/workflows/skywalking.yaml | 2 + docs/en/changes/changes.md | 6 + .../runtime-rule-hot-update.md | 9 +- .../backend/admin-api/dsl-debugging-mal.md | 10 +- .../setup/backend/admin-api/dsl-debugging.md | 2 +- .../setup/backend/admin-api/runtime-rule.md | 5 +- docs/en/setup/backend/backend-meter.md | 24 + .../analyzer/module/AnalyzerModule.java | 6 +- .../provider/AnalyzerModuleProvider.java | 52 ++- .../provider/meter/config/MeterConfig.java | 42 -- .../provider/meter/config/MeterConfigs.java | 78 ---- .../meter/process/MeterProcessService.java | 78 +++- .../meter/process/MeterProcessor.java | 9 +- ...erProcessServiceConverterRegistryTest.java | 83 ++++ .../meter/process/MeterProcessorTest.java | 9 +- .../mal/MALDebugRecorderFactory.java | 9 +- .../dsl/debugging/mal/MALHolderRegistry.java | 3 +- .../runtimerule/apply/MalFileApplier.java | 5 +- .../runtimerule/engine/RuleEngine.java | 3 +- .../runtimerule/engine/mal/MalRuleEngine.java | 17 +- .../DbOverrideRuntimeRuleResolver.java | 3 +- .../module/RuntimeRuleModuleProvider.java | 16 +- .../runtimerule/rest/RuntimeRuleService.java | 2 +- .../src/main/proto/runtime-rule-cluster.proto | 2 +- .../oap/server/core/classloader/Catalog.java | 1 + .../oap/server/core/dsldebug/RuleKey.java | 6 +- .../management/runtimerule/RuntimeRule.java | 2 +- .../library-pprof-parser/pom.xml | 4 - .../library/pprof/type/FrameTreeBuilder.java | 3 +- .../meter/bundled/batch-meter.yaml | 23 + .../runtime-rule/meter/docker-compose.yml | 84 ++++ test/e2e-v2/cases/runtime-rule/meter/e2e.yaml | 69 +++ .../cases/runtime-rule/meter/expected/ok.txt | 1 + .../meter/meter-runtime-rule-flow.sh | 416 ++++++++++++++++++ .../meter/seed-rules/meter-v1.yaml | 25 ++ .../meter/seed-rules/meter-v2.yaml | 27 ++ 36 files changed, 945 insertions(+), 191 deletions(-) delete mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java delete mode 100644 oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java create mode 100644 oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessServiceConverterRegistryTest.java create mode 100644 test/e2e-v2/cases/runtime-rule/meter/bundled/batch-meter.yaml create mode 100644 test/e2e-v2/cases/runtime-rule/meter/docker-compose.yml create mode 100644 test/e2e-v2/cases/runtime-rule/meter/e2e.yaml create mode 100644 test/e2e-v2/cases/runtime-rule/meter/expected/ok.txt create mode 100755 test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh create mode 100644 test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v1.yaml create mode 100644 test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v2.yaml diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 2b9b8c3c57ca..6555d9f245a8 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -426,6 +426,8 @@ jobs: - name: Runtime Rule Cluster (kind) config: test/e2e-v2/cases/runtime-rule/cluster/e2e.yaml runs-on: ubuntu-24.04 + - name: Runtime Rule Meter Hot-Update + config: test/e2e-v2/cases/runtime-rule/meter/e2e.yaml - name: DSL Debug API — MAL config: test/e2e-v2/cases/dsl-debugging/mal/e2e.yaml - name: DSL Debug API — OAL diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 5714d37c68da..0c51ebd78615 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -255,6 +255,11 @@ admin-host only" entry above for the public REST retirement. #### OAP Server +* Support runtime rule hot-update and DSL debugging for the `meter-analyzer-config` catalog, bringing native meter (`MeterReportService`) rules to parity with `otel-rules`. + - Meter rules now load through the same `Rules`/`Rule` pipeline `otel-rules` uses, so they participate in `RuleSetMerger`, are recorded in `StaticRuleRegistry`, support the optional `layerDefinitions` block, and generate source-named expression classes instead of falling back to `MalExpr_`. + - `MeterProcessService` now implements `MalConverterRegistry` and publishes debug holders at boot, so a meter rule can be added / overridden / inactivated at runtime, and attached to a DSL debug session, without restarting the OAP. + - The internal `MeterConfig` / `MeterConfigs` model is removed in favour of the shared one. + - **Behaviour change:** an entry in `meterAnalyzerActiveFiles` (`SW_METER_ANALYZER_ACTIVE_FILES`) with no matching rule file now fails OAP startup instead of being silently ignored, matching how `otel-rules` has always behaved. * Support Elasticsearch 9.x as storage. * Add Node.js runtime metrics via the Node.js agent **`MeterReportService`** pipeline (`meter_instance_nodejs_*`, 1s collect/report). OAP analyzes raw meters through `nodejs-runtime.yaml`. Node.js E2E asserts six `meter_instance_nodejs_*` metrics (`test/e2e-v2/cases/nodejs/e2e.yaml`). * Add PHP runtime PHM meter analyzer (`php-runtime.yaml`) for SkyWalking PHP agent process @@ -353,6 +358,7 @@ * Support trace V1 view in trace single page. #### Documentation +* Document the `meter-analyzer-config` catalog in the runtime-rule hot-update and DSL-debugging references, and add the optional `layerDefinitions` block, the active-files startup-failure behaviour, and a hot-update / debugging section to the meter setup doc. * Update LAL documentation with `sourceAttribute()` function and `layer: auto` mode. * Add Airflow monitoring setup documentation (SWIP-7). * Add iOS app monitoring setup documentation. diff --git a/docs/en/concepts-and-designs/runtime-rule-hot-update.md b/docs/en/concepts-and-designs/runtime-rule-hot-update.md index 48ad38202433..a63c49d7a473 100644 --- a/docs/en/concepts-and-designs/runtime-rule-hot-update.md +++ b/docs/en/concepts-and-designs/runtime-rule-hot-update.md @@ -14,8 +14,9 @@ something goes wrong. The HTTP surface is documented separately in live in management storage (the same persistence layer used for UI templates, UI menus, and other cluster-wide operator state). - **Catalog** — the rule group named in the API, currently `otel-rules`, `log-mal-rules`, - `telegraf-rules`, or `lal`. It mirrors the on-disk directory layout so a rule's - `(catalog, name)` identity is portable between disk and the runtime-rule entry store. + `telegraf-rules`, `meter-analyzer-config`, or `lal`. It mirrors the on-disk directory + layout so a rule's `(catalog, name)` identity is portable between disk and the + runtime-rule entry store. - **Main** — the single OAP node designated to run the on-demand workflow. Every node can compute it locally from the sorted cluster peer list; no election. - **Peer** — every node other than the main. @@ -26,8 +27,8 @@ something goes wrong. The HTTP surface is documented separately in ## Scope: MAL and LAL, not OAL Runtime hot-update covers only the **MAL** (`otel-rules`, `log-mal-rules`, -`telegraf-rules`) and **LAL** (`lal`) catalogs. OAL rules are deliberately out of -scope. Three reasons, in order of weight: +`telegraf-rules`, `meter-analyzer-config`) and **LAL** (`lal`) catalogs. OAL rules are +deliberately out of scope. Three reasons, in order of weight: 1. **OAL targets SkyWalking-native traffic sources; MAL and LAL target third-party data.** OAL rules derive metrics from the fixed set of sources the platform diff --git a/docs/en/setup/backend/admin-api/dsl-debugging-mal.md b/docs/en/setup/backend/admin-api/dsl-debugging-mal.md index d31a7930efb1..7c096a43b487 100644 --- a/docs/en/setup/backend/admin-api/dsl-debugging-mal.md +++ b/docs/en/setup/backend/admin-api/dsl-debugging-mal.md @@ -84,11 +84,11 @@ SW_DSL_DEBUGGING_INJECTION_ENABLED=false # default is true; set false to disab A session targets one MAL metric rule. The key tuple is `(catalog, name, ruleName)`: -| Field | Source | -|------------|----------------------------------------------------------------------------------------------| -| `catalog` | One of `otel-rules`, `log-mal-rules`, `telegraf-rules` — the directory the rule file lives in | -| `name` | The rule **file** name, without `.yaml` | -| `ruleName` | The full metric name (`metricPrefix` + `_` + per-rule `name`) | +| Field | Source | +|------------|-----------------------------------------------------------------------------------------------------------------------| +| `catalog` | One of `otel-rules`, `log-mal-rules`, `telegraf-rules`, `meter-analyzer-config` — the directory the rule file lives in | +| `name` | The rule **file** name, without `.yaml` | +| `ruleName` | The full metric name (`metricPrefix` + `_` + per-rule `name`) | Example — the shipped `otel-rules/vm.yaml` declares a metric prefix `vm` and per-rule name `cpu_total_percentage`. The full metric name is diff --git a/docs/en/setup/backend/admin-api/dsl-debugging.md b/docs/en/setup/backend/admin-api/dsl-debugging.md index 4e3a061f0bd4..1fee1e5a9634 100644 --- a/docs/en/setup/backend/admin-api/dsl-debugging.md +++ b/docs/en/setup/backend/admin-api/dsl-debugging.md @@ -10,7 +10,7 @@ Each DSL has its own probe surface, payload shape, and rule-key conventions — pick the page that matches the rule you're debugging: -- **[MAL](dsl-debugging-mal.md)** — meter rules under `otel-rules`, `log-mal-rules`, `telegraf-rules`. Each captured record is one `SampleFamily` walking through the rule end-to-end (filter → chain ops → meterEmit). Sample payloads carry the complete `SampleFamily` (every sample's name + labels + value + timestamp). +- **[MAL](dsl-debugging-mal.md)** — meter rules under `otel-rules`, `log-mal-rules`, `telegraf-rules`, `meter-analyzer-config`. Each captured record is one `SampleFamily` walking through the rule end-to-end (filter → chain ops → meterEmit). Sample payloads carry the complete `SampleFamily` (every sample's name + labels + value + timestamp). - **[OAL](dsl-debugging-oal.md)** — per-metric dispatcher capture under `catalog=oal`. Each record is one `ISource` walking through (source entry → filter clauses → aggregation function → emit). Source samples carry the rich `ServiceRelation`-style payload (sourceServiceName, destServiceName, layers, latency, status, detectPoint, ...). - **[LAL](dsl-debugging-lal.md)** — log analysis under `catalog=lal`. Each record is one log walking through (text → parser → extractor statements → sink). `granularity=statement` emits one sample per extractor statement; `granularity=block` (default) collapses extractor into one sample. diff --git a/docs/en/setup/backend/admin-api/runtime-rule.md b/docs/en/setup/backend/admin-api/runtime-rule.md index b0a8850ded8e..3b261187dd3c 100644 --- a/docs/en/setup/backend/admin-api/runtime-rule.md +++ b/docs/en/setup/backend/admin-api/runtime-rule.md @@ -143,8 +143,8 @@ Implicit catalog in the path — useful when scripting against a single catalog: - `/runtime/mal/log/{addOrUpdate,inactivate,delete}` → `catalog=log-mal-rules` - `/runtime/lal/{addOrUpdate,inactivate,delete}` → `catalog=lal` -`telegraf-rules` is supported by the canonical `/runtime/rule/...` routes; it does not -currently have a shortcut route. +`telegraf-rules` and `meter-analyzer-config` are supported by the canonical +`/runtime/rule/...` routes; they do not currently have shortcut routes. ### Valid catalogs + names @@ -153,6 +153,7 @@ currently have a shortcut route. | `otel-rules` | OTEL MAL rule YAML files | | `log-mal-rules` | Log-derived MAL rule YAML files | | `telegraf-rules` | Telegraf MAL rule YAML files | +| `meter-analyzer-config` | Native meter protocol MAL rule YAML files | | `lal` | LAL rule YAML files | Rule `name` mirrors the static filesystem layout — a relative path under the catalog root diff --git a/docs/en/setup/backend/backend-meter.md b/docs/en/setup/backend/backend-meter.md index e7c73e2acd4e..44c4ad4d33f8 100644 --- a/docs/en/setup/backend/backend-meter.md +++ b/docs/en/setup/backend/backend-meter.md @@ -77,6 +77,10 @@ section `agent-analyzer` in `application.yml` of skywalking backend. meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:your-custom-meter-conf-without-ext-name} # The multiple files should be separated by "," ``` +Every entry listed in `meterAnalyzerActiveFiles` must have a matching rule file under +`$CLASSPATH/meter-analyzer-config`. An entry with no matching file fails the OAP startup; in previous +releases such an entry was silently ignored. + Meter-analyzer-config file is written in YAML format, defined by the scheme described below. Brackets indicate that a parameter is optional. @@ -91,6 +95,17 @@ expPrefix: expSuffix: # insert metricPrefix into metric name: _ metricPrefix: +# Optional. Declares custom layers inline, registered before the rules in this file compile, +# so that a layer referenced by `expSuffix`/`exp` needs no change in the OAP source. +layerDefinitions: + # Layer name, must match [A-Z][A-Z0-9_]*. + - name: + # Ordinal, unique across all layers and persisted in storage. The ordinal space is + # partitioned by tier: 0-9999 built-in, 10000-99999 boot-time external (this file when + # shipped on disk), 100000+ runtime rules pushed through the hot-update API. + ordinal: + # true = services in this layer are agent-installed (default), false = conjectured. + normal: # Metrics rule allow you to recompute queries. metricsRules: # The name of rule, which combinates with a prefix '_' as the index/table name in storage. @@ -110,3 +125,12 @@ client-side APIs to run these functions. The reasons are as follows: 1. The OAP has to set up caches to calculate the values. 1. Once the agent reconnects to another OAP instance, the time windows of rate calculation break. This leads to inaccurate results. + +## Runtime hot-update and debugging + +Meter-analyzer-config rules are loaded through the same rule pipeline as `otel-rules`, so they can be +added, overridden, inactivated, and reverted to the bundled content at runtime, without restarting the +OAP. They can also be attached to a sampling debug session to inspect the intermediate result of every +stage of a MAL expression. The catalog name to use in both APIs is `meter-analyzer-config`. +See [Runtime Rule Hot-Update API](admin-api/runtime-rule.md) +and [DSL Debug API — MAL](admin-api/dsl-debugging-mal.md). diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java index a406b4b2fb21..340bfd2fcc69 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/module/AnalyzerModule.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.analyzer.module; +import org.apache.skywalking.oap.meter.analyzer.v2.MalConverterRegistry; import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.ISegmentParserService; import org.apache.skywalking.oap.server.library.module.ModuleDefine; @@ -33,7 +34,10 @@ public AnalyzerModule() { public Class[] services() { return new Class[] { ISegmentParserService.class, - IMeterProcessService.class + IMeterProcessService.class, + // Lets the runtime-rule plugin hot-swap a single meter-analyzer-config rule. + // Resolved by MalRuleEngine via moduleManager.find("agent-analyzer"). + MalConverterRegistry.class }; } } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java index 0c19854bb0fd..086f19757abb 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java @@ -18,11 +18,14 @@ package org.apache.skywalking.oap.server.analyzer.provider; +import java.io.IOException; +import java.util.Collections; import java.util.List; import lombok.Getter; +import org.apache.skywalking.oap.meter.analyzer.v2.MalConverterRegistry; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rule; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rules; import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule; -import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; -import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; import org.apache.skywalking.oap.server.analyzer.provider.meter.process.IMeterProcessService; import org.apache.skywalking.oap.server.analyzer.provider.meter.process.MeterProcessService; import org.apache.skywalking.oap.server.analyzer.provider.trace.CacheReadLatencyThresholdsAndWatcher; @@ -43,10 +46,12 @@ import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.oal.rt.CoreOALDefine; import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService; +import org.apache.skywalking.oap.server.core.storage.StorageModule; import org.apache.skywalking.oap.server.library.module.ModuleDefine; import org.apache.skywalking.oap.server.library.module.ModuleProvider; import org.apache.skywalking.oap.server.library.module.ModuleStartException; import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; public class AnalyzerModuleProvider extends ModuleProvider { @@ -63,7 +68,6 @@ public class AnalyzerModuleProvider extends ModuleProvider { @Getter private TraceSamplingPolicyWatcher traceSamplingPolicyWatcher; - private List meterConfigs; @Getter private MeterProcessService processService; @@ -109,10 +113,11 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException { segmentParserService = new SegmentParserServiceImpl(getManager(), moduleConfig); this.registerServiceImplementation(ISegmentParserService.class, segmentParserService); - meterConfigs = MeterConfigs.loadConfig( - moduleConfig.getConfigPath(), moduleConfig.meterAnalyzerActiveFileNames()); processService = new MeterProcessService(getManager()); this.registerServiceImplementation(IMeterProcessService.class, processService); + // Same instance under both contracts: IMeterProcessService is the ingest-side view + // (receiver + Kafka fetcher), MalConverterRegistry is the runtime-rule hot-update view. + this.registerServiceImplementation(MalConverterRegistry.class, processService); } @Override @@ -135,7 +140,31 @@ public void start() throws ModuleStartException { segmentParserService.setListenerManager(listenerManager()); - processService.start(meterConfigs); + processService.start(loadMeterRules()); + } + + /** + * Load the active {@code meter-analyzer-config} rule files through the same + * {@link Rules} loader the otel catalog uses, so meter rules participate in + * {@code RuleSetMerger} (runtime-rule DB overrides win over disk) and land in + * {@code StaticRuleRegistry} (which is what makes {@code /runtime/rule/list}, + * {@code /inactivate} and revert-to-bundled see a shipped meter rule at all). + * + *

Runs in {@code start()} rather than {@code prepare()} — the merge consults the + * runtime-rule override resolver, which needs a live storage module. This mirrors + * {@code OpenTelemetryMetricRequestProcessor.start()}. + */ + private List loadMeterRules() throws ModuleStartException { + final List activeFiles = moduleConfig.meterAnalyzerActiveFileNames(); + // Null when meterAnalyzerActiveFiles is unset; Rules.loadRules would NPE on it. + if (CollectionUtils.isEmpty(activeFiles)) { + return Collections.emptyList(); + } + try { + return Rules.loadRules(moduleConfig.getConfigPath(), activeFiles); + } catch (IOException e) { + throw new ModuleStartException("Load meter analyzer configs failed", e); + } } @Override @@ -148,7 +177,16 @@ public String[] requiredModules() { return new String[] { TelemetryModule.NAME, CoreModule.NAME, - ConfigurationModule.NAME + ConfigurationModule.NAME, + // StorageModule is declared so Storage.start() (which registers the + // runtime_rule management table) runs before this provider's start(), + // guaranteeing the RuntimeRuleOverrideResolver's DB-backed resolver can load + // while loadMeterRules() registers the static meter rules. Without this dep + // the module-system sort places agent-analyzer ahead of Storage, the resolver + // silently no-ops at boot, and an operator's meter-rule override / inactivate + // would not take effect until the reconciler's next tick. Same rationale the + // otel / telegraf / envoy / log-analyzer providers carry. + StorageModule.NAME }; } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java deleted file mode 100644 index 8e13ab8c2c43..000000000000 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.analyzer.provider.meter.config; - -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.skywalking.oap.meter.analyzer.v2.MetricRuleConfig; - -import java.util.List; - -@Data -@NoArgsConstructor -public class MeterConfig implements MetricRuleConfig { - private String metricPrefix; - private String expSuffix; - private String expPrefix; - private String filter; - private List metricsRules; - - @Data - @NoArgsConstructor - public static class Rule implements RuleConfig { - private String name; - private String exp; - } -} diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java deleted file mode 100644 index b1eff96fbcba..000000000000 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.analyzer.provider.meter.config; - -import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.library.module.ModuleStartException; -import org.apache.skywalking.oap.server.library.util.CollectionUtils; -import org.apache.skywalking.oap.server.library.util.ResourceUtils; -import org.yaml.snakeyaml.Yaml; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.Reader; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * Meter config loader. - */ -@Slf4j -public class MeterConfigs { - - /** - * Load all configs from path - */ - public static List loadConfig(String path, List fileNames) throws ModuleStartException { - if (CollectionUtils.isEmpty(fileNames)) { - return Collections.emptyList(); - } - - File[] configs; - try { - configs = ResourceUtils.getPathFiles(path); - } catch (FileNotFoundException e) { - throw new ModuleStartException("Load meter configs failed", e); - } - - return Arrays.stream(configs) - .map(f -> { - String fileName = f.getName(); - int dotIndex = fileName.lastIndexOf('.'); - fileName = (dotIndex == -1) ? fileName : fileName.substring(0, dotIndex); - if (!fileNames.contains(fileName)) { - return null; - } - try (Reader r = new FileReader(f)) { - return new Yaml().loadAs(r, MeterConfig.class); - } catch (IOException e) { - log.warn("Reading file {} failed", f, e); - } - return null; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - -} diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java index 94d0909e5ba3..eb77d812d15a 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessService.java @@ -18,30 +18,65 @@ package org.apache.skywalking.oap.server.analyzer.provider.meter.process; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.oap.meter.analyzer.v2.MalConverterRegistry; import org.apache.skywalking.oap.meter.analyzer.v2.MetricConvert; -import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; +import org.apache.skywalking.oap.meter.analyzer.v2.dsldebug.MalStaticBindingHook; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rule; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem; import org.apache.skywalking.oap.server.library.module.ModuleManager; -import java.util.List; -import java.util.stream.Collectors; - /** * Management all of the meter builders. + * + *

Doubles as the {@link MalConverterRegistry} for the {@code meter-analyzer-config} catalog, + * so the runtime-rule plugin can hot-swap a single meter rule without rebuilding the whole list. + * The registry backs BOTH native-meter ingest paths — the gRPC meter receiver and the Kafka meter + * fetcher — because both resolve their converters through this one service via + * {@link IMeterProcessService}. + * + *

Thread-safety mirrors the otel receiver's registry: a volatile map replaced wholesale under + * {@link #convertersWriteLock}, so ingest threads iterating {@link #converts()} always observe a + * complete pre- or post-swap snapshot, never a torn intermediate. {@link MeterProcessor} re-reads + * {@link #converts()} on every batch, so a hot-added rule takes effect on the next batch with no + * OAP restart. */ -public class MeterProcessService implements IMeterProcessService { +public class MeterProcessService implements IMeterProcessService, MalConverterRegistry { + + /** Catalog wire-name; also the key namespace shared with runtime-rule's converter pushes. */ + private static final String CATALOG = "meter-analyzer-config"; private final ModuleManager manager; - private List metricConverts; + /** + * Copy-on-write snapshot keyed by {@code ":"}. {@link LinkedHashMap} + * preserves rule order so dispatch order stays deterministic across restarts. + */ + private volatile Map converters = Collections.emptyMap(); + private final Object convertersWriteLock = new Object(); public MeterProcessService(ModuleManager manager) { this.manager = manager; } - public void start(List configs) { + /** + * Compile and install every boot-time meter rule. Uses the same install path as a runtime + * hot-update ({@link #addOrReplaceConverter}) so there is exactly one installation route, + * and publishes each rule's per-metric debug holders so a dsl-debugging session can bind to + * a bundled meter rule the same way it binds to an otel one. + */ + public void start(List rules) { final MeterSystem meterSystem = manager.find(CoreModule.NAME).provider().getService(MeterSystem.class); - this.metricConverts = configs.stream().map(c -> new MetricConvert(c, meterSystem)).collect(Collectors.toList()); + for (final Rule rule : rules) { + final MetricConvert convert = new MetricConvert(rule, meterSystem); + addOrReplaceConverter(CATALOG + ":" + rule.getName(), convert); + // No-op unless the dsl-debugging module installed a sink. + MalStaticBindingHook.publish(CATALOG, rule.getName(), convert); + } } /** @@ -53,10 +88,31 @@ public MeterProcessor createProcessor() { } /** - * Getting all converters. + * Getting all converters. Never null; empty before {@link #start} runs. */ - public List converts() { - return metricConverts; + public Collection converts() { + return converters.values(); + } + + @Override + public void addOrReplaceConverter(final String key, final MetricConvert convert) { + synchronized (convertersWriteLock) { + final Map copy = new LinkedHashMap<>(converters); + copy.put(key, convert); + converters = Collections.unmodifiableMap(copy); + } + } + + @Override + public void removeConverter(final String key) { + synchronized (convertersWriteLock) { + if (!converters.containsKey(key)) { + return; + } + final Map copy = new LinkedHashMap<>(converters); + copy.remove(key); + converters = Collections.unmodifiableMap(copy); + } } } diff --git a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java index 4635c5084ed5..80d5268c5e87 100644 --- a/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java +++ b/oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessor.java @@ -30,9 +30,9 @@ import org.apache.skywalking.oap.meter.analyzer.v2.MetricConvert; import org.apache.skywalking.oap.meter.analyzer.v2.dsl.Sample; import org.apache.skywalking.oap.meter.analyzer.v2.dsl.SampleFamilyBuilder; -import org.apache.skywalking.oap.server.library.util.CollectionUtils; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -125,9 +125,10 @@ public void process() { return; } - // Get all meter builders. - final List converts = processService.converts(); - if (CollectionUtils.isEmpty(converts)) { + // Get all meter builders. Re-read per batch so a runtime-rule hot-update lands on the + // next batch without an OAP restart. + final Collection converts = processService.converts(); + if (converts.isEmpty()) { return; } diff --git a/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessServiceConverterRegistryTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessServiceConverterRegistryTest.java new file mode 100644 index 000000000000..52454531d2f0 --- /dev/null +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessServiceConverterRegistryTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.oap.server.analyzer.provider.meter.process; + +import org.apache.skywalking.oap.meter.analyzer.v2.MetricConvert; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Contract test for the {@link org.apache.skywalking.oap.meter.analyzer.v2.MalConverterRegistry} + * half of {@link MeterProcessService} — the surface runtime-rule drives when it hot-updates a + * {@code meter-analyzer-config} rule. Mirrors the otel receiver's equivalent test so both MAL + * registries are held to the same contract. + * + *

Boot-time compilation is covered by {@code MeterProcessorTest}; here we only exercise the + * mutation surface, which is what an operator's {@code /addOrUpdate} and {@code /inactivate} + * reach. + */ +class MeterProcessServiceConverterRegistryTest { + + private static final String KEY = "meter-analyzer-config:java-agent"; + + @Test + void addOrReplaceThenRemoveRoundTrips() { + final MeterProcessService service = newService(); + final MetricConvert first = mock(MetricConvert.class); + final MetricConvert second = mock(MetricConvert.class); + + service.addOrReplaceConverter(KEY, first); + assertEquals(1, service.converts().size()); + assertTrue(service.converts().contains(first)); + + // Re-binding the same key is the FILTER_ONLY hot path — it must replace in place + // rather than accumulate a second entry for the same rule file. + service.addOrReplaceConverter(KEY, second); + assertEquals(1, service.converts().size()); + assertTrue(service.converts().contains(second)); + + service.removeConverter(KEY); + assertTrue(service.converts().isEmpty()); + } + + @Test + void removeConverterOnAbsentKeyIsIdempotent() { + // /delete or /inactivate against a rule this node already tore down must not raise — + // a missing key means "already converged", not a failure. + final MeterProcessService service = newService(); + + assertDoesNotThrow(() -> service.removeConverter("meter-analyzer-config:nonexistent")); + assertTrue(service.converts().isEmpty()); + } + + @Test + void convertsIsEmptyBeforeStart() { + // MeterProcessor reads converts() on every batch and must tolerate the pre-start + // window (ingest can arrive before the boot-time rules finish compiling). + assertTrue(newService().converts().isEmpty()); + } + + private static MeterProcessService newService() { + return new MeterProcessService(mock(ModuleManager.class)); + } +} diff --git a/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java index e0248be718e4..f62314494307 100644 --- a/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java +++ b/oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessorTest.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.analyzer.provider.meter.process; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -25,8 +26,8 @@ import org.apache.skywalking.apm.network.language.agent.v3.MeterBucketValue; import org.apache.skywalking.apm.network.language.agent.v3.MeterData; import org.apache.skywalking.apm.network.language.agent.v3.MeterHistogram; -import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig; -import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rule; +import org.apache.skywalking.oap.meter.analyzer.v2.prometheus.rule.Rules; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.StreamDefinition; import org.apache.skywalking.oap.server.core.analysis.meter.MeterEntity; @@ -77,7 +78,7 @@ public static void init() { } @BeforeEach - public void setup() throws StorageException, ModuleStartException { + public void setup() throws StorageException, ModuleStartException, IOException { meterSystem = spy(new MeterSystem(moduleManager)); when(moduleManager.find(anyString())).thenReturn(mock(ModuleProviderHolder.class)); when(moduleManager.find(CoreModule.NAME).provider()).thenReturn(mock(ModuleServiceHolder.class)); @@ -92,7 +93,7 @@ public void setup() throws StorageException, ModuleStartException { // the shape-mismatch gate at the installer level can surface to stream registration. doNothing().when(mockProcessor).create(any(), (StreamDefinition) any(), any(), any()); final MeterProcessService processService = new MeterProcessService(moduleManager); - List config = MeterConfigs.loadConfig("meter-analyzer-config", Arrays.asList("config")); + List config = Rules.loadRules("meter-analyzer-config", Arrays.asList("config")); processService.start(config); processor = new MeterProcessor(processService); } diff --git a/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALDebugRecorderFactory.java b/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALDebugRecorderFactory.java index c2bb4e7df346..0e2f291e7cf3 100644 --- a/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALDebugRecorderFactory.java +++ b/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALDebugRecorderFactory.java @@ -28,9 +28,9 @@ /** * Builds {@link MALDebugRecorderImpl} for any {@link RuleKey} whose catalog * is a MAL catalog ({@code OTEL_RULES}, {@code LOG_MAL_RULES}, - * {@code TELEGRAF_RULES}). LAL and OAL register their own factories in - * later phases so the registry's dispatch stays first-match-wins without - * a per-DSL conditional. + * {@code TELEGRAF_RULES}, {@code METER_ANALYZER_CONFIG}). LAL and OAL + * register their own factories in later phases so the registry's dispatch + * stays first-match-wins without a per-DSL conditional. */ public final class MALDebugRecorderFactory implements DebugRecorderFactory { @@ -39,7 +39,8 @@ public boolean serves(final RuleKey key) { final Catalog c = key.getCatalog(); return c == Catalog.OTEL_RULES || c == Catalog.LOG_MAL_RULES - || c == Catalog.TELEGRAF_RULES; + || c == Catalog.TELEGRAF_RULES + || c == Catalog.METER_ANALYZER_CONFIG; } @Override diff --git a/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALHolderRegistry.java b/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALHolderRegistry.java index 46f6b91ad259..75219123d226 100644 --- a/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALHolderRegistry.java +++ b/oap-server/server-admin/dsl-debugging/src/main/java/org/apache/skywalking/oap/server/admin/dsl/debugging/mal/MALHolderRegistry.java @@ -90,7 +90,8 @@ public boolean serves(final RuleKey key) { final Catalog c = key.getCatalog(); return c == Catalog.OTEL_RULES || c == Catalog.LOG_MAL_RULES - || c == Catalog.TELEGRAF_RULES; + || c == Catalog.TELEGRAF_RULES + || c == Catalog.METER_ANALYZER_CONFIG; } @Override diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/MalFileApplier.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/MalFileApplier.java index 17cf7a5560ef..1761e913b628 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/MalFileApplier.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/apply/MalFileApplier.java @@ -49,8 +49,9 @@ import org.yaml.snakeyaml.Yaml; /** - * Turns a runtime-rule MAL file (one of the {@code otel-rules} / {@code log-mal-rules} - * catalogs) into a live {@link MetricConvert} on this OAP node. + * Turns a runtime-rule MAL file (one of the {@code otel-rules} / {@code log-mal-rules} / + * {@code telegraf-rules} / {@code meter-analyzer-config} catalogs) into a live + * {@link MetricConvert} on this OAP node. * *

This is the MAL half of the apply pipeline: parse the stored YAML, construct a * {@link MetricConvert}, and let the existing meter-analyzer path register each declared diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/RuleEngine.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/RuleEngine.java index 3056694f175c..b6000636c2ec 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/RuleEngine.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/RuleEngine.java @@ -182,7 +182,8 @@ public interface RuleEngine { /** * Catalogs this engine handles, e.g. {@code {"otel-rules", "log-mal-rules", - * "telegraf-rules"}} for the MAL engine, {@code {"lal"}} for the LAL engine. + * "telegraf-rules", "meter-analyzer-config"}} for the MAL engine, {@code {"lal"}} for + * the LAL engine. * {@link RuleEngineRegistry} reads this once at registration time. */ Set supportedCatalogs(); diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/mal/MalRuleEngine.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/mal/MalRuleEngine.java index 7da4c32dbbd4..2a4684d300f4 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/mal/MalRuleEngine.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/engine/mal/MalRuleEngine.java @@ -67,10 +67,10 @@ /** * MAL implementation of {@link RuleEngine}. Owns the metric-name lifecycle: parse / classify / * compile / register / verify / commit / unregister for {@code otel-rules}, - * {@code log-mal-rules}, and {@code telegraf-rules}. All three catalogs share the same MAL - * syntax, so one engine handles all three — the catalog name only routes which dispatcher the - * MAL converter writes into (MeterSystem for otel-rules / telegraf-rules; LAL-extracted MAL - * for log-mal-rules). + * {@code log-mal-rules}, {@code telegraf-rules}, and {@code meter-analyzer-config}. All four + * catalogs share the same MAL syntax, so one engine handles them all — the catalog name only + * routes which dispatcher the MAL converter writes into (MeterSystem for otel-rules / + * telegraf-rules / meter-analyzer-config; LAL-extracted MAL for log-mal-rules). * *

Holds a stable reference to the scheduler's unified {@code rules} map at construction. * Each rule's MAL-applied artifact lives on {@link AppliedRuleScript#getApplied} (an @@ -88,7 +88,8 @@ */ @Slf4j public final class MalRuleEngine implements RuleEngine { - private static final Set CATALOGS = Set.of("otel-rules", "log-mal-rules", "telegraf-rules"); + private static final Set CATALOGS = + Set.of("otel-rules", "log-mal-rules", "telegraf-rules", "meter-analyzer-config"); private final Map rules; private final ModuleManager moduleManager; @@ -155,6 +156,12 @@ private MalConverterRegistry resolveConverterRegistry(final String catalog) { // String literal keeps telegraf-receiver-plugin out of runtime-rule's pom. moduleName = "receiver-telegraf"; break; + case "meter-analyzer-config": + // String literal keeps agent-analyzer out of runtime-rule's pom. The registry + // is MeterProcessService, which serves both the native meter gRPC receiver and + // the Kafka meter fetcher — one registry covers both ingest paths. + moduleName = "agent-analyzer"; + break; default: return null; } diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/extension/DbOverrideRuntimeRuleResolver.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/extension/DbOverrideRuntimeRuleResolver.java index e9a38a4bcb88..574c37f0e6ea 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/extension/DbOverrideRuntimeRuleResolver.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/extension/DbOverrideRuntimeRuleResolver.java @@ -216,7 +216,8 @@ public static boolean contentHasLayerDefinitions(final String catalog, final Str switch (catalog) { case "otel-rules": case "log-mal-rules": - case "telegraf-rules": { + case "telegraf-rules": + case "meter-analyzer-config": { final Rule rule = new Yaml().loadAs(reader, Rule.class); defs = rule == null ? null : rule.getLayerDefinitions(); break; diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/module/RuntimeRuleModuleProvider.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/module/RuntimeRuleModuleProvider.java index 6c50beab2a97..e1ff2781a157 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/module/RuntimeRuleModuleProvider.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/module/RuntimeRuleModuleProvider.java @@ -121,8 +121,9 @@ * ┌────── MalRuleEngine ──────┐ ┌────── LalRuleEngine ──────┐ * │ catalogs: otel-rules, │ │ catalogs: lal │ * │ log-mal-rules, │ │ │ - * │ telegraf-rules │ │ │ - * │ │ │ │ + * │ telegraf-rules, │ │ │ + * │ meter-analyzer- │ │ │ + * │ config │ │ │ * │ classify(old, new, ina) │ │ classify(old, new, ina) │ * │ claimedKeys(content, src) │ │ claimedKeys(content, src) │ * │ compile → CompiledMalDSL │ │ compile → CompiledLalDSL │ @@ -177,10 +178,13 @@ * *

Catalog → engine routing

* Catalog membership is data-driven through {@code RuleEngineRegistry}: a catalog is "MAL" - * iff a registered engine is {@code MalRuleEngine}. Adding {@code telegraf-rules} support is - * one entry in {@code MalRuleEngine.supportedCatalogs} — REST validation, scheduler routing, - * and tick enumeration pick it up automatically. (Full telegraf apply additionally requires - * the telegraf receiver module to expose a {@code MalConverterRegistry} service.) + * iff a registered engine is {@code MalRuleEngine}. Adding a MAL catalog is one entry in + * {@code MalRuleEngine.supportedCatalogs} — REST validation, scheduler routing, and tick + * enumeration pick it up automatically. A catalog additionally needs its owning receiver / + * analyzer module to expose a {@code MalConverterRegistry} service before converter push + * works; {@code meter-analyzer-config} gets that from {@code agent-analyzer}'s + * {@code MeterProcessService}, while {@code telegraf-rules} still lacks one and therefore + * degrades to "no push". * *

The full architecture (single-main routing, lock acquisition policy, marker-debt * invariant for cold-boot / topology-shift, cross-file ownership semantics, soft-pause / diff --git a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/rest/RuntimeRuleService.java b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/rest/RuntimeRuleService.java index fb5ad4fb9af9..33e8962037ce 100644 --- a/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/rest/RuntimeRuleService.java +++ b/oap-server/server-admin/runtime-rule/src/main/java/org/apache/skywalking/oap/server/receiver/runtimerule/rest/RuntimeRuleService.java @@ -828,7 +828,7 @@ private static HttpResponse renderGetResponse(final String catalog, final String * then be fetched lazily via {@code GET /runtime/rule}). * *

Catalog scope: {@code otel-rules}, {@code log-mal-rules}, {@code telegraf-rules}, - * {@code lal} — the same + * {@code meter-analyzer-config}, {@code lal} — the same * allowlist the write paths use. {@code .oal} files are not exposed here; they live * outside the runtime-rule plugin's scope today. */ diff --git a/oap-server/server-admin/runtime-rule/src/main/proto/runtime-rule-cluster.proto b/oap-server/server-admin/runtime-rule/src/main/proto/runtime-rule-cluster.proto index 54acb2032ca8..2f0756c66aac 100644 --- a/oap-server/server-admin/runtime-rule/src/main/proto/runtime-rule-cluster.proto +++ b/oap-server/server-admin/runtime-rule/src/main/proto/runtime-rule-cluster.proto @@ -64,7 +64,7 @@ service RuntimeRuleClusterService { } message SuspendRequest { - // otel-rules | log-mal-rules | lal + // otel-rules | log-mal-rules | telegraf-rules | meter-analyzer-config | lal string catalog = 1; // relative path under the catalog root, no extension, may contain '/' string name = 2; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/Catalog.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/Catalog.java index ef13f9493f2c..3c163c4004cd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/Catalog.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/classloader/Catalog.java @@ -31,6 +31,7 @@ public enum Catalog { OTEL_RULES("otel-rules"), LOG_MAL_RULES("log-mal-rules"), TELEGRAF_RULES("telegraf-rules"), + METER_ANALYZER_CONFIG("meter-analyzer-config"), LAL("lal"), OAL("oal"); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/dsldebug/RuleKey.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/dsldebug/RuleKey.java index db1621a39192..ea113ec7ba15 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/dsldebug/RuleKey.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/dsldebug/RuleKey.java @@ -34,9 +34,9 @@ *

{@link Catalog} is the existing wire-name-mapped enum already used by * the runtime-rule REST handler; reusing it here keeps the same set of * acceptable values across the whole admin surface. Phase 1 (MAL) covers the - * {@code OTEL_RULES}, {@code LOG_MAL_RULES}, and {@code TELEGRAF_RULES} - * catalogs; phase 2 adds {@code LAL}; phase 3 adds an {@code OAL} value - * when OAL probes land. + * {@code OTEL_RULES}, {@code LOG_MAL_RULES}, {@code TELEGRAF_RULES}, and + * {@code METER_ANALYZER_CONFIG} catalogs; phase 2 adds {@code LAL}; phase 3 + * adds an {@code OAL} value when OAL probes land. * *

{@code ruleName} disambiguates when a single rule file declares * multiple metrics — e.g. an OAL file with several metric definitions, or an diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/runtimerule/RuntimeRule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/runtimerule/RuntimeRule.java index 5034ed6baec4..49bab8ebf1dc 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/runtimerule/RuntimeRule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/management/runtimerule/RuntimeRule.java @@ -39,7 +39,7 @@ *

One row per (catalog, name) pair mirroring the on-disk static layout: *

    *
  • {@code catalog} — {@code otel-rules} | {@code log-mal-rules} | - * {@code telegraf-rules} | {@code lal}
    • + * {@code telegraf-rules} | {@code meter-analyzer-config} | {@code lal} *
  • {@code name} — relative path under the catalog root without extension, may contain * {@code /} (e.g. {@code aws-gateway/gateway-service})
    • *
  • {@code content} — raw file bytes, byte-identical to the original request body; marked diff --git a/oap-server/server-library/library-pprof-parser/pom.xml b/oap-server/server-library/library-pprof-parser/pom.xml index 4c4938a6c4e8..e4107e18bc8e 100755 --- a/oap-server/server-library/library-pprof-parser/pom.xml +++ b/oap-server/server-library/library-pprof-parser/pom.xml @@ -30,10 +30,6 @@ library-pprof-parser - - true - - com.google.protobuf diff --git a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java index ce893cba2889..4e0767569127 100644 --- a/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java +++ b/oap-server/server-library/library-pprof-parser/src/main/java/org/apache/skywalking/oap/server/library/pprof/type/FrameTreeBuilder.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -87,7 +86,7 @@ private void mergeSample(ProfileProto.Sample sample) { children = child.getChildren(); } else { // if the child does not exist, create a new child - RawFrameTree child = new RawFrameTree(locationId, 1, (isEnd ? 1 : 0)); + RawFrameTree child = new RawFrameTree(locationId, 1, isEnd ? 1 : 0); children.put(locationId, child); children = child.getChildren(); } diff --git a/test/e2e-v2/cases/runtime-rule/meter/bundled/batch-meter.yaml b/test/e2e-v2/cases/runtime-rule/meter/bundled/batch-meter.yaml new file mode 100644 index 000000000000..cb820dc9e945 --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/bundled/batch-meter.yaml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mounted into config/meter-analyzer-config/ as a BUNDLED (on-disk) meter rule. +# Consumes the `batch_test` raw meter the e2e mock sender pushes, and emits +# metric `batch_test` (metricPrefix `batch` + rule name `test`). +expSuffix: instance(['service'], ['instance'], Layer.GENERAL) +metricPrefix: batch +metricsRules: + - name: test + exp: batch_test diff --git a/test/e2e-v2/cases/runtime-rule/meter/docker-compose.yml b/test/e2e-v2/cases/runtime-rule/meter/docker-compose.yml new file mode 100644 index 000000000000..2889b7208dda --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/docker-compose.yml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runtime-rule hot-update over the `meter-analyzer-config` catalog. OAP + +# BanyanDB + the e2e mock sender, which pushes native MeterData (the +# `batch_test` raw meter) over gRPC to the meter receiver. +# +# The bundled `batch-meter.yaml` is mounted into config/meter-analyzer-config/ +# so the flow can assert a SHIPPED meter rule is visible to the runtime-rule +# API — that is only true once meter rules load through RuleSetMerger and land +# in StaticRuleRegistry. +services: + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_STORAGE: banyandb + SW_ADMIN_SERVER: default + SW_RECEIVER_RUNTIME_RULE: default + # Needed by phase 4: a debug session can only bind a meter rule once the + # MAL catalog predicates accept METER_ANALYZER_CONFIG and MeterProcessService + # publishes its holders through MalStaticBindingHook. + SW_DSL_DEBUGGING: default + SW_DSL_DEBUGGING_INJECTION_ENABLED: "true" + # Only the bundled test rule, so the assertions are deterministic and the + # boot-time active set stays small. + SW_METER_ANALYZER_ACTIVE_FILES: batch-meter + # Tighten persistence so the bucket-flush gap doesn't dominate the wait. + SW_CORE_PERSISTENT_PERIOD: "10" + volumes: + - ./bundled/batch-meter.yaml:/skywalking/config/meter-analyzer-config/batch-meter.yaml + ports: + - "11800:11800" + - "12800:12800" + - "17128:17128" + networks: + - e2e + + banyandb: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: banyandb + + # Native meter protocol producer: POST /sendBatchMetrics pushes a + # MeterDataCollection carrying the `batch_test` single value for + # service=test-service / instance=test-instance. + sender: + image: "eclipse-temurin:8-jre" + volumes: + - ./../../../java-test-service/e2e-mock-sender/target/e2e-mock-sender-2.0.0.jar:/e2e-mock-sender-2.0.0.jar + command: ["java", "-jar", "/e2e-mock-sender-2.0.0.jar"] + environment: + OAP_HOST: oap + OAP_GRPC_PORT: 11800 + networks: + - e2e + # Fixed host port: the flow script runs on the host and drives the sender + # directly, so it cannot use infra-e2e's ${sender_9093} templating. + ports: + - "9093:9093" + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9093"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml b/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml new file mode 100644 index 000000000000..c2af14379fbb --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runtime Rule — meter-analyzer-config catalog. Drives native MeterData through +# a BUNDLED meter rule and a hot-added pure-runtime meter rule, asserting the +# meter catalog has the same hot-update lifecycle the otel catalog has. + +setup: + env: compose + file: docker-compose.yml + # Five phases, each gated on a minute-bucket boundary plus a persistence + # flush; worst case is ~23m of polling on top of image pull + OAP boot. + timeout: 40m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install jq + command: | + if ! command -v jq >/dev/null 2>&1; then + curl -fsSL -o /tmp/skywalking-infra-e2e/bin/jq \ + https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64 + chmod +x /tmp/skywalking-infra-e2e/bin/jq + fi + - name: drive meter runtime-rule flow + command: | + set -euo pipefail + export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + export OAP_HOST=127.0.0.1 + export OAP_REST_PORT=17128 + export OAP_GQL_PORT=12800 + export SENDER_HOST=127.0.0.1 + export SENDER_PORT=9093 + export SEED_DIR=$(pwd)/test/e2e-v2/cases/runtime-rule/meter/seed-rules + bash test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh + +verify: + retry: + count: 1 + interval: 1s + cases: + - query: swctl --display json --admin-url=http://127.0.0.1:17128 admin runtime-rule list >/dev/null && echo ok + expected: expected/ok.txt + +cleanup: + on: always + collect: + on: failure + output-dir: $SW_INFRA_E2E_LOG_DIR/runtime-rule-meter + items: + - service: oap + paths: + - /skywalking/logs/ diff --git a/test/e2e-v2/cases/runtime-rule/meter/expected/ok.txt b/test/e2e-v2/cases/runtime-rule/meter/expected/ok.txt new file mode 100644 index 000000000000..9766475a4185 --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/expected/ok.txt @@ -0,0 +1 @@ +ok diff --git a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh new file mode 100755 index 000000000000..f679067d0492 --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Runtime-rule hot-update over the `meter-analyzer-config` catalog. +# +# A readiness phase followed by five capability phases, each guarding something this +# catalog did not have before meter rules started loading through meter-analyzer's +# Rules/Rule pipeline: +# +# 0. Admin API reachable (readiness only, asserts no capability). +# 1. BUNDLED VISIBILITY — the shipped batch-meter.yaml appears in +# `runtime-rule list`. Only true once meter rules go through +# RuleSetMerger and are recorded in StaticRuleRegistry. Previously the +# meter loader read YAML directly, so the catalog was invisible here. +# 2. HOT ADD — a pure-runtime meter rule (no on-disk twin) becomes a live, +# queryable metric with no OAP restart. Proves the applied MetricConvert +# reached MeterProcessService's MalConverterRegistry and that +# MeterProcessor picks it up on the next batch. +# 3. EDIT — re-applying the same (catalog, name) with an extra metric +# registers the new metric AND keeps the already-registered one flowing. +# Proves converter REPLACEMENT, not just first insertion. +# 4. DSL DEBUG — a debug session binds a BUNDLED meter rule and captures +# records carrying the verbatim DSL, real samples, and a terminal +# meterEmit. Proves the MAL catalog predicates accept the meter catalog +# and that MeterProcessService publishes its holders at boot. +# 5. INACTIVATE — the runtime rule is soft-paused: the row goes INACTIVE and +# its metrics stop producing new buckets, while the bundled rule keeps +# producing. Proves converter REMOVAL. +# +# FRESHNESS: every "is it flowing / has it stopped" assertion compares the SET +# OF TIME-BUCKET IDS carrying a non-null value against a baseline captured at +# the transition. A metric that merely retains old buckets from before the +# transition therefore does NOT satisfy "still flowing", and a stopped metric +# cannot be masked by history. Bucket ids are compared instead of absolute +# --start/--end timestamps so the assertions are immune to host-vs-container +# clock and timezone skew. +# +set -euo pipefail + +OAP_HOST="${OAP_HOST:-127.0.0.1}" +OAP_REST_PORT="${OAP_REST_PORT:-17128}" +OAP_GQL_PORT="${OAP_GQL_PORT:-12800}" +SENDER_HOST="${SENDER_HOST:-127.0.0.1}" +SENDER_PORT="${SENDER_PORT:-9093}" +SEED_DIR="${SEED_DIR:-./seed-rules}" + +CATALOG="meter-analyzer-config" +BUNDLED_NAME="batch-meter" +RUNTIME_NAME="e2e_rr_meter" + +# Metric emitted by the bundled rule (metricPrefix `batch` + rule `test`). +BUNDLED_METRIC="batch_test" +# Metric emitted by the runtime rule (metricPrefix `e2e_rr_meter` + rule `batch`). +RUNTIME_METRIC="e2e_rr_meter_batch" +# Second metric the v2 EDIT of that same rule adds. +RUNTIME_METRIC_V2="e2e_rr_meter_batch_scaled" + +# The mock sender always reports under this identity. +SVC="test-service" +INST="test-instance" + +REST_BASE="http://${OAP_HOST}:${OAP_REST_PORT}" +GQL_BASE="http://${OAP_HOST}:${OAP_GQL_PORT}/graphql" + +# First-data budget: needs a minute-bucket boundary plus a persistence flush on +# a cold pipeline. +METRIC_BUDGET_S="${METRIC_BUDGET_S:-240}" +# Budget for a NEW bucket once the pipeline is already warm. +FRESH_BUDGET_S="${FRESH_BUDGET_S:-150}" +# Grace after /inactivate for in-flight samples and the open minute bucket to +# land, before the "has it stopped" baseline is taken. +STOP_GRACE_S="${STOP_GRACE_S:-90}" +# How long to keep pushing after that baseline while proving nothing new lands. +STOP_SETTLE_S="${STOP_SETTLE_S:-120}" +# Minimum number of SUCCESSFUL observations before "the metric stopped" may be +# concluded. Guards the negative assertion against a vacuous pass when the query +# path itself is broken for the whole settle window. +MIN_STOPPED_OBSERVATIONS="${MIN_STOPPED_OBSERVATIONS:-3}" +# Budget for an /addOrUpdate to reach ACTIVE. The apply is async — the REST call returns +# after the durable commit, while the schema fence rolls out in the background (its own +# default budget is 180s), so this must not be a single read. +APPLY_LAND_S="${APPLY_LAND_S:-120}" +# Budget for the debug session to capture its first record. +DEBUG_BUDGET_S="${DEBUG_BUDGET_S:-150}" +DBG_CLIENT_ID="${DBG_CLIENT_ID:-e2e-rr-meter-dbg-1}" + +log() { echo "[meter-runtime-rule-flow] $*" >&2; } +fail() { echo "[meter-runtime-rule-flow] FAIL: $*" >&2; exit 1; } + +# Same convention as the otel-catalog flow: every runtime-rule REST call goes +# through swctl's admin command tree. swctl passes --catalog through verbatim, +# so `meter-analyzer-config` needs no CLI change. +admin() { swctl --display json --admin-url="${REST_BASE}" admin "$@"; } + +# Push one batch of native MeterData (raw meter `batch_test`). +push_meter() { + curl -s -XPOST "http://${SENDER_HOST}:${SENDER_PORT}/sendBatchMetrics" >/dev/null \ + || fail "mock sender did not accept /sendBatchMetrics" +} + +# Newline-separated, sorted set of time-bucket ids that currently carry a +# non-null value for a metric. +# +# Return code is load-bearing, and MUST stay that way: a transport error, a +# non-JSON body, or a malformed envelope returns non-zero, which is NOT the same +# as "queried fine, metric has no data yet" (empty stdout, return 0). The +# negative assertion below relies on that distinction — if a broken query were +# silently reported as an empty bucket set, "no new buckets" would be +# indistinguishable from "never managed to look", and the stopped-check would +# pass vacuously. +metric_buckets() { + local metric="$1" + local out + out="$(swctl --display json --base-url="${GQL_BASE}" metrics exec \ + --expression="${metric}" \ + --service-name="${SVC}" --instance-name="${INST}" 2>/dev/null)" \ + || return 1 + # A well-formed response is a JSON object carrying `results`. Anything else + # (empty body, HTML error page, truncated JSON) is a query failure, not an + # empty result set. `results: null` is accepted — that is the legitimate + # "metric not present yet" shape. + printf '%s' "${out}" | jq -e 'type == "object" and has("results")' >/dev/null 2>&1 \ + || return 1 + printf '%s' "${out}" \ + | jq -r '[.results[]?.values[]? | select(.value != null) | .id] | sort | .[]' \ + || return 1 +} + +# Ids present now that were absent from the supplied baseline. Propagates a +# query failure as a non-zero return so callers can tell it apart from "no new +# buckets". +new_buckets_since() { + local metric="$1" + local baseline="$2" + local current + current="$(metric_buckets "${metric}")" || return 1 + comm -13 <(printf '%s\n' "${baseline}" | sort -u) \ + <(printf '%s\n' "${current}" | sort -u) +} + +# Block until the metric produces ANY non-null bucket. Used only for the very +# first data point of a cold pipeline. +wait_metric() { + local metric="$1" + local deadline=$(( $(date +%s) + METRIC_BUDGET_S )) + local buckets + local failures=0 + while (( $(date +%s) < deadline )); do + push_meter + sleep 10 + if ! buckets="$(metric_buckets "${metric}")"; then + failures=$(( failures + 1 )) + continue + fi + if [[ -n "${buckets}" ]]; then + log " ${metric} has data" + return 0 + fi + done + fail "${metric} never produced a value within ${METRIC_BUDGET_S}s (${failures} query failure(s))" +} + +# Block until the metric produces a bucket NEWER than the supplied baseline. +# This is the assertion that actually proves the converter is live right now. +wait_fresh_metric() { + local metric="$1" + local baseline="$2" + local deadline=$(( $(date +%s) + FRESH_BUDGET_S )) + local fresh + local failures=0 + while (( $(date +%s) < deadline )); do + push_meter + sleep 10 + if ! fresh="$(new_buckets_since "${metric}" "${baseline}")"; then + failures=$(( failures + 1 )) + continue + fi + if [[ -n "${fresh}" ]]; then + log " ${metric} produced fresh bucket(s): $(echo "${fresh}" | tr '\n' ' ')" + return 0 + fi + done + fail "${metric} produced no NEW bucket within ${FRESH_BUDGET_S}s (${failures} query failure(s)) — the converter is not live" +} + +# Assert the metric produces NO bucket newer than the baseline for the whole +# settle window, while samples keep being pushed. +# +# This is a NEGATIVE assertion, so "we never saw anything" must NOT count as +# success. Every loop iteration that fails to query is recorded, and the check +# only concludes "stopped" if at least MIN_STOPPED_OBSERVATIONS iterations +# actually observed the metric successfully. +assert_metric_stopped() { + local metric="$1" + local baseline="$2" + local deadline=$(( $(date +%s) + STOP_SETTLE_S )) + local observations=0 + local failures=0 + local fresh + while (( $(date +%s) < deadline )); do + push_meter + sleep 10 + if ! fresh="$(new_buckets_since "${metric}" "${baseline}")"; then + failures=$(( failures + 1 )) + log " WARN: query for ${metric} failed (${failures} so far) — not counted as an observation" + continue + fi + observations=$(( observations + 1 )) + if [[ -n "${fresh}" ]]; then + fail "${metric} produced a NEW bucket after inactivate: $(echo "${fresh}" | tr '\n' ' ')" + fi + done + (( observations >= MIN_STOPPED_OBSERVATIONS )) \ + || fail "cannot conclude ${metric} stopped: only ${observations} successful observation(s) in ${STOP_SETTLE_S}s (${failures} query failure(s)); need >= ${MIN_STOPPED_OBSERVATIONS}" + log " ${metric} produced no new buckets across ${observations} successful observation(s) — converter removed" +} + +# Status string the /list row carries for a (catalog, name), or empty. +rule_status() { + admin runtime-rule list 2>/dev/null \ + | jq -r --arg c "${CATALOG}" --arg n "$1" \ + '.rules[] | select(.catalog == $c and .name == $n) | .status' | head -1 +} + +# ---- phase 0: admin API reachable ----------------------------------------- +log "phase 0: waiting for the runtime-rule admin API" +deadline=$(( $(date +%s) + 180 )) +until admin runtime-rule list >/dev/null 2>&1; do + (( $(date +%s) < deadline )) || fail "runtime-rule admin API never became reachable" + sleep 3 +done +log " admin API up" + +# ---- phase 1: bundled meter rule is visible ------------------------------- +log "phase 1: bundled ${CATALOG}/${BUNDLED_NAME} must be visible to runtime-rule" +admin runtime-rule list \ + | jq -e --arg c "${CATALOG}" --arg n "${BUNDLED_NAME}" \ + '.rules[] | select(.catalog==$c and .name==$n)' >/dev/null \ + || fail "bundled ${CATALOG}/${BUNDLED_NAME} is NOT visible in runtime-rule list — meter rules are not reaching StaticRuleRegistry" +log " bundled rule visible" + +log " and the bundled metric ${BUNDLED_METRIC} produces data" +wait_metric "${BUNDLED_METRIC}" + +# ---- phase 2: hot-add a pure-runtime meter rule --------------------------- +log "phase 2: hot-add ${CATALOG}/${RUNTIME_NAME} (no on-disk twin)" +admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ + -f "${SEED_DIR}/meter-v1.yaml" \ + || fail "addOrUpdate of ${RUNTIME_NAME} returned non-2xx" + +# The apply is asynchronous — addOrUpdate returns once the row is persisted, while the +# schema fence and peer roll-out complete in the background. Poll rather than reading the +# status once, exactly as cases/dsl-debugging/mal/dsl-debug-flow.sh does. +wait_rule_active() { + local name="$1" + local deadline=$(( $(date +%s) + APPLY_LAND_S )) + local status="" + while (( $(date +%s) < deadline )); do + status="$(rule_status "${name}" || true)" + [[ "${status}" == "ACTIVE" ]] && { log " ${CATALOG}/${name} is ACTIVE"; return 0; } + sleep 3 + done + fail "${CATALOG}/${name} did not reach ACTIVE within ${APPLY_LAND_S}s (last saw '${status}')" +} + +wait_rule_active "${RUNTIME_NAME}" + +log " applied; ${RUNTIME_METRIC} must become queryable with no restart" +wait_metric "${RUNTIME_METRIC}" + +# ---- phase 3: EDIT the runtime rule in place ------------------------------ +# Baselines are captured BEFORE the edit, so the post-edit assertions cannot be +# satisfied by buckets the v1 converter already wrote. +log "phase 3: edit ${CATALOG}/${RUNTIME_NAME} in place (v1 -> v2, adds a metric)" +pre_edit_v1="$(metric_buckets "${RUNTIME_METRIC}")" \ + || fail "could not read a baseline for ${RUNTIME_METRIC} before the edit" +# v2's metric does not exist yet, so an empty set is the expected answer here. +# Tolerate a query failure specifically because "unknown metric" may surface as +# an error rather than an empty envelope; an empty baseline only ever makes the +# follow-up fresh-bucket assertion easier to satisfy, never the stopped-check. +pre_edit_v2="$(metric_buckets "${RUNTIME_METRIC_V2}" || true)" + +admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ + -f "${SEED_DIR}/meter-v2.yaml" \ + || fail "addOrUpdate (edit) of ${RUNTIME_NAME} returned non-2xx" + +# Declaring an extra metric makes this a STRUCTURAL apply, so the fence/roll-out window is +# at its widest here. Settle on ACTIVE first, so a stalled apply is reported as such rather +# than surfacing later as a confusing "converter is not live". +wait_rule_active "${RUNTIME_NAME}" + +log " the newly declared ${RUNTIME_METRIC_V2} must produce a fresh bucket" +wait_fresh_metric "${RUNTIME_METRIC_V2}" "${pre_edit_v2}" + +log " and the metric v1 already registered must still produce fresh buckets" +wait_fresh_metric "${RUNTIME_METRIC}" "${pre_edit_v1}" + +# ---- phase 4: DSL debug session against a BUNDLED meter rule -------------- +# Deliberately targets the BUNDLED rule, not the runtime one: binding a runtime +# rule only exercises MalRuleEngine's generic publish path, which already worked +# for otel. Binding a BUNDLED meter rule exercises the wiring that is new here — +# MeterProcessService.start() calling MalStaticBindingHook.publish, plus +# MALHolderRegistry/MALDebugRecorderFactory accepting METER_ANALYZER_CONFIG. +# +# Assertions are deliberately shape-agnostic. The sibling otel case +# (cases/dsl-debugging/mal) asserts a much richer envelope, but those checks are +# bound to its seed rule's shape (multi-metric expression, decoy family, a +# `plus` op). The mock sender emits a single unlabelled raw meter, so the +# meaningful assertion here is "a meter rule can be bound at all, and its +# captures carry a real DSL body, real samples, and a terminal meterEmit". +log "phase 4: DSL debug session on bundled ${CATALOG}/${BUNDLED_NAME}" + +admin dsl-debug status | jq -e '.injectionEnabled == true' >/dev/null \ + || fail "dsl-debug injectionEnabled is not true" + +install_body="$(admin dsl-debug session start \ + --catalog "${CATALOG}" --name "${BUNDLED_NAME}" \ + --rule-name "${BUNDLED_METRIC}" --client-id "${DBG_CLIENT_ID}")" \ + || fail "dsl-debug session start rejected the ${CATALOG} catalog" +session_id="$(echo "${install_body}" | jq -r '.sessionId // empty')" +[[ -n "${session_id}" ]] \ + || fail "session start returned no sessionId — body: ${install_body}" +log " session installed: ${session_id}" + +deadline=$(( $(date +%s) + DEBUG_BUDGET_S )) +collect_body="" +records=0 +while (( $(date +%s) < deadline )); do + push_meter + sleep 5 + collect_body="$(admin dsl-debug session get "${session_id}")" + records="$(echo "${collect_body}" | jq '[.nodes[].records[]] | length')" + (( records > 0 )) && break +done +(( records > 0 )) \ + || fail "no debug records captured for the bundled meter rule within ${DEBUG_BUDGET_S}s — payload: ${collect_body}" +log " captured ${records} record(s)" + +# Every record must carry the verbatim rule body as of capture. +empty_dsl="$(echo "${collect_body}" | jq '[.nodes[].records[] | select((.dsl // "") == "")] | length')" +[[ "${empty_dsl}" == "0" ]] || fail "${empty_dsl} record(s) carry empty .dsl" + +empty_samples="$(echo "${collect_body}" | jq '[.nodes[].records[] | select((.samples | length) == 0)] | length')" +[[ "${empty_samples}" == "0" ]] || fail "${empty_samples} record(s) carry zero samples" + +# At least one capture saw real traffic rather than an empty SampleFamily. +real_traffic="$(echo "${collect_body}" | jq '[.nodes[].records[].samples[] | select((.payload.samples // 0) > 0)] | length')" +(( real_traffic > 0 )) \ + || fail "no sample reports payload.samples > 0 — meter data never reached the rule" + +# And at least one execution closes on a meterEmit carrying this metric. +emit="$(echo "${collect_body}" | jq --arg n "${BUNDLED_METRIC}" \ + '[.nodes[].records[] | .samples[-1] | select(.payload.metric == $n)] | length')" +(( emit > 0 )) \ + || fail "no execution closes with a meterEmit sample carrying payload.metric=${BUNDLED_METRIC}" +log " envelope valid: dsl + samples + terminal meterEmit for ${BUNDLED_METRIC}" + +admin dsl-debug session stop "${session_id}" | jq -e '.localStopped == true' >/dev/null \ + || fail "dsl-debug session stop did not report localStopped" +log " session stopped" + +# ---- phase 5: inactivate the runtime rule --------------------------------- +log "phase 5: inactivate ${CATALOG}/${RUNTIME_NAME}" +admin runtime-rule inactivate --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ + || fail "inactivate of ${RUNTIME_NAME} returned non-2xx" + +# Also async: poll rather than reading once. +inactivate_deadline=$(( $(date +%s) + APPLY_LAND_S )) +status="" +while (( $(date +%s) < inactivate_deadline )); do + status="$(rule_status "${RUNTIME_NAME}" || true)" + [[ "${status}" == "INACTIVE" ]] && break + sleep 3 +done +[[ "${status}" == "INACTIVE" ]] \ + || fail "${CATALOG}/${RUNTIME_NAME} status is '${status}' after ${APPLY_LAND_S}s, expected INACTIVE" +log " row is INACTIVE" + +# Let in-flight samples and the currently-open minute bucket land before taking +# the "nothing new after this point" baseline. +log " waiting ${STOP_GRACE_S}s for in-flight buckets to settle" +grace_deadline=$(( $(date +%s) + STOP_GRACE_S )) +while (( $(date +%s) < grace_deadline )); do + push_meter + sleep 10 +done + +# Both metrics are known to exist by now, so a failure here is a real problem — +# and an empty baseline would weaken the stopped-check that follows. +post_inactivate_runtime="$(metric_buckets "${RUNTIME_METRIC}")" \ + || fail "could not read a post-inactivate baseline for ${RUNTIME_METRIC}" +post_inactivate_bundled="$(metric_buckets "${BUNDLED_METRIC}")" \ + || fail "could not read a post-inactivate baseline for ${BUNDLED_METRIC}" + +log " ${RUNTIME_METRIC} must produce NO new buckets from here on" +assert_metric_stopped "${RUNTIME_METRIC}" "${post_inactivate_runtime}" + +# Same window, opposite expectation: the bundled rule is untouched by the +# runtime rule's teardown and must still be writing fresh buckets. +log " while bundled ${BUNDLED_METRIC} must still produce fresh buckets" +wait_fresh_metric "${BUNDLED_METRIC}" "${post_inactivate_bundled}" + +log "ALL PHASES PASSED" diff --git a/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v1.yaml b/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v1.yaml new file mode 100644 index 000000000000..b87e1cd60690 --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v1.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pure-runtime meter rule: has NO on-disk twin, so it can only reach the meter +# pipeline through runtime-rule /addOrUpdate landing a converter in +# MeterProcessService's MalConverterRegistry. Reuses the mock sender's existing +# `batch_test` raw meter so no Java test-service change is needed. +# Emits metric `e2e_rr_meter_batch`. +expSuffix: instance(['service'], ['instance'], Layer.GENERAL) +metricPrefix: e2e_rr_meter +metricsRules: + - name: batch + exp: batch_test diff --git a/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v2.yaml b/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v2.yaml new file mode 100644 index 000000000000..3cbdc09182dd --- /dev/null +++ b/test/e2e-v2/cases/runtime-rule/meter/seed-rules/meter-v2.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# EDIT of meter-v1.yaml under the same (catalog, name). Declares one ADDITIONAL +# metric, so the apply classifies as STRUCTURAL: the engine must re-compile the +# bundle, register the new metric with MeterSystem, and swap the converter in +# MeterProcessService's registry — all without dropping the metric that v1 +# already registered. +expSuffix: instance(['service'], ['instance'], Layer.GENERAL) +metricPrefix: e2e_rr_meter +metricsRules: + - name: batch + exp: batch_test + - name: batch_scaled + exp: batch_test * 2 From 880593b14a6227a66c94171522be036875d99518 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Thu, 6 Aug 2026 17:26:25 -0700 Subject: [PATCH 2/4] Address review: synchronise the meter e2e on the async apply phase The row's ACTIVE status is not a synchronisation point for an apply. A STRUCTURAL addOrUpdate returns immediately at FENCING with an applyId while the schema fence and peer roll-out finish in the background, and on an edit the row is already ACTIVE from the previous apply - so polling for ACTIVE returned instantly and proved nothing, and could never surface FAILED or DEGRADED. Adopt the await_apply_terminal helper the otel mal-storage flow uses: poll GET /runtime/rule/status by applyId until APPLIED/DEGRADED, fail on FAILED. Also re-baseline the v1 metric AFTER the edit reaches a terminal phase. The previous baseline was captured before the edit, so the "v1 still flowing" assertion could be satisfied by buckets the OLD converter emitted while the structural apply was still fencing - proving the old converter was alive rather than that the replacement kept v1's metric registered. Raise the case timeout to 50m to keep every internal budget tripping before the harness deadline: a harness timeout cancels the job, and upload-artifact: if failure() does not run on cancellation, so a timeout kill yields no logs to debug from. --- test/e2e-v2/cases/runtime-rule/meter/e2e.yaml | 9 +- .../meter/meter-runtime-rule-flow.sh | 96 ++++++++++++++----- 2 files changed, 78 insertions(+), 27 deletions(-) diff --git a/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml b/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml index c2af14379fbb..249f75710bef 100644 --- a/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml +++ b/test/e2e-v2/cases/runtime-rule/meter/e2e.yaml @@ -20,9 +20,12 @@ setup: env: compose file: docker-compose.yml - # Five phases, each gated on a minute-bucket boundary plus a persistence - # flush; worst case is ~23m of polling on top of image pull + OAP boot. - timeout: 40m + # Six phases, each gated on an async-apply terminal phase and/or a minute-bucket + # boundary plus a persistence flush; pathological worst case is ~32m of polling on + # top of image pull + OAP boot. Deliberately generous: a harness timeout CANCELS the + # job, and `upload-artifact: if failure()` does not run on cancellation, so a timeout + # kill yields no logs. Every internal budget must trip before this one does. + timeout: 50m init-system-environment: ../../../script/env steps: - name: set PATH diff --git a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh index f679067d0492..83e9acd1becf 100755 --- a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh +++ b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh @@ -89,6 +89,9 @@ STOP_SETTLE_S="${STOP_SETTLE_S:-120}" # concluded. Guards the negative assertion against a vacuous pass when the query # path itself is broken for the whole settle window. MIN_STOPPED_OBSERVATIONS="${MIN_STOPPED_OBSERVATIONS:-3}" +# Budget for the async apply state machine to reach a terminal phase on +# GET /runtime/rule/status. BanyanDB's meta->data-node schema sync can take 1-2 minutes. +APPLY_TERMINAL_S="${APPLY_TERMINAL_S:-200}" # Budget for an /addOrUpdate to reach ACTIVE. The apply is async — the REST call returns # after the durable commit, while the schema fence rolls out in the background (its own # default budget is 180s), so this must not be a single read. @@ -257,26 +260,65 @@ wait_metric "${BUNDLED_METRIC}" # ---- phase 2: hot-add a pure-runtime meter rule --------------------------- log "phase 2: hot-add ${CATALOG}/${RUNTIME_NAME} (no on-disk twin)" -admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ - -f "${SEED_DIR}/meter-v1.yaml" \ +add_resp="$(admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ + -f "${SEED_DIR}/meter-v1.yaml")" \ || fail "addOrUpdate of ${RUNTIME_NAME} returned non-2xx" -# The apply is asynchronous — addOrUpdate returns once the row is persisted, while the -# schema fence and peer roll-out complete in the background. Poll rather than reading the -# status once, exactly as cases/dsl-debugging/mal/dsl-debug-flow.sh does. -wait_rule_active() { - local name="$1" - local deadline=$(( $(date +%s) + APPLY_LAND_S )) - local status="" - while (( $(date +%s) < deadline )); do - status="$(rule_status "${name}" || true)" - [[ "${status}" == "ACTIVE" ]] && { log " ${CATALOG}/${name} is ACTIVE"; return 0; } - sleep 3 +# Drive the async apply to a terminal phase, mirroring +# cases/runtime-rule/mal-storage/runtime-rule-flow.sh's helper. +# +# The row's ACTIVE status is NOT a synchronisation point: a STRUCTURAL addOrUpdate returns +# immediately at FENCING with an applyId while the schema fence and peer roll-out finish in +# the background, and on an EDIT the row is already ACTIVE from the previous apply — so +# polling for ACTIVE would return instantly and prove nothing. The applyId phase is the only +# signal that distinguishes "committed and rolled out" from "still fencing", and it is also +# the only one that can surface FAILED / DEGRADED. +# +# A synchronous apply (filter_only / inactivate) carries no applyId; the response is already +# durable on return, so this is a no-op there. +await_apply_terminal() { + local resp="$1" + local rule_name="${2:-${RUNTIME_NAME}}" + local apply_id + apply_id="$(echo "${resp}" | jq -r '.applyId // empty' 2>/dev/null || true)" + if [[ -z "${apply_id}" ]]; then + log " (synchronous apply, no applyId — already durable)" + return 0 + fi + local deadline=$(( $(date +%s) + APPLY_TERMINAL_S )) + local body phase="" + while :; do + body="$(curl -s "${REST_BASE}/runtime/rule/status?applyId=${apply_id}&catalog=${CATALOG}&name=${rule_name}" 2>/dev/null || true)" + phase="$(echo "${body}" | jq -r '.phase // empty' 2>/dev/null || true)" + case "${phase}" in + APPLIED|DEGRADED) + log " apply ${apply_id} reached ${phase} (durable)" + return 0 + ;; + FAILED) + fail "apply ${apply_id} of ${CATALOG}/${rule_name} reached FAILED: ${body}" + ;; + esac + if (( $(date +%s) >= deadline )); then + fail "apply ${apply_id} of ${CATALOG}/${rule_name} did not reach a terminal phase within ${APPLY_TERMINAL_S}s (last phase='${phase}', body: ${body})" + fi + sleep 2 done - fail "${CATALOG}/${name} did not reach ACTIVE within ${APPLY_LAND_S}s (last saw '${status}')" } -wait_rule_active "${RUNTIME_NAME}" +await_apply_terminal "${add_resp}" + +# Belt-and-braces: the row must also be visible as ACTIVE. +active_deadline=$(( $(date +%s) + APPLY_LAND_S )) +row_status="" +while (( $(date +%s) < active_deadline )); do + row_status="$(rule_status "${RUNTIME_NAME}" || true)" + [[ "${row_status}" == "ACTIVE" ]] && break + sleep 3 +done +[[ "${row_status}" == "ACTIVE" ]] \ + || fail "${CATALOG}/${RUNTIME_NAME} did not reach ACTIVE within ${APPLY_LAND_S}s (last saw '${row_status}')" +log " ${CATALOG}/${RUNTIME_NAME} is ACTIVE" log " applied; ${RUNTIME_METRIC} must become queryable with no restart" wait_metric "${RUNTIME_METRIC}" @@ -285,28 +327,34 @@ wait_metric "${RUNTIME_METRIC}" # Baselines are captured BEFORE the edit, so the post-edit assertions cannot be # satisfied by buckets the v1 converter already wrote. log "phase 3: edit ${CATALOG}/${RUNTIME_NAME} in place (v1 -> v2, adds a metric)" -pre_edit_v1="$(metric_buckets "${RUNTIME_METRIC}")" \ - || fail "could not read a baseline for ${RUNTIME_METRIC} before the edit" # v2's metric does not exist yet, so an empty set is the expected answer here. # Tolerate a query failure specifically because "unknown metric" may surface as # an error rather than an empty envelope; an empty baseline only ever makes the # follow-up fresh-bucket assertion easier to satisfy, never the stopped-check. pre_edit_v2="$(metric_buckets "${RUNTIME_METRIC_V2}" || true)" -admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ - -f "${SEED_DIR}/meter-v2.yaml" \ +edit_resp="$(admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ + -f "${SEED_DIR}/meter-v2.yaml")" \ || fail "addOrUpdate (edit) of ${RUNTIME_NAME} returned non-2xx" # Declaring an extra metric makes this a STRUCTURAL apply, so the fence/roll-out window is -# at its widest here. Settle on ACTIVE first, so a stalled apply is reported as such rather -# than surfacing later as a confusing "converter is not live". -wait_rule_active "${RUNTIME_NAME}" +# at its widest here. Block until the apply is durably terminal before asserting anything — +# the row is already ACTIVE from phase 2, so row status proves nothing about THIS apply. +await_apply_terminal "${edit_resp}" log " the newly declared ${RUNTIME_METRIC_V2} must produce a fresh bucket" wait_fresh_metric "${RUNTIME_METRIC_V2}" "${pre_edit_v2}" -log " and the metric v1 already registered must still produce fresh buckets" -wait_fresh_metric "${RUNTIME_METRIC}" "${pre_edit_v1}" +# v1's baseline is taken HERE, after the edit is terminal — not before it. A baseline +# captured pre-edit would be satisfied by buckets the OLD v1 converter emitted while the +# structural apply was still fencing (and while the v2 wait above burned minutes), which +# would prove the old converter was alive, not that the REPLACEMENT converter kept v1's +# metric registered. Re-baselining makes the assertion strictly post-replacement. +post_edit_v1="$(metric_buckets "${RUNTIME_METRIC}")" \ + || fail "could not read a post-edit baseline for ${RUNTIME_METRIC}" + +log " and the metric v1 already registered must still produce buckets NEWER than the edit" +wait_fresh_metric "${RUNTIME_METRIC}" "${post_edit_v1}" # ---- phase 4: DSL debug session against a BUNDLED meter rule -------------- # Deliberately targets the BUNDLED rule, not the runtime one: binding a runtime From f6ee7c9af85592129c51fc7189e54584bb14834c Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Thu, 6 Aug 2026 18:04:12 -0700 Subject: [PATCH 3/4] Fix meter e2e phase 1: probe /runtime/rule/bundled, not /list CI showed phase 1 failing with "bundled batch-meter is NOT visible in runtime-rule list". The feature is fine - the OAP log confirms the boot resolver ran for meter-analyzer-config, StaticRuleLoader recorded the bundled rules, and MalRuleEngine completed a bundled fall-over for batch-meter. The assertion was simply pointed at the wrong endpoint. /list's bundled branch only walks snapshot entries carrying a non-null DSLRuntimeState, and MalRuleEngine.installBundled deliberately resets state to null after a bundled fall-over so the next gone-keys pass skips it as an untouched bundled-only entry. A fall-over'd bundled rule is therefore legitimately absent from /list. GET /runtime/rule/bundled reads StaticRuleRegistry directly, which is exactly the registration this change adds, so it is both the correct probe and still a genuine regression guard - before this change the meter catalog was empty there. Also poll rather than assert once: agent-analyzer now declares StorageModule and so starts late in the boot sequence, meaning the admin API can answer before its static rules are recorded. --- .../meter/meter-runtime-rule-flow.sh | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh index 83e9acd1becf..1efb79e7d288 100755 --- a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh +++ b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh @@ -21,10 +21,11 @@ # Rules/Rule pipeline: # # 0. Admin API reachable (readiness only, asserts no capability). -# 1. BUNDLED VISIBILITY — the shipped batch-meter.yaml appears in -# `runtime-rule list`. Only true once meter rules go through -# RuleSetMerger and are recorded in StaticRuleRegistry. Previously the -# meter loader read YAML directly, so the catalog was invisible here. +# 1. BUNDLED VISIBILITY — the shipped batch-meter.yaml is listed by +# GET /runtime/rule/bundled, which reads StaticRuleRegistry directly. +# Only true once meter rules go through RuleSetMerger and are recorded +# there. Previously the meter loader read YAML directly, so the catalog +# was invisible to the whole runtime-rule surface. # 2. HOT ADD — a pure-runtime meter rule (no on-disk twin) becomes a live, # queryable metric with no OAP restart. Proves the applied MetricConvert # reached MeterProcessService's MalConverterRegistry and that @@ -92,6 +93,10 @@ MIN_STOPPED_OBSERVATIONS="${MIN_STOPPED_OBSERVATIONS:-3}" # Budget for the async apply state machine to reach a terminal phase on # GET /runtime/rule/status. BanyanDB's meta->data-node schema sync can take 1-2 minutes. APPLY_TERMINAL_S="${APPLY_TERMINAL_S:-200}" +# Budget for the bundled meter rules to appear in StaticRuleRegistry. agent-analyzer now +# declares StorageModule, so it starts late in the boot sequence and the admin API can +# answer before its static rules are recorded. +BUNDLED_VISIBLE_S="${BUNDLED_VISIBLE_S:-120}" # Budget for an /addOrUpdate to reach ACTIVE. The apply is async — the REST call returns # after the durable commit, while the schema fence rolls out in the background (its own # default budget is 180s), so this must not be a single read. @@ -249,11 +254,30 @@ log " admin API up" # ---- phase 1: bundled meter rule is visible ------------------------------- log "phase 1: bundled ${CATALOG}/${BUNDLED_NAME} must be visible to runtime-rule" -admin runtime-rule list \ - | jq -e --arg c "${CATALOG}" --arg n "${BUNDLED_NAME}" \ - '.rules[] | select(.catalog==$c and .name==$n)' >/dev/null \ - || fail "bundled ${CATALOG}/${BUNDLED_NAME} is NOT visible in runtime-rule list — meter rules are not reaching StaticRuleRegistry" -log " bundled rule visible" +# Probe GET /runtime/rule/bundled, NOT /list. /bundled reads StaticRuleRegistry directly, +# which is exactly the registration this change adds. /list is the wrong endpoint here: its +# bundled branch only walks snapshot entries carrying a non-null DSLRuntimeState, and +# MalRuleEngine.installBundled deliberately resets state to null after a bundled fall-over +# (so the next gone-keys pass skips it as an untouched bundled-only entry) — so a +# fall-over'd bundled rule is legitimately absent from /list. +# +# Polled because agent-analyzer now starts late in the boot sequence (it declares +# StorageModule), so the admin API can answer before its rules are registered. +bundled_deadline=$(( $(date +%s) + BUNDLED_VISIBLE_S )) +bundled_seen=0 +while (( $(date +%s) < bundled_deadline )); do + # Shape-tolerant: matches a bare array of rows or any envelope that nests them. + if admin runtime-rule bundled --catalog "${CATALOG}" 2>/dev/null \ + | jq -e --arg n "${BUNDLED_NAME}" \ + '[.. | objects | select(.name? == $n)] | length > 0' >/dev/null 2>&1; then + bundled_seen=1 + break + fi + sleep 3 +done +(( bundled_seen == 1 )) \ + || fail "bundled ${CATALOG}/${BUNDLED_NAME} is NOT listed by /runtime/rule/bundled within ${BUNDLED_VISIBLE_S}s — meter rules are not reaching StaticRuleRegistry" +log " bundled rule visible via /runtime/rule/bundled" log " and the bundled metric ${BUNDLED_METRIC} produces data" wait_metric "${BUNDLED_METRIC}" From ee893c537b757094a7feb1dc50054ac0e9a82c12 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Thu, 6 Aug 2026 19:28:42 -0700 Subject: [PATCH 4/4] Retry transient 503 cluster_not_ready in the meter e2e CI got through phases 0-4 (bundled visibility via /runtime/rule/bundled, hot-add, structural edit with converter replacement, and a DSL debug session binding a bundled meter rule) and then failed phase 5 with HTTP 503 on /inactivate. That is the cluster routing layer's transient cluster_not_ready while its peer refresh is in flight. The otel catalog's flow already documents it as happening reliably right after a STRUCTURAL apply, and phase 3 is exactly that, so phase 5 lands inside the window. Adopt the same retry_admin treatment for the three mutating calls. Unlike the otel helper this keeps stdout clean - stderr is captured to a temp file instead of being folded in with 2>&1 - because the add/edit responses are piped into jq to read .applyId, and a merged stderr line would corrupt that JSON and silently reduce await_apply_terminal to a no-op. Both streams are scanned for the 503 marker so a change in where swctl writes its fatal envelope cannot quietly disable the retry. --- .../meter/meter-runtime-rule-flow.sh | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh index 1efb79e7d288..fc397577f2fd 100755 --- a/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh +++ b/test/e2e-v2/cases/runtime-rule/meter/meter-runtime-rule-flow.sh @@ -113,6 +113,42 @@ fail() { echo "[meter-runtime-rule-flow] FAIL: $*" >&2; exit 1; } # so `meter-analyzer-config` needs no CLI change. admin() { swctl --display json --admin-url="${REST_BASE}" admin "$@"; } +# Retry a runtime-rule admin call while it returns 503. The cluster routing layer +# transiently answers 503 cluster_not_ready while its peer refresh is in flight, which the +# otel catalog's flow documents as happening reliably right after a STRUCTURAL apply — and +# phase 3 is exactly that, so phase 5's inactivate lands inside the window. +# +# Unlike the otel helper this keeps stdout CLEAN (stderr is captured separately rather than +# folded in with 2>&1), because callers pipe the response into jq to read .applyId — a +# merged stderr line would corrupt that JSON and silently reduce await_apply_terminal to a +# no-op. +RETRY_BUDGET_S="${RETRY_BUDGET_S:-90}" +retry_admin() { + local deadline=$(( $(date +%s) + RETRY_BUDGET_S )) + local errf out rc + errf="$(mktemp)" + while :; do + rc=0 + out="$(admin "$@" 2>"${errf}")" || rc=$? + if (( rc == 0 )); then + rm -f "${errf}" + printf '%s' "${out}" + return 0 + fi + # Check BOTH streams: swctl's fatal envelope has been seen on stderr, but stdout is + # checked too so a change in where the CLI writes it cannot silently disable the retry. + if [[ "${out}$(cat "${errf}" 2>/dev/null)" == *"HTTP 503"* ]] \ + && (( $(date +%s) < deadline )); then + log " transient 503 on 'admin $*' — retrying" + sleep 3 + continue + fi + log " admin $* failed (rc=${rc}): $(tr '\n' ' ' < "${errf}" 2>/dev/null)" + rm -f "${errf}" + return "${rc}" + done +} + # Push one batch of native MeterData (raw meter `batch_test`). push_meter() { curl -s -XPOST "http://${SENDER_HOST}:${SENDER_PORT}/sendBatchMetrics" >/dev/null \ @@ -284,7 +320,7 @@ wait_metric "${BUNDLED_METRIC}" # ---- phase 2: hot-add a pure-runtime meter rule --------------------------- log "phase 2: hot-add ${CATALOG}/${RUNTIME_NAME} (no on-disk twin)" -add_resp="$(admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ +add_resp="$(retry_admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ -f "${SEED_DIR}/meter-v1.yaml")" \ || fail "addOrUpdate of ${RUNTIME_NAME} returned non-2xx" @@ -357,7 +393,7 @@ log "phase 3: edit ${CATALOG}/${RUNTIME_NAME} in place (v1 -> v2, adds a metric) # follow-up fresh-bucket assertion easier to satisfy, never the stopped-check. pre_edit_v2="$(metric_buckets "${RUNTIME_METRIC_V2}" || true)" -edit_resp="$(admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ +edit_resp="$(retry_admin runtime-rule add --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ -f "${SEED_DIR}/meter-v2.yaml")" \ || fail "addOrUpdate (edit) of ${RUNTIME_NAME} returned non-2xx" @@ -446,8 +482,8 @@ log " session stopped" # ---- phase 5: inactivate the runtime rule --------------------------------- log "phase 5: inactivate ${CATALOG}/${RUNTIME_NAME}" -admin runtime-rule inactivate --catalog "${CATALOG}" --name "${RUNTIME_NAME}" \ - || fail "inactivate of ${RUNTIME_NAME} returned non-2xx" +retry_admin runtime-rule inactivate --catalog "${CATALOG}" --name "${RUNTIME_NAME}" >/dev/null \ + || fail "inactivate of ${RUNTIME_NAME} returned non-2xx (see the admin error logged above)" # Also async: poll rather than reading once. inactivate_deadline=$(( $(date +%s) + APPLY_LAND_S ))