diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 87ac17bec01a..76c1832f3b32 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -635,6 +635,14 @@ http { proxy_ssl_trusted_certificate {* ssl.ssl_trusted_certificate *}; {% end %} + # opentelemetry_set_ngx_var starts + {% if opentelemetry_set_ngx_var then %} + set $opentelemetry_context_traceparent ''; + set $opentelemetry_trace_id ''; + set $opentelemetry_span_id ''; + {% end %} + # opentelemetry_set_ngx_var ends + # http server configuration snippet starts {% if http_server_configuration_snippet then %} {* http_server_configuration_snippet *} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index c1a575ed4f87..a5a6975d80bd 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -542,6 +542,10 @@ Please modify "admin_key" in conf/config.yaml . end end + local opentelemetry_set_ngx_var + if enabled_plugins["opentelemetry"] and yaml_conf.plugin_attr["opentelemetry"] then + opentelemetry_set_ngx_var = yaml_conf.plugin_attr["opentelemetry"].set_ngx_var + end -- Using template.render local sys_conf = { lua_path = env.pkg_path_org, @@ -562,6 +566,7 @@ Please modify "admin_key" in conf/config.yaml . control_server_addr = control_server_addr, prometheus_server_addr = prometheus_server_addr, proxy_mirror_timeouts = proxy_mirror_timeouts, + opentelemetry_set_ngx_var = opentelemetry_set_ngx_var } if not yaml_conf.apisix then diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua index f8013e6f7675..0c84fad49da2 100644 --- a/apisix/plugins/opentelemetry.lua +++ b/apisix/plugins/opentelemetry.lua @@ -47,6 +47,7 @@ local type = type local pairs = pairs local ipairs = ipairs local unpack = unpack +local string_format = string.format local lrucache = core.lrucache.new({ type = 'plugin', count = 128, ttl = 24 * 60 * 60, @@ -112,6 +113,11 @@ local attr_schema = { }, default = {}, }, + set_ngx_var = { + type = "boolean", + description = "set nginx variables", + default = false, + }, }, } @@ -332,6 +338,17 @@ function _M.rewrite(conf, api_ctx) kind = span_kind.server, attributes = attributes, }) + + if plugin_info.set_ngx_var then + local span_context = ctx:span():context() + ngx_var.opentelemetry_context_traceparent = string_format("00-%s-%s-%02x", + span_context.trace_id, + span_context.span_id, + span_context.trace_flags) + ngx_var.opentelemetry_trace_id = span_context.trace_id + ngx_var.opentelemetry_span_id = span_context.span_id + end + api_ctx.otel_context_token = ctx:attach() -- inject trace context into the headers of upstream HTTP request diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 05dc50ad72d1..f15f6d1df604 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -570,6 +570,7 @@ plugin_attr: # Plugin attributes inactive_timeout: 1 # Set the timeout for spans to wait in the export queue before being sent, # if the queue is not full. max_export_batch_size: 16 # Set the maximum number of spans to include in each batch sent to the + set_ngx_var: false # export opentelemetry variables to nginx variables # OpenTelemetry collector. prometheus: # Plugin: prometheus export_uri: /apisix/prometheus/metrics # Set the URI for the Prometheus metrics endpoint. diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md index eca682a061aa..55171d539f74 100644 --- a/docs/en/latest/plugins/opentelemetry.md +++ b/docs/en/latest/plugins/opentelemetry.md @@ -89,6 +89,29 @@ plugin_attr: max_export_batch_size: 2 ``` +## Variables + +The following nginx variables are set by OpenTelemetry: + +- `opentelemetry_context_traceparent` - [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), e.g.: `00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01` +- `opentelemetry_trace_id` - Trace Id of the current span +- `opentelemetry_span_id` - Span Id of the current span + +How to use variables? you have to add it to your configuration file (`conf/config.yaml`): + +```yaml title="./conf/config.yaml" +http: + enable_access_log: true + access_log: "/dev/stdout" + access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}' + access_log_format_escape: json +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + set_ngx_var: true +``` + ## Enable Plugin To enable the Plugin, you have to add it to your configuration file (`conf/config.yaml`): diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md index e4794c8e2ca8..474aafb012d3 100644 --- a/docs/zh/latest/plugins/opentelemetry.md +++ b/docs/zh/latest/plugins/opentelemetry.md @@ -88,6 +88,29 @@ plugin_attr: max_export_batch_size: 2 ``` +## 如何使用变量 + +以下`nginx`变量是由`opentelemetry` 设置的。 + +- `opentelemetry_context_traceparent` - [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), 例如：`00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01` +- `opentelemetry_trace_id` - 当前 span 的 trace_id +- `opentelemetry_span_id` - 当前 span 的 span_id + +如何使用？你需要在配置文件（`./conf/config.yaml`）设置如下： + +```yaml title="./conf/config.yaml" +http: + enable_access_log: true + access_log: "/dev/stdout" + access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}' + access_log_format_escape: json +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + set_ngx_var: true +``` + ## 如何启用 `opentelemetry` 插件默认为禁用状态，你需要在配置文件（`./conf/config.yaml`）中开启该插件： diff --git a/t/cli/test_opentelemetry_set_ngx_var.sh b/t/cli/test_opentelemetry_set_ngx_var.sh new file mode 100755 index 000000000000..8db6ea420db2 --- /dev/null +++ b/t/cli/test_opentelemetry_set_ngx_var.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. ./t/cli/common.sh + +echo ' +plugins: + - opentelemetry +plugin_attr: + opentelemetry: + set_ngx_var: true +' > conf/config.yaml + +make init + +if ! grep "set \$opentelemetry_context_traceparent '';" conf/nginx.conf > /dev/null; then + echo "failed: opentelemetry_context_traceparent not found in nginx.conf" + exit 1 +fi + +if ! grep "set \$opentelemetry_trace_id '';" conf/nginx.conf > /dev/null; then + echo "failed: opentelemetry_trace_id not found in nginx.conf" + exit 1 +fi + +if ! grep "set \$opentelemetry_span_id '';" conf/nginx.conf > /dev/null; then + echo "failed: opentelemetry_span_id not found in nginx.conf" + exit 1 +fi + + +echo "passed: opentelemetry_set_ngx_var configuration is validated" diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t new file mode 100644 index 000000000000..6171d12f6276 --- /dev/null +++ b/t/plugin/opentelemetry3.t @@ -0,0 +1,170 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->extra_yaml_config) { + my $extra_yaml_config = <<_EOC_; +plugins: + - http-logger + - opentelemetry +plugin_attr: + opentelemetry: + set_ngx_var: true + batch_span_processor: + max_export_batch_size: 1 + inactive_timeout: 0.5 +_EOC_ + $block->set_value("extra_yaml_config", $extra_yaml_config); + } + + my $upstream_server_config = $block->upstream_server_config // <<_EOC_; + set \$opentelemetry_context_traceparent ""; + set \$opentelemetry_trace_id ""; + set \$opentelemetry_span_id ""; + access_log logs/error.log opentelemetry_log; +_EOC_ + + $block->set_value("upstream_server_config", $upstream_server_config); + + my $http_config = $block->http_config // <<_EOC_; + log_format opentelemetry_log '{"time": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}'; +_EOC_ + + $block->set_value("http_config", $http_config); + + if (!$block->extra_init_by_lua) { + my $extra_init_by_lua = <<_EOC_; +-- mock exporter http client +local client = require("opentelemetry.trace.exporter.http_client") +client.do_request = function() + ngx.log(ngx.INFO, "opentelemetry export span") + return "ok" +end +_EOC_ + + $block->set_value("extra_init_by_lua", $extra_init_by_lua); + } + + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + $block; +}); + +run_tests; + +__DATA__ + +=== TEST 1: add plugin metadata +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/http-logger', + ngx.HTTP_PUT, + [[{ + "log_format": { + "opentelemetry_context_traceparent": "$opentelemetry_context_traceparent", + "opentelemetry_trace_id": "$opentelemetry_trace_id", + "opentelemetry_span_id": "$opentelemetry_span_id" + } + }]] + ) + if code >= 300 then + ngx.status = code + return body + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "http-logger": { + "uri": "http://127.0.0.1:1980/log", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "concat_method": "new_line" + }, + "opentelemetry": { + "sampler": { + "name": "always_on" + } + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >=300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed + + + +=== TEST 2: trigger opentelemetry with open set variables +--- request +GET /hello +--- response_body +hello world +--- wait: 1 +--- grep_error_log eval +qr/opentelemetry export span/ +--- grep_error_log_out +opentelemetry export span +--- error_log eval +qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/ + + + +=== TEST 3: trigger opentelemetry with disable set variables +--- yaml_config +plugin_attr: + opentelemetry: + set_ngx_var: false +--- request +GET /hello +--- response_body +hello world +--- error_log eval +qr/request log: \{.*"opentelemetry_context_traceparent":"".*\}/