From 0ddfa3733253a4e37ca0f5d7b7c15505b28d7957 Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Thu, 16 Feb 2023 17:29:42 +0800
Subject: [PATCH 1/7] feat(opentelemetry): add opentelemetry variables

---
 apisix/plugins/opentelemetry.lua        | 16 +++++++++++++++
 docs/en/latest/plugins/opentelemetry.md | 26 +++++++++++++++++++++++++
 docs/zh/latest/plugins/opentelemetry.md | 26 +++++++++++++++++++++++++
 3 files changed, 68 insertions(+)

diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua
index f8013e6f7675..5aae1c266aba 100644
--- a/apisix/plugins/opentelemetry.lua
+++ b/apisix/plugins/opentelemetry.lua
@@ -112,6 +112,11 @@ local attr_schema = {
             },
             default = {},
         },
+        set_ngx_var = {
+          type = "boolean",
+          description = "set nginx variables",
+          default = false,
+        },
     },
 }
 
@@ -332,6 +337,17 @@ function _M.rewrite(conf, api_ctx)
         kind = span_kind.server,
         attributes = attributes,
     })
+
+    if plugin_info.set_ngx_var then
+      local span_context = ctx:span():context()
+      ngx_var.opentelemetry_context_traceparent = string.format("00-%s-%s-%02x",
+                                                                 span_context.trace_id,
+                                                                 span_context.span_id,
+                                                                 span_context.trace_flags)
+      ngx_var.opentelemetry_trace_id = span_context.trace_id
+      ngx_var.opentelemetry_span_id = span_context.span_id
+    end
+
     api_ctx.otel_context_token = ctx:attach()
 
     -- inject trace context into the headers of upstream HTTP request
diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md
index d8ba30234dd5..ce57915353ca 100644
--- a/docs/en/latest/plugins/opentelemetry.md
+++ b/docs/en/latest/plugins/opentelemetry.md
@@ -89,6 +89,32 @@ plugin_attr:
       max_export_batch_size: 2
 ```
 
+## Variables
+
+The following nginx variables are set by OpenTelemetry:
+
+- `opentelemetry_context_traceparent` -  [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), e.g.: `00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01`
+- `opentelemetry_trace_id` - Trace Id of the current span
+- `opentelemetry_span_id` -  Span Id of the current span
+
+How to use variables? you have to add it to your configuration file (`conf/config.yaml`):
+
+```yaml title="./conf/config.yaml"
+    nginx_config:
+      http_server_configuration_snippet: |
+        set $opentelemetry_context_traceparent ""
+        set $opentelemetry_trace_id ""
+        set $opentelemetry_span_id ""
+    http:
+        enable_access_log: true
+        access_log: "/dev/stdout"
+        access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+        access_log_format_escape: json
+    plugin_attr:
+      opentelemetry:
+        set_ngx_var: true
+```
+
 ## Enabling the Plugin
 
 To enable the Plugin, you have to add it to your configuration file (`conf/config.yaml`):
diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md
index d8509eb196e4..a0e69f460783 100644
--- a/docs/zh/latest/plugins/opentelemetry.md
+++ b/docs/zh/latest/plugins/opentelemetry.md
@@ -82,6 +82,32 @@ plugin_attr:
       max_export_batch_size: 2
 ```
 
+## 如何使用变量
+
+以下`nginx`变量是由`opentelemetry` 设置的。
+
+- `opentelemetry_context_traceparent` -  [W3C trace context](https://www.w3.org/TR/trace-context/#trace-context-http-headers-format), 例如：`00-0af7651916cd43dd8448eb211c80319c-b9c7c989f97918e1-01`
+- `opentelemetry_trace_id` - 当前 span 的 trace_id
+- `opentelemetry_span_id` - 当前 span 的 span_id
+
+如何使用？你需要在配置文件（`./conf/config.yaml`）设置如下：
+
+```yaml title="./conf/config.yaml"
+    nginx_config:
+      http_server_configuration_snippet: |
+        set $opentelemetry_context_traceparent ""
+        set $opentelemetry_trace_id ""
+        set $opentelemetry_span_id ""
+    http:
+        enable_access_log: true
+        access_log: "/dev/stdout"
+        access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+        access_log_format_escape: json
+    plugin_attr:
+      opentelemetry:
+        set_ngx_var: true
+```
+
 ## 如何启用
 
 `opentelemetry` 插件默认为禁用状态，你需要在配置文件（`./conf/config.yaml`）中开启该插件：

From 48e3e69b07a47ccdf3217c8764e4fec7d481e9ff Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Fri, 3 Mar 2023 19:11:19 +0800
Subject: [PATCH 2/7] feat: add opentelemetry test

---
 t/plugin/opentelemetry3.t | 168 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 168 insertions(+)
 create mode 100644 t/plugin/opentelemetry3.t

diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t
new file mode 100644
index 000000000000..61d62d085cf4
--- /dev/null
+++ b/t/plugin/opentelemetry3.t
@@ -0,0 +1,168 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+log_level("info");
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!$block->extra_yaml_config) {
+        my $extra_yaml_config = <<_EOC_;
+plugins:
+    - http-logger
+    - opentelemetry
+plugin_attr:
+    opentelemetry:
+        set_ngx_var: true
+        batch_span_processor:
+            max_export_batch_size: 1
+            inactive_timeout: 0.5
+_EOC_
+        $block->set_value("extra_yaml_config", $extra_yaml_config);
+    }
+
+    my $upstream_server_config = $block->upstream_server_config // <<_EOC_;
+    set \$opentelemetry_context_traceparent "";
+    set \$opentelemetry_trace_id "";
+    set \$opentelemetry_span_id "";
+    access_log /tmp/access.log opentelemetry_log; 
+_EOC_
+
+    $block->set_value("upstream_server_config", $upstream_server_config);
+
+    my $http_config = $block->http_config // <<_EOC_;
+    log_format opentelemetry_log '{"time": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}';
+_EOC_
+
+    $block->set_value("http_config", $http_config);
+
+    if (!$block->extra_init_by_lua) {
+        my $extra_init_by_lua = <<_EOC_;
+-- mock exporter http client
+local client = require("opentelemetry.trace.exporter.http_client")
+client.do_request = function()
+    ngx.log(ngx.INFO, "opentelemetry export span")
+    return "ok"
+end
+_EOC_
+
+        $block->set_value("extra_init_by_lua", $extra_init_by_lua);
+    }
+
+
+    if (!$block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    $block;
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: add plugin metadata
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/http-logger',
+                ngx.HTTP_PUT,
+                [[{
+                    "log_format": {
+                        "opentelemetry_context_traceparent": "$opentelemetry_context_traceparent",
+                        "opentelemetry_trace_id": "$opentelemetry_trace_id",
+                        "opentelemetry_span_id": "$opentelemetry_span_id"
+                    }
+                }]]
+                )
+            if code >= 300 then
+                ngx.status = code
+                return body
+            end
+
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "http-logger": {
+                                "uri": "http://127.0.0.1:1980/log",
+                                "batch_max_size": 1,
+                                "max_retry_count": 1,
+                                "retry_delay": 2,
+                                "buffer_duration": 2,
+                                "inactive_timeout": 2,
+                                "concat_method": "new_line"
+                        },
+                        "opentelemetry": {
+                            "sampler": {
+                                "name": "always_on"
+                            }
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1982": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/hello"
+                }]]
+                )
+
+            if code >=300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+
+
+=== TEST 2: trigger opentelemetry with open set variables
+--- request
+GET /hello
+--- response_body
+hello world
+--- wait: 1
+--- grep_error_log eval
+qr/opentelemetry export span/
+--- grep_error_log_out
+opentelemetry export span
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/
+
+
+=== TEST 3: trigger opentelemetry with disable set variables
+--- yaml_config
+plugin_attr:
+    opentelemetry:
+        set_ngx_var: false
+--- request
+GET /hello
+--- response_body
+hello world
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"".*\}/

From 3dfa96bc94be41734440f4a3558818e381729c54 Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Tue, 7 Mar 2023 10:47:29 +0800
Subject: [PATCH 3/7] feat: use ngx_tpl.lua generate variables

---
 apisix/cli/ngx_tpl.lua                  |  8 +++++
 apisix/cli/ops.lua                      |  6 ++++
 apisix/plugins/opentelemetry.lua        |  3 +-
 docs/en/latest/plugins/opentelemetry.md | 23 ++++++------
 docs/zh/latest/plugins/opentelemetry.md | 23 ++++++------
 t/cli/test_opentelemetry_set_ngx_var.sh | 48 +++++++++++++++++++++++++
 t/plugin/opentelemetry3.t               |  2 +-
 7 files changed, 85 insertions(+), 28 deletions(-)
 create mode 100644 t/cli/test_opentelemetry_set_ngx_var.sh

diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index 7083d4201ceb..91386bf0b2d9 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -632,6 +632,14 @@ http {
         proxy_ssl_trusted_certificate {* ssl.ssl_trusted_certificate *};
         {% end %}
 
+        # opentelemetry_set_ngx_var starts
+        {% if opentelemetry_set_ngx_var then %}
+        set $opentelemetry_context_traceparent          '';
+        set $opentelemetry_trace_id                     '';
+        set $opentelemetry_span_id                      '';
+        {% end %}
+        # opentelemetry_set_ngx_var ends
+
         # http server configuration snippet starts
         {% if http_server_configuration_snippet then %}
         {* http_server_configuration_snippet *}
diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index ef069f815010..aa0c27e3226f 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -531,6 +531,11 @@ Please modify "admin_key" in conf/config.yaml .
         end
     end
 
+    local opentelemetry_set_ngx_var
+    if enabled_plugins["opentelemetry"] and yaml_conf.plugin_attr["opentelemetry"] then
+        opentelemetry_set_ngx_var = yaml_conf.plugin_attr["opentelemetry"].set_ngx_var
+    end
+
     -- Using template.render
     local sys_conf = {
         lua_path = env.pkg_path_org,
@@ -551,6 +556,7 @@ Please modify "admin_key" in conf/config.yaml .
         prometheus_server_addr = prometheus_server_addr,
         proxy_mirror_timeouts = proxy_mirror_timeouts,
         conf_server = conf_server,
+        opentelemetry_set_ngx_var = opentelemetry_set_ngx_var
     }
 
     if not yaml_conf.apisix then
diff --git a/apisix/plugins/opentelemetry.lua b/apisix/plugins/opentelemetry.lua
index 5aae1c266aba..0c84fad49da2 100644
--- a/apisix/plugins/opentelemetry.lua
+++ b/apisix/plugins/opentelemetry.lua
@@ -47,6 +47,7 @@ local type    = type
 local pairs   = pairs
 local ipairs  = ipairs
 local unpack  = unpack
+local string_format = string.format
 
 local lrucache = core.lrucache.new({
     type = 'plugin', count = 128, ttl = 24 * 60 * 60,
@@ -340,7 +341,7 @@ function _M.rewrite(conf, api_ctx)
 
     if plugin_info.set_ngx_var then
       local span_context = ctx:span():context()
-      ngx_var.opentelemetry_context_traceparent = string.format("00-%s-%s-%02x",
+      ngx_var.opentelemetry_context_traceparent = string_format("00-%s-%s-%02x",
                                                                  span_context.trace_id,
                                                                  span_context.span_id,
                                                                  span_context.trace_flags)
diff --git a/docs/en/latest/plugins/opentelemetry.md b/docs/en/latest/plugins/opentelemetry.md
index ce57915353ca..30b2738e300e 100644
--- a/docs/en/latest/plugins/opentelemetry.md
+++ b/docs/en/latest/plugins/opentelemetry.md
@@ -100,19 +100,16 @@ The following nginx variables are set by OpenTelemetry:
 How to use variables? you have to add it to your configuration file (`conf/config.yaml`):
 
 ```yaml title="./conf/config.yaml"
-    nginx_config:
-      http_server_configuration_snippet: |
-        set $opentelemetry_context_traceparent ""
-        set $opentelemetry_trace_id ""
-        set $opentelemetry_span_id ""
-    http:
-        enable_access_log: true
-        access_log: "/dev/stdout"
-        access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
-        access_log_format_escape: json
-    plugin_attr:
-      opentelemetry:
-        set_ngx_var: true
+http:
+    enable_access_log: true
+    access_log: "/dev/stdout"
+    access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+    access_log_format_escape: json
+plugins:
+  - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
 ```
 
 ## Enabling the Plugin
diff --git a/docs/zh/latest/plugins/opentelemetry.md b/docs/zh/latest/plugins/opentelemetry.md
index a0e69f460783..740e1ded92b2 100644
--- a/docs/zh/latest/plugins/opentelemetry.md
+++ b/docs/zh/latest/plugins/opentelemetry.md
@@ -93,19 +93,16 @@ plugin_attr:
 如何使用？你需要在配置文件（`./conf/config.yaml`）设置如下：
 
 ```yaml title="./conf/config.yaml"
-    nginx_config:
-      http_server_configuration_snippet: |
-        set $opentelemetry_context_traceparent ""
-        set $opentelemetry_trace_id ""
-        set $opentelemetry_span_id ""
-    http:
-        enable_access_log: true
-        access_log: "/dev/stdout"
-        access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
-        access_log_format_escape: json
-    plugin_attr:
-      opentelemetry:
-        set_ngx_var: true
+http:
+    enable_access_log: true
+    access_log: "/dev/stdout"
+    access_log_format: '{"time": "$time_iso8601","opentelemetry_context_traceparent": "$opentelemetry_context_traceparent","opentelemetry_trace_id": "$opentelemetry_trace_id","opentelemetry_span_id": "$opentelemetry_span_id","remote_addr": "$remote_addr","uri": "$uri"}'
+    access_log_format_escape: json
+plugins:
+  - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
 ```
 
 ## 如何启用
diff --git a/t/cli/test_opentelemetry_set_ngx_var.sh b/t/cli/test_opentelemetry_set_ngx_var.sh
new file mode 100644
index 000000000000..23ac5c7fd35c
--- /dev/null
+++ b/t/cli/test_opentelemetry_set_ngx_var.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+echo '
+plugins:
+    - opentelemetry
+plugin_attr:
+  opentelemetry:
+    set_ngx_var: true
+' > conf/config.yaml
+
+make init
+
+if ! grep "set \$opentelemetry_context_traceparent          '';" conf/nginx.conf > /dev/null; then
+    echo "failed: opentelemetry_context_traceparent not found in nginx.conf"
+    exit 1
+fi
+
+if ! grep "set \$opentelemetry_trace_id                     '';" conf/nginx.conf > /dev/null; then
+    echo "failed: opentelemetry_trace_id not found in nginx.conf"
+    exit 1
+fi
+
+if ! grep "set \$opentelemetry_span_id                      '';" conf/nginx.conf > /dev/null; then
+    echo "failed: $opentelemetry_span_id not found in nginx.conf"
+    exit 1
+fi
+
+
+echo "passed: opentelemetry_set_ngx_var configuration is validated"
diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t
index 61d62d085cf4..db4fce7f45d8 100644
--- a/t/plugin/opentelemetry3.t
+++ b/t/plugin/opentelemetry3.t
@@ -44,7 +44,7 @@ _EOC_
     set \$opentelemetry_context_traceparent "";
     set \$opentelemetry_trace_id "";
     set \$opentelemetry_span_id "";
-    access_log /tmp/access.log opentelemetry_log; 
+    access_log logs/error.log opentelemetry_log; 
 _EOC_
 
     $block->set_value("upstream_server_config", $upstream_server_config);

From 807663e5f7fdf698f2564d607b0525937fc27873 Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Tue, 7 Mar 2023 11:08:33 +0800
Subject: [PATCH 4/7] fix: code lint error

---
 t/plugin/opentelemetry3.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t
index db4fce7f45d8..d551956a2bca 100644
--- a/t/plugin/opentelemetry3.t
+++ b/t/plugin/opentelemetry3.t
@@ -44,7 +44,7 @@ _EOC_
     set \$opentelemetry_context_traceparent "";
     set \$opentelemetry_trace_id "";
     set \$opentelemetry_span_id "";
-    access_log logs/error.log opentelemetry_log; 
+    access_log logs/error.log opentelemetry_log;
 _EOC_
 
     $block->set_value("upstream_server_config", $upstream_server_config);

From c7fb26a6cac48294beb6300c3eb5e774cd6e06d9 Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Tue, 14 Mar 2023 10:23:16 +0800
Subject: [PATCH 5/7] fix: ci test error

---
 apisix/cli/ops.lua                      |   1 -
 t/cli/test_opentelemetry_set_ngx_var.sh |   2 +-
 t/plugin/opentelemetry3.t               | 120 ++++++++----------------
 3 files changed, 38 insertions(+), 85 deletions(-)
 mode change 100644 => 100755 t/cli/test_opentelemetry_set_ngx_var.sh

diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
index aa0c27e3226f..36ea47af3eb8 100644
--- a/apisix/cli/ops.lua
+++ b/apisix/cli/ops.lua
@@ -535,7 +535,6 @@ Please modify "admin_key" in conf/config.yaml .
     if enabled_plugins["opentelemetry"] and yaml_conf.plugin_attr["opentelemetry"] then
         opentelemetry_set_ngx_var = yaml_conf.plugin_attr["opentelemetry"].set_ngx_var
     end
-
     -- Using template.render
     local sys_conf = {
         lua_path = env.pkg_path_org,
diff --git a/t/cli/test_opentelemetry_set_ngx_var.sh b/t/cli/test_opentelemetry_set_ngx_var.sh
old mode 100644
new mode 100755
index 23ac5c7fd35c..8db6ea420db2
--- a/t/cli/test_opentelemetry_set_ngx_var.sh
+++ b/t/cli/test_opentelemetry_set_ngx_var.sh
@@ -40,7 +40,7 @@ if ! grep "set \$opentelemetry_trace_id                     '';" conf/nginx.conf
 fi
 
 if ! grep "set \$opentelemetry_span_id                      '';" conf/nginx.conf > /dev/null; then
-    echo "failed: $opentelemetry_span_id not found in nginx.conf"
+    echo "failed: opentelemetry_span_id not found in nginx.conf"
     exit 1
 fi
 
diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t
index d551956a2bca..b6da488b1c68 100644
--- a/t/plugin/opentelemetry3.t
+++ b/t/plugin/opentelemetry3.t
@@ -17,18 +17,24 @@
 
 use t::APISIX 'no_plan';
 
-repeat_each(1);
-no_long_string();
-no_root_location();
-log_level("info");
+log_level('debug');
 
 add_block_preprocessor(sub {
     my ($block) = @_;
 
     if (!$block->extra_yaml_config) {
         my $extra_yaml_config = <<_EOC_;
+nginx_config:
+  http_server_configuration_snippet: |
+    set \$opentelemetry_context_traceparent ""
+    set \$opentelemetry_trace_id ""
+    set \$opentelemetry_span_id ""
+  http:
+    enable_access_log: true
+    access_log: "/tmp/access.log"
+    access_log_format: '{"timestamp": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}'
+    access_log_format_escape: json
 plugins:
-    - http-logger
     - opentelemetry
 plugin_attr:
     opentelemetry:
@@ -40,35 +46,6 @@ _EOC_
         $block->set_value("extra_yaml_config", $extra_yaml_config);
     }
 
-    my $upstream_server_config = $block->upstream_server_config // <<_EOC_;
-    set \$opentelemetry_context_traceparent "";
-    set \$opentelemetry_trace_id "";
-    set \$opentelemetry_span_id "";
-    access_log logs/error.log opentelemetry_log;
-_EOC_
-
-    $block->set_value("upstream_server_config", $upstream_server_config);
-
-    my $http_config = $block->http_config // <<_EOC_;
-    log_format opentelemetry_log '{"time": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}';
-_EOC_
-
-    $block->set_value("http_config", $http_config);
-
-    if (!$block->extra_init_by_lua) {
-        my $extra_init_by_lua = <<_EOC_;
--- mock exporter http client
-local client = require("opentelemetry.trace.exporter.http_client")
-client.do_request = function()
-    ngx.log(ngx.INFO, "opentelemetry export span")
-    return "ok"
-end
-_EOC_
-
-        $block->set_value("extra_init_by_lua", $extra_init_by_lua);
-    }
-
-
     if (!$block->request) {
         $block->set_value("request", "GET /t");
     }
@@ -80,39 +57,15 @@ run_tests;
 
 __DATA__
 
-=== TEST 1: add plugin metadata
+=== TEST 1: add plugin
 --- config
     location /t {
         content_by_lua_block {
             local t = require("lib.test_admin").test
-            local code, body = t('/apisix/admin/plugin_metadata/http-logger',
-                ngx.HTTP_PUT,
-                [[{
-                    "log_format": {
-                        "opentelemetry_context_traceparent": "$opentelemetry_context_traceparent",
-                        "opentelemetry_trace_id": "$opentelemetry_trace_id",
-                        "opentelemetry_span_id": "$opentelemetry_span_id"
-                    }
-                }]]
-                )
-            if code >= 300 then
-                ngx.status = code
-                return body
-            end
-
             local code, body = t('/apisix/admin/routes/1',
                 ngx.HTTP_PUT,
                 [[{
                     "plugins": {
-                        "http-logger": {
-                                "uri": "http://127.0.0.1:1980/log",
-                                "batch_max_size": 1,
-                                "max_retry_count": 1,
-                                "retry_delay": 2,
-                                "buffer_duration": 2,
-                                "inactive_timeout": 2,
-                                "concat_method": "new_line"
-                        },
                         "opentelemetry": {
                             "sampler": {
                                 "name": "always_on"
@@ -121,15 +74,15 @@ __DATA__
                     },
                     "upstream": {
                         "nodes": {
-                            "127.0.0.1:1982": 1
+                            "127.0.0.1:1980": 1
                         },
                         "type": "roundrobin"
                     },
-                    "uri": "/hello"
+                    "uri": "/opentracing"
                 }]]
                 )
 
-            if code >=300 then
+            if code >= 300 then
                 ngx.status = code
             end
             ngx.say(body)
@@ -141,28 +94,29 @@ GET /t
 passed
 
 
-=== TEST 2: trigger opentelemetry with open set variables
+
+=== TEST 2: trigger opentelemetry
 --- request
-GET /hello
+GET /opentracing
+--- access_log
 --- response_body
-hello world
---- wait: 1
---- grep_error_log eval
-qr/opentelemetry export span/
---- grep_error_log_out
-opentelemetry export span
---- error_log eval
-qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/
-
-
-=== TEST 3: trigger opentelemetry with disable set variables
---- yaml_config
-plugin_attr:
-    opentelemetry:
-        set_ngx_var: false
+opentracing
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: allow integer worker processes
+--- config
+    location /t {
+        content_by_lua_block {
+            local config = require("apisix.core").config.local_conf()
+        }
+    }
+--- extra_yaml_config
+nginx_config:
+    
 --- request
-GET /hello
+GET /t
 --- response_body
-hello world
---- error_log eval
-qr/request log: \{.*"opentelemetry_context_traceparent":"".*\}/
+1

From ecaf2c29e48101c6b4bc45cfdf83c165936e9add Mon Sep 17 00:00:00 2001
From: yaokl <yaokl@ickey.cn>
Date: Tue, 14 Mar 2023 10:29:16 +0800
Subject: [PATCH 6/7] fix: ci lint error

---
 t/plugin/opentelemetry3.t | 114 +++++++++++++++++++++++++++-----------
 1 file changed, 81 insertions(+), 33 deletions(-)

diff --git a/t/plugin/opentelemetry3.t b/t/plugin/opentelemetry3.t
index b6da488b1c68..6171d12f6276 100644
--- a/t/plugin/opentelemetry3.t
+++ b/t/plugin/opentelemetry3.t
@@ -17,24 +17,18 @@
 
 use t::APISIX 'no_plan';
 
-log_level('debug');
+repeat_each(1);
+no_long_string();
+no_root_location();
+log_level("info");
 
 add_block_preprocessor(sub {
     my ($block) = @_;
 
     if (!$block->extra_yaml_config) {
         my $extra_yaml_config = <<_EOC_;
-nginx_config:
-  http_server_configuration_snippet: |
-    set \$opentelemetry_context_traceparent ""
-    set \$opentelemetry_trace_id ""
-    set \$opentelemetry_span_id ""
-  http:
-    enable_access_log: true
-    access_log: "/tmp/access.log"
-    access_log_format: '{"timestamp": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}'
-    access_log_format_escape: json
 plugins:
+    - http-logger
     - opentelemetry
 plugin_attr:
     opentelemetry:
@@ -46,6 +40,35 @@ _EOC_
         $block->set_value("extra_yaml_config", $extra_yaml_config);
     }
 
+    my $upstream_server_config = $block->upstream_server_config // <<_EOC_;
+    set \$opentelemetry_context_traceparent "";
+    set \$opentelemetry_trace_id "";
+    set \$opentelemetry_span_id "";
+    access_log logs/error.log opentelemetry_log;
+_EOC_
+
+    $block->set_value("upstream_server_config", $upstream_server_config);
+
+    my $http_config = $block->http_config // <<_EOC_;
+    log_format opentelemetry_log '{"time": "\$time_iso8601","opentelemetry_context_traceparent": "\$opentelemetry_context_traceparent","opentelemetry_trace_id": "\$opentelemetry_trace_id","opentelemetry_span_id": "\$opentelemetry_span_id","remote_addr": "\$remote_addr","uri": "\$uri"}';
+_EOC_
+
+    $block->set_value("http_config", $http_config);
+
+    if (!$block->extra_init_by_lua) {
+        my $extra_init_by_lua = <<_EOC_;
+-- mock exporter http client
+local client = require("opentelemetry.trace.exporter.http_client")
+client.do_request = function()
+    ngx.log(ngx.INFO, "opentelemetry export span")
+    return "ok"
+end
+_EOC_
+
+        $block->set_value("extra_init_by_lua", $extra_init_by_lua);
+    }
+
+
     if (!$block->request) {
         $block->set_value("request", "GET /t");
     }
@@ -57,15 +80,39 @@ run_tests;
 
 __DATA__
 
-=== TEST 1: add plugin
+=== TEST 1: add plugin metadata
 --- config
     location /t {
         content_by_lua_block {
             local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/http-logger',
+                ngx.HTTP_PUT,
+                [[{
+                    "log_format": {
+                        "opentelemetry_context_traceparent": "$opentelemetry_context_traceparent",
+                        "opentelemetry_trace_id": "$opentelemetry_trace_id",
+                        "opentelemetry_span_id": "$opentelemetry_span_id"
+                    }
+                }]]
+                )
+            if code >= 300 then
+                ngx.status = code
+                return body
+            end
+
             local code, body = t('/apisix/admin/routes/1',
                 ngx.HTTP_PUT,
                 [[{
                     "plugins": {
+                        "http-logger": {
+                                "uri": "http://127.0.0.1:1980/log",
+                                "batch_max_size": 1,
+                                "max_retry_count": 1,
+                                "retry_delay": 2,
+                                "buffer_duration": 2,
+                                "inactive_timeout": 2,
+                                "concat_method": "new_line"
+                        },
                         "opentelemetry": {
                             "sampler": {
                                 "name": "always_on"
@@ -74,15 +121,15 @@ __DATA__
                     },
                     "upstream": {
                         "nodes": {
-                            "127.0.0.1:1980": 1
+                            "127.0.0.1:1982": 1
                         },
                         "type": "roundrobin"
                     },
-                    "uri": "/opentracing"
+                    "uri": "/hello"
                 }]]
                 )
 
-            if code >= 300 then
+            if code >=300 then
                 ngx.status = code
             end
             ngx.say(body)
@@ -95,28 +142,29 @@ passed
 
 
 
-=== TEST 2: trigger opentelemetry
+=== TEST 2: trigger opentelemetry with open set variables
 --- request
-GET /opentracing
---- access_log
+GET /hello
 --- response_body
-opentracing
---- no_error_log
-[error]
+hello world
+--- wait: 1
+--- grep_error_log eval
+qr/opentelemetry export span/
+--- grep_error_log_out
+opentelemetry export span
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"00-\w{32}-\w{16}-01".*\}/
 
 
 
-=== TEST 3: allow integer worker processes
---- config
-    location /t {
-        content_by_lua_block {
-            local config = require("apisix.core").config.local_conf()
-        }
-    }
---- extra_yaml_config
-nginx_config:
-    
+=== TEST 3: trigger opentelemetry with disable set variables
+--- yaml_config
+plugin_attr:
+    opentelemetry:
+        set_ngx_var: false
 --- request
-GET /t
+GET /hello
 --- response_body
-1
+hello world
+--- error_log eval
+qr/request log: \{.*"opentelemetry_context_traceparent":"".*\}/

From 0b3c38df3c0850bbf82c5da8e8a33200088b0b9e Mon Sep 17 00:00:00 2001
From: monkeyDluffy6017 <monkeydluffy6017@gmail.com>
Date: Thu, 21 Sep 2023 22:39:27 +0800
Subject: [PATCH 7/7] refactor: add configuration to config-default.yaml

---
 conf/config-default.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 05dc50ad72d1..f15f6d1df604 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -570,6 +570,7 @@ plugin_attr:          # Plugin attributes
       inactive_timeout: 1           # Set the timeout for spans to wait in the export queue before being sent,
                                     # if the queue is not full.
       max_export_batch_size: 16     # Set the maximum number of spans to include in each batch sent to the
+    set_ngx_var: false              # export opentelemetry variables to nginx variables
                                     # OpenTelemetry collector.
   prometheus:                               # Plugin: prometheus
     export_uri: /apisix/prometheus/metrics  # Set the URI for the Prometheus metrics endpoint.